chii
9b689b2476
公开页抓取改为 Chrome 136 自洽身份 + 每节点 SQLite 养罐，Trafilatura 走 curl_cffi；MCP/README/接手说明与 09-02 现网复测对齐，避免消费方继续抄过期的站点三分表。 Co-authored-by: Cursor <cursoragent@cursor.com>
116 lines
3.5 KiB
Go
116 lines
3.5 KiB
Go
// searx_parse.go：searxng json 样本解析（RawResult.Extra → 中间条目）。
|
||
//
|
||
// 形状权威：bench/searxng-cn/samples/t1-1.excerpt.json（title/url/content/engine
|
||
// + publishedDate 可选 + unresponsive_engines）。与 dock 包的响应解码解耦：
|
||
// 模版层只认 Extra 里的 searx_results。
|
||
package scheduler
|
||
|
||
import (
|
||
"encoding/json"
|
||
"net/url"
|
||
"strings"
|
||
"time"
|
||
|
||
"onesvm.com/onesvm/browser-server/internal/contract"
|
||
)
|
||
|
||
// searxItem 搜索条目（与 dock 侧响应形状一致；publishedDate 为引擎返回字符串）。
|
||
type searxItem struct {
|
||
Title string `json:"title"`
|
||
URL string `json:"url"`
|
||
Content string `json:"content"`
|
||
Engine string `json:"engine"`
|
||
PublishedDate *string `json:"publishedDate"`
|
||
}
|
||
|
||
// parseSearxResults 从 RawResult.Extra 解析结果条目与 unresponsive 引擎名。
|
||
func parseSearxResults(raw *contract.RawResult) ([]searxItem, []string) {
|
||
var items []searxItem
|
||
if v, ok := raw.Extra["searx_results"]; ok {
|
||
if b, err := json.Marshal(v); err == nil {
|
||
_ = json.Unmarshal(b, &items)
|
||
}
|
||
}
|
||
var unres []string
|
||
if v, ok := raw.Extra["unresponsive_engines"]; ok {
|
||
var pairs []json.RawMessage
|
||
if b, err := json.Marshal(v); err == nil && json.Unmarshal(b, &pairs) == nil {
|
||
for _, p := range pairs {
|
||
var pair [2]string
|
||
if json.Unmarshal(p, &pair) == nil && len(pair) == 2 {
|
||
unres = append(unres, pair[0]+"("+pair[1]+")")
|
||
}
|
||
}
|
||
}
|
||
}
|
||
return items, unres
|
||
}
|
||
|
||
// isJunkSearchHit 丢掉搜索引擎吸到的站点壳页（Amazon 首页/账号/影音、
|
||
// 无 snippet 的门户根路径）。先过滤再裁 max_results，避免 16 条里 10 条不可引用。
|
||
func isJunkSearchHit(rawURL, title, content string) bool {
|
||
u, err := url.Parse(rawURL)
|
||
if err != nil || u.Hostname() == "" {
|
||
return true
|
||
}
|
||
host := strings.ToLower(u.Hostname())
|
||
path := strings.ToLower(strings.TrimSuffix(u.Path, "/"))
|
||
if junkSearchHosts[host] {
|
||
return true
|
||
}
|
||
if isAmazonHost(host) && isAmazonShellPath(path) {
|
||
return true
|
||
}
|
||
snippet := strings.TrimSpace(content)
|
||
if snippet == "" && isBareSiteRoot(path) {
|
||
return true
|
||
}
|
||
if strings.Contains(snippet, "由于此网站的设置，我们无法提供该页面的具体描述") {
|
||
return true
|
||
}
|
||
if strings.TrimSpace(title) == "" {
|
||
return true
|
||
}
|
||
return false
|
||
}
|
||
|
||
// junkSearchHosts 明确的导航/非内容域（搜索发现层无引用价值）。
|
||
var junkSearchHosts = map[string]bool{
|
||
"music.amazon.com": true,
|
||
"www.primevideo.com": true,
|
||
"primevideo.com": true,
|
||
"pharmacy.amazon.com": true,
|
||
"hiring.amazon.com": true,
|
||
}
|
||
|
||
func isAmazonHost(host string) bool {
|
||
return host == "www.amazon.com" || host == "amazon.com" ||
|
||
host == "www.amazon.cn" || host == "amazon.cn" ||
|
||
host == "gs.amazon.cn" || host == "globalstore.amazon.cn"
|
||
}
|
||
|
||
func isAmazonShellPath(path string) bool {
|
||
if path == "" || path == "/-/zh" || path == "/ref=nav_logo" {
|
||
return true
|
||
}
|
||
if strings.HasPrefix(path, "/your-account") || strings.HasPrefix(path, "/gp/css") ||
|
||
strings.HasPrefix(path, "/gp/yourstore") || strings.HasPrefix(path, "/ap/signin") {
|
||
return true
|
||
}
|
||
return false
|
||
}
|
||
|
||
func isBareSiteRoot(path string) bool {
|
||
return path == "" || path == "/"
|
||
}
|
||
|
||
// publishedAtOf 引擎时间串 → 东八区 Time（多格式容错；失败给 nil 由调用方置空）。
|
||
func publishedAtOf(s string) (contract.Time, bool) {
|
||
layouts := []string{time.RFC3339, "2006-01-02 15:04:05", "2006-01-02"}
|
||
for _, l := range layouts {
|
||
if t, err := time.Parse(l, s); err == nil {
|
||
return contract.NewTime(t), true
|
||
}
|
||
}
|
||
return contract.Time{}, false
|
||
}
|