// searx_parse.go:searxng json 样本解析(RawResult.Extra → 中间条目)。 // // 形状权威:bench/searxng-cn/samples/t1-1.excerpt.json(title/url/content/engine // + publishedDate 可选 + unresponsive_engines)。与 dock 包的响应解码解耦: // 模版层只认 Extra 里的 searx_results。 package scheduler import ( "encoding/json" "net/url" "strings" "time" "onesvm.com/onesvm/browser-server/internal/contract" ) // searxItem 搜索条目(与 dock 侧响应形状一致;publishedDate 为引擎返回字符串)。 type searxItem struct { Title string `json:"title"` URL string `json:"url"` Content string `json:"content"` Engine string `json:"engine"` PublishedDate *string `json:"publishedDate"` } // parseSearxResults 从 RawResult.Extra 解析结果条目与 unresponsive 引擎名。 func parseSearxResults(raw *contract.RawResult) ([]searxItem, []string) { var items []searxItem if v, ok := raw.Extra["searx_results"]; ok { if b, err := json.Marshal(v); err == nil { _ = json.Unmarshal(b, &items) } } var unres []string if v, ok := raw.Extra["unresponsive_engines"]; ok { var pairs []json.RawMessage if b, err := json.Marshal(v); err == nil && json.Unmarshal(b, &pairs) == nil { for _, p := range pairs { var pair [2]string if json.Unmarshal(p, &pair) == nil && len(pair) == 2 { unres = append(unres, pair[0]+"("+pair[1]+")") } } } } return items, unres } // isJunkSearchHit 丢掉搜索引擎吸到的站点壳页(Amazon 首页/账号/影音、 // 无 snippet 的门户根路径)。先过滤再裁 max_results,避免 16 条里 10 条不可引用。 func isJunkSearchHit(rawURL, title, content string) bool { u, err := url.Parse(rawURL) if err != nil || u.Hostname() == "" { return true } host := strings.ToLower(u.Hostname()) path := strings.ToLower(strings.TrimSuffix(u.Path, "/")) if junkSearchHosts[host] { return true } if isAmazonHost(host) && isAmazonShellPath(path) { return true } snippet := strings.TrimSpace(content) if snippet == "" && isBareSiteRoot(path) { return true } if strings.Contains(snippet, "由于此网站的设置,我们无法提供该页面的具体描述") { return true } if strings.TrimSpace(title) == "" { return true } return false } // junkSearchHosts 明确的导航/非内容域(搜索发现层无引用价值)。 var junkSearchHosts = map[string]bool{ "music.amazon.com": true, "www.primevideo.com": true, "primevideo.com": true, "pharmacy.amazon.com": true, "hiring.amazon.com": true, } func isAmazonHost(host string) bool { return host == "www.amazon.com" || host == "amazon.com" || host == "www.amazon.cn" || host == "amazon.cn" || host == "gs.amazon.cn" || host == "globalstore.amazon.cn" } func isAmazonShellPath(path string) bool { if path == "" || path == "/-/zh" || path == "/ref=nav_logo" { return true } if strings.HasPrefix(path, "/your-account") || strings.HasPrefix(path, "/gp/css") || strings.HasPrefix(path, "/gp/yourstore") || strings.HasPrefix(path, "/ap/signin") { return true } return false } func isBareSiteRoot(path string) bool { return path == "" || path == "/" } // publishedAtOf 引擎时间串 → 东八区 Time(多格式容错;失败给 nil 由调用方置空)。 func publishedAtOf(s string) (contract.Time, bool) { layouts := []string{time.RFC3339, "2006-01-02 15:04:05", "2006-01-02"} for _, l := range layouts { if t, err := time.Parse(l, s); err == nil { return contract.NewTime(t), true } } return contract.Time{}, false }