diff --git a/docs/mcp-usage-20260901.md b/docs/mcp-usage-20260901.md index 9bfab63..7180e5f 100644 --- a/docs/mcp-usage-20260901.md +++ b/docs/mcp-usage-20260901.md @@ -2,7 +2,7 @@ type: runbook status: active created: 2026-09-01 -updated: 2026-09-02 +updated: 2026-09-08 step: P3 / plan-20260901-02 --- @@ -240,6 +240,8 @@ step: P3 / plan-20260901-02 **国内(`region=domestic`)**——`searxng-cn` 聚合百度 / 必应中国 / 360 / 搜狗(已去掉 wikipedia,避免百科噪声占位)。 +模版层对 `search` 结果做规则清洗(无模型):标题/摘要含【推广】/Sponsored 等广告标记会丢;同一可注册域(eTLD+1)最多保留 1 条,避免 1688 等站霸屏。`/tag` 专题页与有 snippet 的门户首页不杀。 + | 引擎 | 现网表现 | 含义 | |---|---|---| | 必应中国 | 稳定出结果 | 主力 | @@ -252,10 +254,10 @@ step: P3 / plan-20260901-02 | query | 耗时 | 条数 | 能用的 | |---|---:|---:|---| | `跨境电商 出口退税 2026` | 4.7s | 6 | 国税/财政部退运免税公告、海外仓「离境即退税」、12366;雨果/社区首页是噪声 | -| `背背佳 矫正 市场 供应链 品牌` | 1.6s | 8 | 百科 snippet(可孚 / 杜国楹)、官网、搜狐测评;**1688 批发占一半** | +| `背背佳 矫正 市场 供应链 品牌` | 1.6s | 8 | 百科 snippet(可孚 / 杜国楹)、官网、搜狐测评;上线同域限额前 **1688 批发曾占一半**(现同域最多 1 条) | | `膳食补充剂 跨境 监管 市场 2026` | 9.9s | 8 | 商务部 FDA 扣留、东财/中华网抖音治理、海关办法;百科「膳食」与指南 PDF 是偏题 | -→ 常规政策/监管词能出一手链。供应链词会被批发站淹没。突发猛打时结果变少(上游反爬,不是故障)。 +→ 常规政策/监管词能出一手链。供应链词在同域限额后不再被批发站占满;突发猛打时结果变少(上游反爬,不是故障)。 **国外(`region=overseas`)**——**当前仅 Bing 可用**。Google / DuckDuckGo / Brave / Startpage / Qwant 在现有数据中心代理出口下全部 CAPTCHA/429。**禁止假定多源聚合。** diff --git a/server/internal/contract/contract_test.go b/server/internal/contract/contract_test.go index d284bb1..3db6d43 100644 --- a/server/internal/contract/contract_test.go +++ b/server/internal/contract/contract_test.go @@ -80,6 +80,42 @@ func TestEnvelopeSearchGolden(t *testing.T) { } } +// TestEnvelopeSearchRoundtrip 反序列化必须还原 Data,再 Marshal 不得丢掉 results。 +func TestEnvelopeSearchRoundtrip(t *testing.T) { + src := Envelope{ + OK: true, Kind: "search", RequestID: "r-rt", TookMs: 12, + Usage: Usage{Credits: 1, Engine: "searxng-cn"}, + Data: &SearchPayload{ + Query: "往返", + Results: []SearchResult{{ + ID: "r1", Title: "标题", URL: "https://example.com", Content: "摘要", Score: 0.9, Engine: "bing", + }}, + }, + } + b, err := json.Marshal(src) + if err != nil { + t.Fatal(err) + } + var got Envelope + if err := json.Unmarshal(b, &got); err != nil { + t.Fatal(err) + } + p, ok := got.Data.(*SearchPayload) + if !ok || p == nil || len(p.Results) != 1 || p.Results[0].Title != "标题" { + t.Fatalf("往返丢掉 results: %#v", got.Data) + } + b2, err := json.Marshal(got) + if err != nil { + t.Fatal(err) + } + var wire struct { + Results []SearchResult `json:"results"` + } + if err := json.Unmarshal(b2, &wire); err != nil || len(wire.Results) != 1 { + t.Fatalf("再序列化丢掉 results: %s", b2) + } +} + // TestEnvelopeReadShape read 字段齐备性(mcp-usage §2.2)。 func TestEnvelopeReadShape(t *testing.T) { md := "正文" diff --git a/server/internal/contract/json.go b/server/internal/contract/json.go index 8ef1510..ad2bd45 100644 --- a/server/internal/contract/json.go +++ b/server/internal/contract/json.go @@ -7,3 +7,72 @@ import "encoding/json" func jsonMarshal(v any) ([]byte, error) { return json.Marshal(v) } + +// UnmarshalJSON 从平铺 wire 还原 Data。缺此方法时 encoding/json 只填顶层字段, +// Data 恒为 nil;再 Marshal 会丢掉 results/markdown(gateway 缓存命中改写即踩中)。 +func (e *Envelope) UnmarshalJSON(b []byte) error { + var top envelopeTop + if err := json.Unmarshal(b, &top); err != nil { + return err + } + e.OK = top.OK + e.Kind = top.Kind + e.RequestID = top.RequestID + e.TookMs = top.TookMs + e.Usage = top.Usage + e.Provenance = top.Provenance + e.Error = top.Error + switch top.Kind { + case "search": + var w searchWire + if err := json.Unmarshal(b, &w); err != nil { + return err + } + p := &SearchPayload{Query: w.Query, Answer: w.Answer, Results: w.Results} + p.EnsureEmptySlice() + e.Data = p + case "read": + var w readWire + if err := json.Unmarshal(b, &w); err != nil { + return err + } + p := readPayloadFromWire(w) + p.EnsureEmptySlice() + e.Data = p + } + return nil +} + +func readPayloadFromWire(w readWire) *ReadPayload { + p := &ReadPayload{ + Description: w.Description, + HTML: w.HTML, + ScreenshotURL: w.ScreenshotURL, + Extracted: w.Extracted, + Links: w.Links, + Images: w.Images, + Warnings: w.Warnings, + } + if w.URL != nil { + p.URL = *w.URL + } + if w.FinalURL != nil { + p.FinalURL = *w.FinalURL + } + if w.Title != nil { + p.Title = *w.Title + } + if w.Markdown != nil { + p.Markdown = *w.Markdown + } + if w.Truncated != nil { + p.Truncated = *w.Truncated + } + if w.CharCount != nil { + p.CharCount = *w.CharCount + } + if w.Metadata != nil { + p.Metadata = *w.Metadata + } + return p +} diff --git a/server/internal/gateway/cache.go b/server/internal/gateway/cache.go index 14995c4..30baec3 100644 --- a/server/internal/gateway/cache.go +++ b/server/internal/gateway/cache.go @@ -102,8 +102,11 @@ func rewriteCached(body []byte, now time.Time) []byte { return b } -// Put 写缓存(仅 search 成功信封)。超容量按 lastUsed 淘汰最旧。 +// Put 写缓存(仅 search 成功且非空结果)。超容量按 lastUsed 淘汰最旧。 func (c *SearchCache) Put(in *contract.SearchInput, body []byte) { + if isEmptySearchEnvelope(body) { + return + } c.mu.Lock() defer c.mu.Unlock() now := c.now() diff --git a/server/internal/gateway/pipelinecore.go b/server/internal/gateway/pipelinecore.go index ded6e07..44c4855 100644 --- a/server/internal/gateway/pipelinecore.go +++ b/server/internal/gateway/pipelinecore.go @@ -113,8 +113,8 @@ func (s *Server) pipeline(rc runCtx) result { // ok=false → Release(失败回收预扣)。scheduler 不碰 quota 表(单写者纪律)。 s.settleQuota(rc, isOKEnvelope(rawEnv)) - // 10. 缓存写入(仅 search 成功) - if rc.Intent == IntentSearch && isOKEnvelope(rawEnv) { + // 10. 缓存写入(仅 search 成功且非空结果)。零结果不缓存:引擎抖一次会把空信封冻 300s。 + if rc.Intent == IntentSearch && isOKEnvelope(rawEnv) && !isEmptySearchEnvelope(rawEnv) { s.deps.Cache.Put(rc.Search, rawEnv) } return result{Body: rawEnv} diff --git a/server/internal/gateway/util.go b/server/internal/gateway/util.go index c94d189..a4e69a8 100644 --- a/server/internal/gateway/util.go +++ b/server/internal/gateway/util.go @@ -34,3 +34,16 @@ func isOKEnvelope(raw []byte) bool { } return probe.OK } + +// isEmptySearchEnvelope 判断 search 信封是否零结果。 +// SearXNG 引擎全挂仍可能 HTTP 200 + ok=true + results=[];写进 300s 缓存会把瞬时空结果冻成「消费端空值」。 +func isEmptySearchEnvelope(raw []byte) bool { + var probe struct { + OK bool `json:"ok"` + Results []json.RawMessage `json:"results"` + } + if err := jsonUnmarshal(raw, &probe); err != nil { + return true + } + return probe.OK && len(probe.Results) == 0 +} diff --git a/server/internal/gateway/v1_test.go b/server/internal/gateway/v1_test.go index 730e282..a69fda2 100644 --- a/server/internal/gateway/v1_test.go +++ b/server/internal/gateway/v1_test.go @@ -27,6 +27,14 @@ func searchOKEnvelope(reqID string) map[string]any { } } +func searchHitEnvelope(reqID string) map[string]any { + env := searchOKEnvelope(reqID) + env["results"] = []any{ + map[string]any{"id": "r1", "title": "示例结果", "url": "https://example.com", "content": "摘要", "score": 0.9, "engine": "bing"}, + } + return env +} + // TestV1SearchSuccess 200 + 信封透传 + X-Session-Remaining 头。 func TestV1SearchSuccess(t *testing.T) { e := newTestEnv(t) @@ -374,11 +382,11 @@ func TestSearchCacheHit(t *testing.T) { e := newTestEnv(t) e.sched.autoComplete = true e.sched.doneEnv = func(rid string) json.RawMessage { - if _, ok := jobsIntent(e.sched, rid, "read"); ok { + if match, found := jobsIntent(e.sched, rid, "read"); found && match { b, _ := json.Marshal(readOKEnvelope(rid)) return b } - b, _ := json.Marshal(searchOKEnvelope(rid)) + b, _ := json.Marshal(searchHitEnvelope(rid)) return b } rec := e.postV1(t, "/v1/search", `{"query":"缓存测试","region":"domestic"}`) @@ -404,6 +412,15 @@ func TestSearchCacheHit(t *testing.T) { if env.Usage.Credits != 0 { t.Fatalf("缓存命中 credits 应 0: %d", env.Usage.Credits) } + var hitWire struct { + Results []contract.SearchResult `json:"results"` + } + if err := json.Unmarshal(rec2.Body.Bytes(), &hitWire); err != nil { + t.Fatal(err) + } + if len(hitWire.Results) == 0 { + t.Fatalf("缓存命中不得丢掉 results: %s", rec2.Body.String()) + } // read 不缓存:同参数两次都出网 for i := 0; i < 2; i++ { recR := e.postV1(t, "/v1/read", `{"url":"https://example.com/page"}`) @@ -416,6 +433,37 @@ func TestSearchCacheHit(t *testing.T) { } } +// TestSearchEmptyNotCached 零结果信封不进 300s 缓存:引擎抖空后下次必须再出网。 +func TestSearchEmptyNotCached(t *testing.T) { + e := newTestEnv(t) + e.sched.autoComplete = true + e.sched.doneEnv = func(rid string) json.RawMessage { + empty := searchOKEnvelope(rid) + empty["results"] = []any{} + b, _ := json.Marshal(empty) + return b + } + rec := e.postV1(t, "/v1/search", `{"query":"空结果不缓存","region":"domestic"}`) + if rec.Code != http.StatusOK { + t.Fatalf("首发应 200: %d %s", rec.Code, rec.Body.String()) + } + first := e.sched.enqueues.Load() + rec2 := e.postV1(t, "/v1/search", `{"query":"空结果不缓存","region":"domestic"}`) + if rec2.Code != http.StatusOK { + t.Fatalf("二发应 200: %d %s", rec2.Code, rec2.Body.String()) + } + if e.sched.enqueues.Load() != first+1 { + t.Fatalf("零结果不应缓存: enqueue %d want %d", e.sched.enqueues.Load(), first+1) + } + var env contract.Envelope + if err := json.Unmarshal(rec2.Body.Bytes(), &env); err != nil { + t.Fatal(err) + } + if env.Provenance.Cached { + t.Fatal("零结果二次请求不得标 cached") + } +} + // jobsIntent 在 mock 的 reqLog 里按 request_id 找意图。 func jobsIntent(m *mockScheduler, rid, intent string) (bool, bool) { m.mu.Lock() diff --git a/server/internal/scheduler/scheduler_test.go b/server/internal/scheduler/scheduler_test.go index fc63f6a..0871fad 100644 --- a/server/internal/scheduler/scheduler_test.go +++ b/server/internal/scheduler/scheduler_test.go @@ -55,7 +55,7 @@ func stubSearchAdapter(t *testing.T) *httptest.Server { return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte(`{"query":"q","unresponsive_engines":[],"results":[ {"title":"T1","url":"https://a.test/1","content":"内容一","engine":"baidu"}, - {"title":"T2","url":"https://a.test/2","content":"内容二","engine":"bing"}]}`)) + {"title":"T2","url":"https://b.test/2","content":"内容二","engine":"bing"}]}`)) })) } diff --git a/server/internal/scheduler/search_filter.go b/server/internal/scheduler/search_filter.go new file mode 100644 index 0000000..37094ca --- /dev/null +++ b/server/internal/scheduler/search_filter.go @@ -0,0 +1,104 @@ +// search_filter.go:search 发现层 Layer 1 规则过滤(无模型、微秒级)。 +// +// 闸门顺序:广告/反爬占位 → 站内搜索死循环(窄集)→ 既有壳页 isJunkSearchHit +// → 同 eTLD+1 硬限额 1 条。全部 hits 过滤完后再由调用方裁 max_results。 +package scheduler + +import ( + "net/url" + "regexp" + "strings" + + "onesvm.com/onesvm/browser-server/internal/session" +) + +var ( + adTitleRegex = regexp.MustCompile(`(?i)(【推广】|【广告】|广告[::]|赞助商链接|商业推广|加V看全文|进群领取|(?m)^sponsored\b|(?m)^ad\b|sponsored links|advertisement)`) + spamSnippetRegex = regexp.MustCompile(`(?i)(please enable javascript|access denied|403 forbidden|安全验证|验证码|checking your browser before accessing)`) + searchLoopTitleRegex = regexp.MustCompile(`的搜索结果|共找到`) +) + +// filterSearchHits 对 Raw hits 跑完闸门 1–3(含既有壳页),保留 SearXNG 原序。 +func filterSearchHits(items []searxItem) []searxItem { + seen := make(map[string]struct{}, len(items)) + out := make([]searxItem, 0, len(items)) + for _, it := range items { + if dropSearchHit(it) { + continue + } + etld := searchCrowdKey(it.URL) + if etld == "" { + continue + } + if _, ok := seen[etld]; ok { + continue + } + seen[etld] = struct{}{} + out = append(out, it) + } + return out +} + +func dropSearchHit(it searxItem) bool { + if adTitleRegex.MatchString(it.Title) || adTitleRegex.MatchString(it.Content) { + return true + } + if spamSnippetRegex.MatchString(it.Title) || spamSnippetRegex.MatchString(it.Content) { + return true + } + if searchLoopTitleRegex.MatchString(it.Title) { + return true + } + u, err := url.Parse(it.URL) + if err != nil || u.Hostname() == "" { + return true + } + path := u.EscapedPath() + if isSearchLoopPath(path) || isAuthOrCartPath(path) { + return true + } + return isJunkSearchHit(it.URL, it.Title, it.Content) +} + +// isSearchLoopPath 只杀 /search 与 /so/ 死循环,不杀 /tag /topic /research。 +func isSearchLoopPath(path string) bool { + p := strings.ToLower(path) + trimmed := strings.TrimSuffix(p, "/") + if trimmed == "/search" || strings.HasPrefix(p, "/search/") || strings.Contains(p, "/search/") { + return true + } + if strings.HasSuffix(trimmed, "/search") { + return true + } + if strings.HasPrefix(p, "/so/") || strings.Contains(p, "/so/") { + return true + } + return false +} + +func isAuthOrCartPath(path string) bool { + p := strings.ToLower(strings.TrimSuffix(path, "/")) + for _, seg := range []string{"login", "signin", "auth", "cart", "checkout"} { + if p == "/"+seg || strings.HasPrefix(p, "/"+seg+"/") || strings.HasSuffix(p, "/"+seg) || strings.Contains(p, "/"+seg+"/") { + return true + } + } + return false +} + +// searchCrowdKey 拥挤控制用的可注册域。复用 session.ETLD1(含 com.cn), +// 另把 gov.cn / edu.cn 当多段后缀,避免全国政府站挤成 1 条。 +func searchCrowdKey(rawURL string) string { + u, err := url.Parse(rawURL) + if err == nil { + host := strings.ToLower(u.Hostname()) + parts := strings.Split(host, ".") + if len(parts) >= 3 { + last2 := parts[len(parts)-2] + "." + parts[len(parts)-1] + if last2 == "gov.cn" || last2 == "edu.cn" { + return parts[len(parts)-3] + "." + last2 + } + } + } + return session.ETLD1(rawURL) +} diff --git a/server/internal/scheduler/search_filter_test.go b/server/internal/scheduler/search_filter_test.go new file mode 100644 index 0000000..cac2bd5 --- /dev/null +++ b/server/internal/scheduler/search_filter_test.go @@ -0,0 +1,107 @@ +package scheduler + +import ( + "testing" +) + +func TestFilterSearchHitsAdAndLoop(t *testing.T) { + items := []searxItem{ + {Title: "【推广】限时批发", URL: "https://spam.example.com/p/1", Content: "厂家直销", Engine: "baidu"}, + {Title: "Sponsored earbuds deal", URL: "https://ads.example.com/p/2", Content: "buy now", Engine: "bing"}, + {Title: "正常政策稿", URL: "https://example.com/search?q=退税", Content: "站内搜索页", Engine: "bing"}, + {Title: "专题标签页", URL: "https://blog.example.com/tag/export-tax", Content: "这是一篇关于出口退税的专题文章摘要超过三十个字。", Engine: "bing"}, + {Title: "国家税务总局政策", URL: "https://www.gov.cn/", Content: "跨境电商出口退税政策解读,2026年起实施无纸化申报。", Engine: "bing"}, + {Title: "", URL: "https://empty-title.example.com/a", Content: "有摘要", Engine: "bing"}, + {Title: "坏链", URL: "not-a-url", Content: "摘要", Engine: "bing"}, + } + got := filterSearchHits(items) + urls := urlsOf(got) + mustHave := []string{"https://blog.example.com/tag/export-tax", "https://www.gov.cn/"} + mustDrop := []string{ + "https://spam.example.com/p/1", + "https://ads.example.com/p/2", + "https://example.com/search?q=退税", + "https://empty-title.example.com/a", + "not-a-url", + } + for _, u := range mustHave { + if !containsURL(urls, u) { + t.Errorf("应保留 %s,得 %v", u, urls) + } + } + for _, u := range mustDrop { + if containsURL(urls, u) { + t.Errorf("应丢弃 %s,得 %v", u, urls) + } + } +} + +func TestFilterSearchHitsHostCrowding(t *testing.T) { + items := make([]searxItem, 0, 9) + for i := 0; i < 8; i++ { + items = append(items, searxItem{ + Title: "1688 批发条", + URL: "https://www.1688.com/offer/" + string(rune('a'+i)), + Content: "工厂货源现货批发一件代发摘要足够长。", + Engine: "360search", + }) + } + items = append(items, searxItem{ + Title: "税务总局退税问答", + URL: "https://www.chinatax.gov.cn/chinatax/n810341/n810765/c5191234/content.html", + Content: "跨境电商出口企业享受退税政策的条件与申报材料。", + Engine: "bing", + }) + got := filterSearchHits(items) + if len(got) != 2 { + t.Fatalf("8 条同域 + 1 条政策站应留 2 条,得 %d: %+v", len(got), urlsOf(got)) + } + if got[0].URL != items[0].URL { + t.Errorf("同域应留原序第一条: %s", got[0].URL) + } + if got[1].URL != items[8].URL { + t.Errorf("政策站应递补: %s", got[1].URL) + } +} + +func TestSearchCrowdKeyGovCN(t *testing.T) { + a := searchCrowdKey("https://www.gov.cn/zhengce/x.htm") + b := searchCrowdKey("https://www.chinatax.gov.cn/n810341/content.html") + c := searchCrowdKey("https://12366.chinatax.gov.cn/sscx/detail") + if a == b { + t.Fatalf("不同部委 gov.cn 不应挤成同一 key: %s", a) + } + if b != c { + t.Fatalf("同一税务机关子域应同一 key: %s vs %s", b, c) + } +} + +func TestFilterSearchHitsKeepsResearchPath(t *testing.T) { + items := []searxItem{{ + Title: "市场研究", + URL: "https://www.example.com/research/earbuds-2026", + Content: "蓝牙耳机市场份额预测与渠道结构分析正文摘要。", + Engine: "bing", + }} + got := filterSearchHits(items) + if len(got) != 1 { + t.Fatalf("/research 不应当 /search 杀掉: %+v", urlsOf(got)) + } +} + +func urlsOf(items []searxItem) []string { + out := make([]string, 0, len(items)) + for _, it := range items { + out = append(out, it.URL) + } + return out +} + +func containsURL(urls []string, want string) bool { + for _, u := range urls { + if u == want { + return true + } + } + return false +} diff --git a/server/internal/scheduler/template.go b/server/internal/scheduler/template.go index 6b62761..b8c438b 100644 --- a/server/internal/scheduler/template.go +++ b/server/internal/scheduler/template.go @@ -112,8 +112,8 @@ func (t *Template) buildDenied(in TemplateInput, ruleID, reason string, warnings return env } -// buildSearch kind=search 数据块:searxng 位次 → SearchResult(content ≤800 截断、 -// score 归一、max_results 裁剪、published_at 东八区)。 +// buildSearch kind=search 数据块:规则过滤全量 hits → 裁 max_results → SearchResult +// (content ≤800、位次分 1-0.05*i、published_at 东八区)。 func (t *Template) buildSearch(in TemplateInput, warnings []string) (*contract.SearchPayload, *contract.Envelope) { items, unres := parseSearxResults(in.Raw) if len(unres) > 0 { @@ -127,17 +127,12 @@ func (t *Template) buildSearch(in TemplateInput, warnings []string) (*contract.S if max > 20 { max = 20 // mcp-usage §2.1:≤20 } - kept := 0 - dropped := 0 - for _, it := range items { - if isJunkSearchHit(it.URL, it.Title, it.Content) { - dropped++ - continue - } - if kept >= max { - break - } - i := kept + filtered := filterSearchHits(items) + dropped := len(items) - len(filtered) + if len(filtered) > max { + filtered = filtered[:max] + } + for i, it := range filtered { res := contract.SearchResult{ ID: fmt.Sprintf("r%d", i+1), Title: it.Title, @@ -152,7 +147,6 @@ func (t *Template) buildSearch(in TemplateInput, warnings []string) (*contract.S } } payload.Results = append(payload.Results, res) - kept++ } if dropped > 0 { warnings = append(warnings, fmt.Sprintf("filtered_nav_hits:%d", dropped))