// dock_test.go:适配器单测(无外网,httptest 环回 + 样本形状对齐 bench)。 // // T1 契约:mock 响应形状 = bench 实测样本形状 // (searxng-cn/samples/t1-1.excerpt.json、trafilatura-http/samples/t2-1 与 // t2-cross-govcn.excerpt.json)。 package dock import ( "context" "encoding/json" "net/http" "net/http/httptest" "strings" "sync" "testing" "time" "onesvm.com/onesvm/browser-server/internal/contract" ) // sampleSearxJSON 构造与 bench t1-1.excerpt.json 同形状的响应(字段名一致)。 func sampleSearxJSON() string { return `{ "query": "跨境电商 出口退税 政策 2026", "number_of_results": null, "unresponsive_engines": [["sogou", "Suspended: CAPTCHA"]], "results": [ {"title": "海关总署公告", "url": "https://hainan.chinatax.gov.cn/xxgk_6_1/06163393.html", "content": "对自2026年1月1日至2027年12月31日期间…免征进口关税", "engine": "baidu"}, {"title": "雨果跨境", "url": "https://m.cifnews.com/", "content": "雨果跨境以雨果网作为流量依托", "engine": "bing", "publishedDate": "2026-08-23T00:00:00+08:00"} ] }` } // TestSearxExecute 样本形状解析 + 查询参数对齐(format=json/safesearch=0)。 func TestSearxExecute(t *testing.T) { var gotPath string var gotQuery string srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { gotPath = r.URL.Path gotQuery = r.URL.RawQuery _, _ = w.Write([]byte(sampleSearxJSON())) })) defer srv.Close() a := NewSearx("searxng-cn", srv.URL, "zh-CN") if err := a.Init(context.Background()); err != nil { t.Fatalf("Init: %v", err) } job := contract.JobEnvelope{Intent: "search", Search: &contract.SearchInput{Query: "跨境电商 出口退税"}} raw, eb := a.Execute(context.Background(), job) if eb != nil { t.Fatalf("Execute 不应失败: %+v", eb) } if gotPath != "/search" { t.Errorf("路径应 /search,得 %s", gotPath) } for _, want := range []string{"format=json", "safesearch=0", "language=zh-CN"} { if !strings.Contains(gotQuery, want) { t.Errorf("query 缺 %s: %s", want, gotQuery) } } items, unres := mapSearxItems(raw) if len(items) != 2 { t.Fatalf("应解析 2 条,得 %d", len(items)) } if items[0].Engine != "baidu" || items[0].URL == "" { t.Errorf("首条形状不符: %+v", items[0]) } if len(unres) != 1 || !strings.Contains(unres[0], "sogou") { t.Errorf("unresponsive 应含 sogou: %v", unres) } // 健康位。 if h := a.Health(); !h.OK { t.Errorf("健康位应为 true: %+v", h) } } // mapSearxItems 测试辅助:从 Extra 取回条目(与 scheduler 侧解析同形状)。 type mapSearxItem struct { Title string `json:"title"` URL string `json:"url"` Content string `json:"content"` Engine string `json:"engine"` PublishedDate *string `json:"publishedDate"` } func mapSearxItems(raw *contract.RawResult) ([]mapSearxItem, []string) { var items []mapSearxItem if v, ok := raw.Extra["searx_results"]; ok { if b, err := json.Marshal(v); err == nil { _ = json.Unmarshal(b, &items) } } var unres []string if v, ok := raw.Extra["unresponsive_engines"]; ok { if b, err := json.Marshal(v); err == nil { var pairs [][]string _ = json.Unmarshal(b, &pairs) for _, p := range pairs { if len(p) == 2 { unres = append(unres, p[0]+"("+p[1]+")") } } } } return items, unres } // TestSearxConcurrencyClamp 并发钳 6 + host 最小间隔:20 个并发请求串行化 // (150ms 间隔 → 总耗时 ≥150ms×(需等待次数),至少验证无并发穿透)。 func TestSearxConcurrencyClamp(t *testing.T) { var mu sync.Mutex concurrent := 0 maxConcurrent := 0 srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { mu.Lock() concurrent++ if concurrent > maxConcurrent { maxConcurrent = concurrent } mu.Unlock() time.Sleep(30 * time.Millisecond) mu.Lock() concurrent-- mu.Unlock() _, _ = w.Write([]byte(sampleSearxJSON())) })) defer srv.Close() a := NewSearx("searxng-cn", srv.URL, "zh-CN") // 信号量钳 6 是硬上限:并发 20 打入,maxConcurrent ≤ 6。 var wg sync.WaitGroup for i := 0; i < 20; i++ { wg.Add(1) go func() { defer wg.Done() _, _ = a.Execute(context.Background(), contract.JobEnvelope{ Intent: "search", Search: &contract.SearchInput{Query: "q"}}) }() } wg.Wait() if maxConcurrent > searxMaxConcurrent { t.Errorf("并发穿透:max=%d > %d", maxConcurrent, searxMaxConcurrent) } } // TestTrafilaturaContract 契约:请求体 {url,max_chars};响应样本形状(成功 + empty_extract)。 func TestTrafilaturaContract(t *testing.T) { var gotBody map[string]any srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/health": _, _ = w.Write([]byte(`{"ok":true}`)) case "/v1/read": _ = json.NewDecoder(r.Body).Decode(&gotBody) // t2-1.excerpt.json 同形状(成功)。 _, _ = w.Write([]byte(`{"ok":true,"title":"跨境电子商务出口退运商品税收优惠政策公告", "markdown":"| 索引号 | 11460000008174507Q/2026-14228 |\n\n为支持跨境电子商务新业态发展,现将…公告如下:", "char_count":1122,"truncated":false,"url":"https://hainan.chinatax.gov.cn/xxgk_6_1/06163393.html"}`)) } })) defer srv.Close() a := NewTrafilatura(srv.URL) _ = a.Init(context.Background()) job := contract.JobEnvelope{Intent: "read", Read: &contract.ReadInput{URL: "https://hainan.chinatax.gov.cn/xxgk_6_1/06163393.html", MaxChars: 20000}} raw, eb := a.Execute(context.Background(), job) if eb != nil { t.Fatalf("Execute 不应失败: %+v", eb) } if gotBody["url"] != job.Read.URL || gotBody["max_chars"] != float64(20000) { t.Errorf("请求体契约不符: %v", gotBody) } if raw.Title == "" || raw.Markdown == "" { t.Errorf("RawResult 字段缺失: %+v", raw) } if raw.Extra["char_count"] != 1122 { t.Errorf("char_count 应透传 1122,得 %v", raw.Extra["char_count"]) } } // TestTrafilaturaEmptyExtract t2-cross-govcn 形状:ok=false + fail_class=empty_extract // → 上层触发降级链(此处断言错误为 upstream + 消息含 empty_extract)。 func TestTrafilaturaEmptyExtract(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte(`{"ok":false,"title":"","char_count":null,"truncated":null, "url":"https://www.gov.cn/lianbo/202602/content_7057577.htm", "error":"empty_extract","fail_class":"empty_extract","markdown_head":""}`)) })) defer srv.Close() a := NewTrafilatura(srv.URL) job := contract.JobEnvelope{Intent: "read", Read: &contract.ReadInput{URL: "https://www.gov.cn/lianbo/x.htm"}} raw, eb := a.Execute(context.Background(), job) if raw != nil || eb == nil { t.Fatalf("空正文应返回错误: raw=%v eb=%v", raw, eb) } if eb.Code != contract.CodeUpstream { t.Errorf("empty_extract 应为 upstream(触发降级链),得 %s", eb.Code) } } // TestDetectVendor detectVendor 正则族(cdp_fetch.mjs 移植)逐类断言。 func TestDetectVendor(t *testing.T) { cases := []struct { title, text, htmlHead, hdr string looksBlocked bool status, textLen int want string }{ {"Just a moment...", "", "cf-challenge", "", false, 403, 500, "cloudflare"}, {"", "Please verify you are a human (captcha)", "", "", false, 200, 300, "captcha"}, {"Access Denied", "", "", "server:AkamaiGHost", false, 403, 50, "waf"}, {"", "", "", "", false, 200, 30, "empty"}, {"Normal Page", strings.Repeat("正文内容", 100), "", "", false, 200, 400, "none"}, } for i, c := range cases { got := detectVendor(c.title, c.text, c.htmlHead, c.hdr, c.looksBlocked, c.status, c.textLen) if got != c.want { t.Errorf("case %d: detectVendor=%s want=%s", i, got, c.want) } } } // TestExtractResultGolden CDP evaluate 返回形状 golden(字段名锁死)。 func TestExtractResultGolden(t *testing.T) { ex := extractResult{Title: "T", Text: "body", HTMLLen: 100, TextLen: 4, HTMLHead: "", FinalURL: "https://x/", ReadyState: "complete", LooksBlocked: false} b, err := json.Marshal(ex) if err != nil { t.Fatal(err) } var m map[string]any if err := json.Unmarshal(b, &m); err != nil { t.Fatal(err) } for _, k := range []string{"title", "text", "htmlLen", "textLen", "htmlHead", "finalUrl", "readyState", "looksBlocked"} { if _, ok := m[k]; !ok { t.Errorf("extractResult 缺字段 %s(cdp_fetch.mjs EXTRACT_JS 形状漂移)", k) } } } // TestRegistryHealth 摘除语义:探活失败 → Health().ok=false → HealthyAdapters 过滤。 func TestRegistryHealth(t *testing.T) { bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusServiceUnavailable) })) defer bad.Close() a := NewTrafilatura(bad.URL) _ = a.Init(context.Background()) if h := a.Health(); h.OK { t.Fatalf("503 探活后应不健康: %+v", h) } reg := NewRegistry() reg.Register(a) if got := reg.HealthyAdapters("read"); len(got) != 0 { t.Errorf("不健康适配器应被摘除,得 %v", got) } } // TestTextToMarkdown 纯文本段落保持(\n\n 保段)。 func TestTextToMarkdown(t *testing.T) { in := "第一段第一行\n第一段第二行\n\n第二段" got := textToMarkdown(in) if !strings.Contains(got, "第一段第一行\n第一段第二行") { t.Errorf("段内换行应保留: %q", got) } if !strings.Contains(got, "\n\n第二段") { t.Errorf("段间应 \\n\\n: %q", got) } } // TestRewriteWsURL rewriteWs 语义(cdp_fetch.mjs:ws host 对齐 http 端点)。 func TestRewriteWsURL(t *testing.T) { got := rewriteWsURL("ws://127.0.0.1:9222/devtools/browser/abc", "lightpanda:9222") if got != "ws://lightpanda:9222/devtools/browser/abc" { t.Errorf("rewriteWsURL: %s", got) } } func TestWsHandshakeHostLoopback(t *testing.T) { if got := wsHandshakeHost("ws://127.0.0.1:9222/"); got != "127.0.0.1:9222" { t.Fatalf("loopback Host 头: %s", got) } if got := wsHandshakeHost("ws://lightpanda:9222/"); got != "" { t.Fatalf("非回环不应改 Host: %s", got) } } func TestSearxHTTPStatusHealthLatch(t *testing.T) { n := 0 srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/healthz" { w.WriteHeader(http.StatusOK) return } n++ if n == 1 { w.WriteHeader(http.StatusNotFound) return } w.WriteHeader(http.StatusBadGateway) })) defer srv.Close() a := NewSearx("searxng-cn", srv.URL, "zh-CN") _ = a.Init(context.Background()) job := contract.JobEnvelope{Intent: "search", Search: &contract.SearchInput{Query: "q"}} _, _ = a.Execute(context.Background(), job) if h := a.Health(); !h.OK { t.Fatalf("404 不应闩死适配器: %+v", h) } _, _ = a.Execute(context.Background(), job) if h := a.Health(); h.OK { t.Fatalf("502 应将实例标不健康: %+v", h) } } // TestSearxCaps 区域/意图能力标签。 func TestSearxCaps(t *testing.T) { cn := NewSearx("searxng-cn", "http://x", "zh-CN").Capabilities() if err := cn.Match("search", contract.RegionDomestic, contract.RenderNone); err != nil { t.Errorf("cn 应匹配 domestic search: %v", err) } if err := cn.Match("search", contract.RegionOverseas, contract.RenderNone); err == nil { t.Error("cn 不应匹配 overseas") } gl := NewSearx("searxng-global", "http://x", "en").Capabilities() if err := gl.Match("search", contract.RegionOverseas, contract.RenderNone); err != nil { t.Errorf("global 应匹配 overseas: %v", err) } } var _ = time.Second // 保导入