// searxng.go:SearXNG 适配器(searxng-cn / searxng-global 共用实现,实例参数化)。 // // 复用声明:查询参数(format=json、language、time_range、safesearch=0)与 // cn 实例引擎集(baidu/sogou/360search/bing cn)对齐 bench/searxng-cn/settings.yml; // global 实例 Bing-only 姿态对齐 bench/searxng-global(settings.yml + 实测结论 // 「仅 Bing 可用」)。响应样本形状以 bench/searxng-cn/samples/t1-1.excerpt.json 为准 // (T1 契约:mock = 真实响应形状)。 // // 限速纪律(plan-final §2.4 实测结论):多副本不能分散 CAPTCHA,只能限速—— // 并发钳 6(semaphore)+ 每 host 最小间隔 150ms。 package dock import ( "context" "encoding/json" "fmt" "net/http" "net/url" "strings" "sync" "time" "onesvm.com/onesvm/browser-server/internal/contract" ) // searxHostMinInterval 每 host 最小间隔(plan-final §2.4:150ms)。 const searxHostMinInterval = 150 * time.Millisecond // searxMaxConcurrent 并发钳(cn 实测钳 6 ≈9.5qps)。 const searxMaxConcurrent = 6 // SearxAdapter SearXNG 搜索适配器。 type SearxAdapter struct { name string // searxng-cn | searxng-global baseURL string lang string // limiter 每 host 最小间隔节流器。 limiter *hostThrottle sem chan struct{} hc *http.Client mu sync.Mutex healthy bool lastErr string } // NewSearx 构造实例。name: searxng-cn | searxng-global。 func NewSearx(name, baseURL, lang string) *SearxAdapter { return &SearxAdapter{ name: name, baseURL: strings.TrimRight(baseURL, "/"), lang: lang, limiter: newHostThrottle(searxHostMinInterval), sem: make(chan struct{}, searxMaxConcurrent), hc: &http.Client{Timeout: 15 * time.Second}, healthy: true, } } // Capabilities 能力标签(design §4.4:search / render=none / 区域按实例)。 func (s *SearxAdapter) Capabilities() contract.Caps { region := contract.RegionDomestic if s.name == "searxng-global" { region = contract.RegionOverseas } return contract.Caps{ Intents: []string{"search"}, Render: contract.RenderNone, Regions: []string{region}, MaxConcurrent: searxMaxConcurrent, ProxyRequired: region == contract.RegionOverseas, Formats: []string{}, } } // Init 预热:GET /healthz 探活(失败不 panic,健康位打 false 由路由摘除)。 func (s *SearxAdapter) Init(ctx context.Context) error { hc := &http.Client{Timeout: 5 * time.Second} req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.baseURL+"/healthz", nil) if err != nil { s.markHealth(false, err.Error()) return nil // Init 不阻塞启动;Health() 上报不健康 } resp, err := hc.Do(req) if err != nil { s.markHealth(false, err.Error()) return nil } defer resp.Body.Close() s.markHealth(resp.StatusCode == http.StatusOK, fmt.Sprintf("healthz=%d", resp.StatusCode)) return nil } func (s *SearxAdapter) markHealth(ok bool, msg string) { s.mu.Lock() s.healthy = ok s.lastErr = msg s.mu.Unlock() } // Health 健康上报(slots_free 按信号量余量;进程内无法读远端 RSS 上报 0 并注明)。 func (s *SearxAdapter) Health() contract.Health { s.mu.Lock() ok, msg := s.healthy, s.lastErr s.mu.Unlock() return contract.Health{ OK: ok, RSSBytes: 0, // 远端引擎进程 RSS 进程内不可读,恒 0(部署轮由引擎容器 self-report) StartupMs: 0, SlotsFree: searxMaxConcurrent - len(s.sem), Details: msg, } } // Teardown 优雅停机(无本地进程,无操作)。 func (s *SearxAdapter) Teardown(_ context.Context) error { return nil } // searxItem SearXNG json 响应单条结果(真实样本形状:title/url/content/engine, // publishedDate 可选)。 type searxItem struct { Title string `json:"title"` URL string `json:"url"` Content string `json:"content"` Engine string `json:"engine"` PublishedDate *string `json:"publishedDate"` } // searxResponse SearXNG json 响应(样本 t1-1.excerpt.json 同形状)。 type searxResponse struct { Query string `json:"query"` NumberOfResults *int `json:"number_of_results"` UnresponsiveEngines json.RawMessage `json:"unresponsive_engines"` Results []searxItem `json:"results"` } // Execute 搜索执行:GET /search?q=…&format=json&language=…&time_range=…&safesearch=0。 func (s *SearxAdapter) Execute(ctx context.Context, job contract.JobEnvelope) (*contract.RawResult, *contract.ErrBody) { if job.Search == nil { return nil, &contract.ErrBody{Code: contract.CodeUpstream, Message: "search 任务缺 SearchInput"} } in := job.Search q := strings.TrimSpace(in.Query) if q == "" { return nil, &contract.ErrBody{Code: contract.CodeUpstream, Message: "query 为空"} } params := url.Values{} params.Set("q", q) params.Set("format", "json") params.Set("safesearch", "0") if in.Lang != nil && *in.Lang != "" { params.Set("language", *in.Lang) } else if s.lang != "" { params.Set("language", s.lang) } if in.TimeRange != nil && *in.TimeRange != "" { params.Set("time_range", *in.TimeRange) } target := s.baseURL + "/search?" + params.Encode() // 限速:并发钳 + host 最小间隔(plan-final §2.4 设计结论)。 s.limiter.wait(ctx) select { case s.sem <- struct{}{}: defer func() { <-s.sem }() case <-ctx.Done(): return nil, &contract.ErrBody{Code: contract.CodeTimeout, Message: "searxng 槽位等待取消"} } req, err := http.NewRequestWithContext(ctx, http.MethodGet, target, nil) if err != nil { return nil, &contract.ErrBody{Code: contract.CodeUpstream, Message: err.Error()} } req.Header.Set("Accept", "application/json") resp, err := s.hc.Do(req) if err != nil { s.markHealth(false, err.Error()) return nil, &contract.ErrBody{Code: errCodeOf(err), Message: "searxng 请求失败: " + err.Error()} } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { msg := fmt.Sprintf("searxng HTTP %d", resp.StatusCode) s.markHealth(resp.StatusCode < 500, msg) return nil, &contract.ErrBody{Code: contract.CodeUpstream, Message: msg} } var sr searxResponse if err := json.NewDecoder(resp.Body).Decode(&sr); err != nil { return nil, &contract.ErrBody{Code: contract.CodeUpstream, Message: "searxng 响应解码: " + err.Error()} } s.markHealth(true, "") // 位次 score 原始值存 Extra(1 - 0.05*rank,模版层截断 [0,1] 归一)。 // RawResult 不做裁剪(模版层按 max_results 裁剪),全量带位次。 var rawJSON json.RawMessage rawJSON, _ = json.Marshal(sr.Results) return &contract.RawResult{ Engine: s.name, Extra: map[string]any{ "searx_results": json.RawMessage(rawJSON), "unresponsive_engines": sr.UnresponsiveEngines, "query": sr.Query, }, }, nil } // errCodeOf 网络错误 → contract 错误码(timeout / upstream)。 func errCodeOf(err error) string { msg := err.Error() if strings.Contains(msg, "context deadline exceeded") || strings.Contains(msg, "Client.Timeout") { return contract.CodeTimeout } return contract.CodeUpstream } // hostThrottle 每 host 最小间隔节流器(单实例仅一个 host,保持接口便于测试)。 type hostThrottle struct { mu sync.Mutex interval time.Duration next map[string]time.Time } func newHostThrottle(interval time.Duration) *hostThrottle { return &hostThrottle{interval: interval, next: map[string]time.Time{}} } // wait 阻塞至该 host 可请求(ctx 取消即返回)。 func (t *hostThrottle) wait(ctx context.Context) { const key = "searxng" // 单实例单 host t.mu.Lock() waitFor := time.Until(t.next[key].Add(t.interval)) t.next[key] = time.Now().Add(maxDuration(waitFor, 0) + t.interval) t.mu.Unlock() if waitFor <= 0 { return } timer := time.NewTimer(waitFor) defer timer.Stop() select { case <-ctx.Done(): case <-timer.C: } } func maxDuration(a, b time.Duration) time.Duration { if a > b { return a } return b } // Name 适配器展示名(registry 键)。 func (s *SearxAdapter) Name() string { return s.name }