// trafilatura.go:Trafilatura HTTP 适配器(read / render=none)。 // // 复用声明:POST /v1/read {url, max_chars} 契约与响应形状 // {ok, title, markdown, char_count, truncated, url, error, fail_class} // 逐字段对齐 bench/trafilatura-http/app.py(同一代码构建的引擎服务)。 // 响应样本以 bench/trafilatura-http/samples/t2-1.excerpt.json(成功)与 // t2-cross-govcn.excerpt.json(empty_extract)为准(T1 契约:mock=真实形状)。 // 空正文(empty_extract)不在此降级——触发上层(scheduler)降级链升级 lightpanda。 package dock import ( "bytes" "context" "encoding/json" "fmt" "io" "net/http" "sync" "time" "onesvm.com/onesvm/browser-server/internal/contract" ) // trafMaxConcurrent 引擎侧 semaphore=8(app.py SEM=8,复用声明)。 const trafMaxConcurrent = 8 // TrafilaturaAdapter trafilatura 精读适配器。 type TrafilaturaAdapter struct { baseURL string sem chan struct{} hc *http.Client mu sync.Mutex healthy bool lastErr string } // NewTrafilatura 构造。baseURL 默认 http://trafilatura:8080。 func NewTrafilatura(baseURL string) *TrafilaturaAdapter { return &TrafilaturaAdapter{ baseURL: baseURL, sem: make(chan struct{}, trafMaxConcurrent), hc: &http.Client{Timeout: 30 * time.Second}, healthy: true, } } // Capabilities read / none / 双区域(代理由引擎侧 httpx proxy env 承担)。 func (t *TrafilaturaAdapter) Capabilities() contract.Caps { return contract.Caps{ Intents: []string{"read"}, Render: contract.RenderNone, Regions: []string{contract.RegionDomestic, contract.RegionOverseas}, MaxConcurrent: trafMaxConcurrent, ProxyRequired: false, // 引擎侧环境变量配代理,适配器无需经 ProxyManager Formats: []string{"markdown"}, } } // Init 探活 GET /health(app.py:/health 返回 {ok:true,service:...})。 func (t *TrafilaturaAdapter) Init(ctx context.Context) error { hc := &http.Client{Timeout: 5 * time.Second} resp, err := hc.Get(t.baseURL + "/health") if err != nil { t.markHealth(false, err.Error()) return nil } defer resp.Body.Close() t.markHealth(resp.StatusCode == http.StatusOK, fmt.Sprintf("health=%d", resp.StatusCode)) return nil } func (t *TrafilaturaAdapter) markHealth(ok bool, msg string) { t.mu.Lock() t.healthy = ok t.lastErr = msg t.mu.Unlock() } // Health 健康上报(RSS 0:远端引擎进程不可读,同 searxng 注明)。 func (t *TrafilaturaAdapter) Health() contract.Health { t.mu.Lock() ok, msg := t.healthy, t.lastErr t.mu.Unlock() return contract.Health{OK: ok, RSSBytes: 0, StartupMs: 0, SlotsFree: trafMaxConcurrent - len(t.sem), Details: msg} } // Teardown 无本地进程。 func (t *TrafilaturaAdapter) Teardown(_ context.Context) error { return nil } // trafRequest /v1/read 请求体(app.py do_POST 契约)。 type trafRequest struct { URL string `json:"url"` MaxChars int `json:"max_chars"` Impersonate string `json:"impersonate,omitempty"` Headers map[string]string `json:"headers,omitempty"` Cookies []contract.Cookie `json:"cookies,omitempty"` } // trafResponse /v1/read 响应体(app.py _extract 返回形状)。 type trafResponse struct { OK bool `json:"ok"` Title string `json:"title"` Markdown string `json:"markdown"` CharCount *int `json:"char_count"` Truncated *bool `json:"truncated"` URL string `json:"url"` Error *string `json:"error"` FailClass *string `json:"fail_class"` SetCookies []contract.Cookie `json:"set_cookies"` StatusCode int `json:"status_code"` Poisoned bool `json:"poisoned"` } // Execute 精读执行。 func (t *TrafilaturaAdapter) Execute(ctx context.Context, job contract.JobEnvelope) (*contract.RawResult, *contract.ErrBody) { if job.Read == nil { return nil, &contract.ErrBody{Code: contract.CodeUpstream, Message: "read 任务缺 ReadInput"} } in := job.Read maxChars := in.MaxChars if maxChars <= 0 { maxChars = 20000 } trReq := trafRequest{URL: in.URL, MaxChars: maxChars} if job.Session != nil { trReq.Impersonate = job.Session.Impersonate trReq.Headers = job.Session.Headers trReq.Cookies = job.Session.Cookies } body, err := json.Marshal(trReq) if err != nil { return nil, &contract.ErrBody{Code: contract.CodeUpstream, Message: err.Error()} } select { case t.sem <- struct{}{}: defer func() { <-t.sem }() case <-ctx.Done(): return nil, &contract.ErrBody{Code: contract.CodeTimeout, Message: "trafilatura 槽位等待取消"} } req, err := http.NewRequestWithContext(ctx, http.MethodPost, t.baseURL+"/v1/read", bytes.NewReader(body)) if err != nil { return nil, &contract.ErrBody{Code: contract.CodeUpstream, Message: err.Error()} } req.Header.Set("Content-Type", "application/json") resp, err := t.hc.Do(req) if err != nil { // 单次超时不当整实例不健康(healthWatch 只认 /health)。 return nil, &contract.ErrBody{Code: errCodeOf(err), Message: "trafilatura 请求失败: " + err.Error()} } defer resp.Body.Close() raw, err := io.ReadAll(io.LimitReader(resp.Body, 32*1024*1024)) if err != nil { return nil, &contract.ErrBody{Code: errCodeOf(err), Message: "trafilatura 读响应: " + err.Error()} } var tr trafResponse if err := json.Unmarshal(raw, &tr); err != nil { return nil, &contract.ErrBody{Code: contract.CodeUpstream, Message: fmt.Sprintf("trafilatura 响应解码(HTTP %d): %v", resp.StatusCode, err)} } t.markHealth(true, "") if !tr.OK { failClass := "" if tr.FailClass != nil { failClass = *tr.FailClass } errMsg := "empty_extract" if tr.Error != nil { errMsg = *tr.Error } code := contract.CodeUpstream if failClass == "blocked" || tr.Poisoned { code = contract.CodeBlocked return &contract.RawResult{SetCookies: tr.SetCookies, Poisoned: true, StatusCode: tr.StatusCode, Engine: "trafilatura"}, &contract.ErrBody{Code: code, Message: fmt.Sprintf("trafilatura 拦截(fail_class=%s): %s", failClass, errMsg)} } return nil, &contract.ErrBody{Code: code, Message: fmt.Sprintf("trafilatura 空正文(fail_class=%s): %s", failClass, errMsg)} } md := tr.Markdown truncated := false charCount := len([]rune(md)) if tr.CharCount != nil { charCount = *tr.CharCount } if tr.Truncated != nil { truncated = *tr.Truncated } status := resp.StatusCode if tr.StatusCode > 0 { status = tr.StatusCode } return &contract.RawResult{ Title: tr.Title, Markdown: md, FinalURL: firstNonEmpty(tr.URL, in.URL), StatusCode: status, Engine: "trafilatura", Extra: map[string]any{ "char_count": charCount, "truncated": truncated, }, SetCookies: tr.SetCookies, Poisoned: tr.Poisoned, }, nil } // Name 适配器展示名(registry 键)。 func (t *TrafilaturaAdapter) Name() string { return "trafilatura" }