onesvm-browser-server/server/internal/dock/trafilatura.go

// trafilatura.go：Trafilatura HTTP 适配器（read / render=none）。
//
// 复用声明：POST /v1/read {url, max_chars} 契约与响应形状
// {ok, title, markdown, char_count, truncated, url, error, fail_class}
// 逐字段对齐 bench/trafilatura-http/app.py（同一代码构建的引擎服务）。
// 响应样本以 bench/trafilatura-http/samples/t2-1.excerpt.json（成功）与
// t2-cross-govcn.excerpt.json（empty_extract）为准（T1 契约：mock=真实形状）。
// 空正文（empty_extract）不在此降级——触发上层（scheduler）降级链升级 lightpanda。
package dock

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"sync"
	"time"

	"onesvm.com/onesvm/browser-server/internal/contract"
)

// trafMaxConcurrent 引擎侧 semaphore=8（app.py SEM=8，复用声明）。
const trafMaxConcurrent = 8

// TrafilaturaAdapter trafilatura 精读适配器。
type TrafilaturaAdapter struct {
	baseURL string
	sem     chan struct{}
	hc      *http.Client
	mu      sync.Mutex
	healthy bool
	lastErr string
}

// NewTrafilatura 构造。baseURL 默认 http://trafilatura:8080。
func NewTrafilatura(baseURL string) *TrafilaturaAdapter {
	return &TrafilaturaAdapter{
		baseURL: baseURL,
		sem:     make(chan struct{}, trafMaxConcurrent),
		hc:      &http.Client{Timeout: 30 * time.Second},
		healthy: true,
	}
}

// Capabilities read / none / 双区域（代理由引擎侧 httpx proxy env 承担）。
func (t *TrafilaturaAdapter) Capabilities() contract.Caps {
	return contract.Caps{
		Intents:       []string{"read"},
		Render:        contract.RenderNone,
		Regions:       []string{contract.RegionDomestic, contract.RegionOverseas},
		MaxConcurrent: trafMaxConcurrent,
		ProxyRequired: false, // 引擎侧环境变量配代理，适配器无需经 ProxyManager
		Formats:       []string{"markdown"},
	}
}

// Init 探活 GET /health（app.py：/health 返回 {ok:true,service:...}）。
func (t *TrafilaturaAdapter) Init(ctx context.Context) error {
	hc := &http.Client{Timeout: 5 * time.Second}
	resp, err := hc.Get(t.baseURL + "/health")
	if err != nil {
		t.markHealth(false, err.Error())
		return nil
	}
	defer resp.Body.Close()
	t.markHealth(resp.StatusCode == http.StatusOK, fmt.Sprintf("health=%d", resp.StatusCode))
	return nil
}

func (t *TrafilaturaAdapter) markHealth(ok bool, msg string) {
	t.mu.Lock()
	t.healthy = ok
	t.lastErr = msg
	t.mu.Unlock()
}

// Health 健康上报（RSS 0：远端引擎进程不可读，同 searxng 注明）。
func (t *TrafilaturaAdapter) Health() contract.Health {
	t.mu.Lock()
	ok, msg := t.healthy, t.lastErr
	t.mu.Unlock()
	return contract.Health{OK: ok, RSSBytes: 0, StartupMs: 0,
		SlotsFree: trafMaxConcurrent - len(t.sem), Details: msg}
}

// Teardown 无本地进程。
func (t *TrafilaturaAdapter) Teardown(_ context.Context) error { return nil }

// trafRequest /v1/read 请求体（app.py do_POST 契约）。
type trafRequest struct {
	URL         string            `json:"url"`
	MaxChars    int               `json:"max_chars"`
	Impersonate string            `json:"impersonate,omitempty"`
	Headers     map[string]string `json:"headers,omitempty"`
	Cookies     []contract.Cookie `json:"cookies,omitempty"`
}

// trafResponse /v1/read 响应体（app.py _extract 返回形状）。
type trafResponse struct {
	OK         bool              `json:"ok"`
	Title      string            `json:"title"`
	Markdown   string            `json:"markdown"`
	CharCount  *int              `json:"char_count"`
	Truncated  *bool             `json:"truncated"`
	URL        string            `json:"url"`
	Error      *string           `json:"error"`
	FailClass  *string           `json:"fail_class"`
	SetCookies []contract.Cookie `json:"set_cookies"`
	StatusCode int               `json:"status_code"`
	Poisoned   bool              `json:"poisoned"`
}

// Execute 精读执行。
func (t *TrafilaturaAdapter) Execute(ctx context.Context, job contract.JobEnvelope) (*contract.RawResult, *contract.ErrBody) {
	if job.Read == nil {
		return nil, &contract.ErrBody{Code: contract.CodeUpstream, Message: "read 任务缺 ReadInput"}
	}
	in := job.Read
	maxChars := in.MaxChars
	if maxChars <= 0 {
		maxChars = 20000
	}
	trReq := trafRequest{URL: in.URL, MaxChars: maxChars}
	if job.Session != nil {
		trReq.Impersonate = job.Session.Impersonate
		trReq.Headers = job.Session.Headers
		trReq.Cookies = job.Session.Cookies
	}
	body, err := json.Marshal(trReq)
	if err != nil {
		return nil, &contract.ErrBody{Code: contract.CodeUpstream, Message: err.Error()}
	}
	select {
	case t.sem <- struct{}{}:
		defer func() { <-t.sem }()
	case <-ctx.Done():
		return nil, &contract.ErrBody{Code: contract.CodeTimeout, Message: "trafilatura 槽位等待取消"}
	}
	req, err := http.NewRequestWithContext(ctx, http.MethodPost, t.baseURL+"/v1/read", bytes.NewReader(body))
	if err != nil {
		return nil, &contract.ErrBody{Code: contract.CodeUpstream, Message: err.Error()}
	}
	req.Header.Set("Content-Type", "application/json")
	resp, err := t.hc.Do(req)
	if err != nil {
		// 单次超时不当整实例不健康（healthWatch 只认 /health）。
		return nil, &contract.ErrBody{Code: errCodeOf(err), Message: "trafilatura 请求失败: " + err.Error()}
	}
	defer resp.Body.Close()
	raw, err := io.ReadAll(io.LimitReader(resp.Body, 32*1024*1024))
	if err != nil {
		return nil, &contract.ErrBody{Code: errCodeOf(err), Message: "trafilatura 读响应: " + err.Error()}
	}
	var tr trafResponse
	if err := json.Unmarshal(raw, &tr); err != nil {
		return nil, &contract.ErrBody{Code: contract.CodeUpstream,
			Message: fmt.Sprintf("trafilatura 响应解码（HTTP %d）: %v", resp.StatusCode, err)}
	}
	t.markHealth(true, "")
	if !tr.OK {
		failClass := ""
		if tr.FailClass != nil {
			failClass = *tr.FailClass
		}
		errMsg := "empty_extract"
		if tr.Error != nil {
			errMsg = *tr.Error
		}
		code := contract.CodeUpstream
		if failClass == "blocked" || tr.Poisoned {
			code = contract.CodeBlocked
			return &contract.RawResult{SetCookies: tr.SetCookies, Poisoned: true, StatusCode: tr.StatusCode, Engine: "trafilatura"},
				&contract.ErrBody{Code: code,
					Message: fmt.Sprintf("trafilatura 拦截（fail_class=%s）: %s", failClass, errMsg)}
		}
		return nil, &contract.ErrBody{Code: code,
			Message: fmt.Sprintf("trafilatura 空正文（fail_class=%s）: %s", failClass, errMsg)}
	}
	md := tr.Markdown
	truncated := false
	charCount := len([]rune(md))
	if tr.CharCount != nil {
		charCount = *tr.CharCount
	}
	if tr.Truncated != nil {
		truncated = *tr.Truncated
	}
	status := resp.StatusCode
	if tr.StatusCode > 0 {
		status = tr.StatusCode
	}
	return &contract.RawResult{
		Title:      tr.Title,
		Markdown:   md,
		FinalURL:   firstNonEmpty(tr.URL, in.URL),
		StatusCode: status,
		Engine:     "trafilatura",
		Extra: map[string]any{
			"char_count": charCount,
			"truncated":  truncated,
		},
		SetCookies: tr.SetCookies,
		Poisoned:   tr.Poisoned,
	}, nil
}

// Name 适配器展示名（registry 键）。
func (t *TrafilaturaAdapter) Name() string { return "trafilatura" }