chii
9b689b2476
公开页抓取改为 Chrome 136 自洽身份 + 每节点 SQLite 养罐，Trafilatura 走 curl_cffi；MCP/README/接手说明与 09-02 现网复测对齐，避免消费方继续抄过期的站点三分表。 Co-authored-by: Cursor <cursoragent@cursor.com>
230 lines
8.6 KiB
Go
230 lines
8.6 KiB
Go
// routing.go：能力路由（标签制，design §4.4）+ read 降级链（plan-final §3）。
|
||
//
|
||
// 路由表（intent + region + render → adapter 名）：
|
||
//
|
||
// search + domestic → searxng-cn
|
||
// search + overseas → searxng-global
|
||
// read + render=none → trafilatura
|
||
// read + render=light → lightpanda
|
||
// read + render=full → headless-shell
|
||
//
|
||
// 降级链（read）：trafilatura 空正文（empty_extract 类）→ lightpanda（render=light）
|
||
// → headless-shell（仅特权/保真需求）→ 全败 blocked/upstream + warnings，不静默降质。
|
||
// 适配器 Health().ok=false → 从路由摘除走降级（design §4.5 条款 2）。
|
||
// overseas 任务：经 proxymanager 选出口；proxymanager 不可达 → fail-closed 排队
|
||
// 等待（O3：回队+告警，不降级直连）。
|
||
package scheduler
|
||
|
||
import (
|
||
"context"
|
||
"strings"
|
||
"time"
|
||
|
||
"onesvm.com/onesvm/browser-server/internal/contract"
|
||
"onesvm.com/onesvm/browser-server/internal/session"
|
||
)
|
||
|
||
// 路由目标常量。
|
||
const (
|
||
adapterSearxCN = "searxng-cn"
|
||
adapterSearxGlobal = "searxng-global"
|
||
adapterTraf = "trafilatura"
|
||
adapterPanda = "lightpanda"
|
||
adapterShell = "headless-shell"
|
||
)
|
||
|
||
// routeTarget 单步路由决策。
|
||
type routeTarget struct {
|
||
adapter string
|
||
minRender string
|
||
}
|
||
|
||
// routeSearch 搜索路由（region 决定实例；无降级——引擎不可用即 upstream 重试）。
|
||
func routeSearch(region string) string {
|
||
if region == contract.RegionOverseas {
|
||
return adapterSearxGlobal
|
||
}
|
||
return adapterSearxCN
|
||
}
|
||
|
||
// routeRead 首选 read 适配器：render=none → trafilatura；light → panda；full → shell。
|
||
// 意图判定依据 ReadInput.Formats / Extract（首版 read 默认 markdown → none）；
|
||
// formats 含 html/screenshot 或显式 render 需求走升级。
|
||
func routeRead(job contract.JobEnvelope) string {
|
||
if job.Read == nil {
|
||
return adapterTraf
|
||
}
|
||
for _, f := range job.Read.Formats {
|
||
switch f {
|
||
case "html", "screenshot":
|
||
return adapterShell // 特权 formats 需要完整 DOM
|
||
}
|
||
}
|
||
return adapterTraf
|
||
}
|
||
|
||
// routeResult 单次执行结论。
|
||
type routeResult struct {
|
||
raw *contract.RawResult
|
||
errBody *contract.ErrBody
|
||
adapter string // 最终成功的适配器名
|
||
proxy string // provenance.proxy_exit
|
||
warnings []string
|
||
tookMs int64
|
||
}
|
||
|
||
// executeWithRouting 能力路由 + 降级链执行（runJob 内核）。
|
||
func (c *Core) executeWithRouting(ctx context.Context, job contract.JobEnvelope) (*contract.RawResult, *contract.ErrBody, routeResult) {
|
||
t0 := time.Now()
|
||
switch job.Intent {
|
||
case "search":
|
||
return c.execSearch(ctx, job, t0)
|
||
case "read":
|
||
return c.execReadChain(ctx, job, t0)
|
||
default:
|
||
return nil, &contract.ErrBody{Code: contract.CodeUpstream, Message: "未知意图 " + job.Intent}, routeResult{tookMs: ms(t0)}
|
||
}
|
||
}
|
||
|
||
// execSearch 搜索执行（无降级链；overseas 需 proxymanager 可达）。
|
||
func (c *Core) execSearch(ctx context.Context, job contract.JobEnvelope, t0 time.Time) (*contract.RawResult, *contract.ErrBody, routeResult) {
|
||
region := contract.RegionDomestic
|
||
if job.Search != nil {
|
||
region = job.Search.Region
|
||
}
|
||
name := routeSearch(region)
|
||
// overseas：proxymanager 不可达 → fail-closed 排队等待（O3 回队+告警）。
|
||
if region == contract.RegionOverseas {
|
||
if !c.proxyOK.Load() {
|
||
return nil, &contract.ErrBody{Code: contract.CodeUnavailable, RetryAfterS: retryPtr(10),
|
||
Message: "proxymanager 不可达，overseas 任务排队等待（fail-closed，不降级直连）"},
|
||
routeResult{tookMs: ms(t0)}
|
||
}
|
||
}
|
||
a, ok := c.reg.Get(name)
|
||
if !ok || !a.Health().OK {
|
||
return nil, &contract.ErrBody{Code: contract.CodeUpstream, Message: "搜索适配器不可用: " + name}, routeResult{tookMs: ms(t0)}
|
||
}
|
||
proxy := "direct"
|
||
if region == contract.RegionOverseas {
|
||
pe, err := c.proxy.ExitForDomain(ctx, jobSearchURL(job))
|
||
if err != nil {
|
||
return nil, &contract.ErrBody{Code: contract.CodeUnavailable, RetryAfterS: retryPtr(10),
|
||
Message: "出口查询失败（fail-closed）: " + err.Error()}, routeResult{tookMs: ms(t0)}
|
||
}
|
||
proxy = pe.Proxy
|
||
}
|
||
raw, eb := a.Execute(ctx, job)
|
||
return raw, eb, routeResult{adapter: name, proxy: proxyExitLabel(region, proxy), tookMs: ms(t0)}
|
||
}
|
||
|
||
// execReadChain read 降级链：trafilatura → lightpanda → headless-shell。
|
||
// 每步失败原因进 warnings；全败 blocked/upstream（不静默降质）。
|
||
func (c *Core) execReadChain(ctx context.Context, job contract.JobEnvelope, t0 time.Time) (*contract.RawResult, *contract.ErrBody, routeResult) {
|
||
primary := routeRead(job)
|
||
// 渲染档需求（formats html/screenshot）直接进 shell 单步（特权路径不走 A/B）。
|
||
if primary == adapterShell {
|
||
return c.execOne(ctx, job, adapterShell, "direct", t0)
|
||
}
|
||
// 链：A(trafilatura) → B(lightpanda) → C(headless-shell)。
|
||
chain := []string{adapterTraf, adapterPanda, adapterShell}
|
||
// 从需求档位起走（render=light 意图直接从 B 起）。
|
||
if primary == adapterPanda {
|
||
chain = []string{adapterPanda, adapterShell}
|
||
} else if primary == adapterShell {
|
||
chain = []string{adapterShell}
|
||
}
|
||
proxy := "direct"
|
||
egress := "direct"
|
||
if job.Read != nil && job.Read.Region == contract.RegionOverseas {
|
||
if !c.proxyOK.Load() {
|
||
return nil, &contract.ErrBody{Code: contract.CodeUnavailable, RetryAfterS: retryPtr(10),
|
||
Message: "proxymanager 不可达，overseas read 排队等待（fail-closed）"}, routeResult{tookMs: ms(t0)}
|
||
}
|
||
pe, err := c.proxy.ExitForDomain(ctx, job.Read.URL)
|
||
if err != nil {
|
||
return nil, &contract.ErrBody{Code: contract.CodeUnavailable, RetryAfterS: retryPtr(10),
|
||
Message: "出口查询失败（fail-closed）: " + err.Error()}, routeResult{tookMs: ms(t0)}
|
||
}
|
||
proxy = pe.Proxy
|
||
egress = session.EgressKey(job.Read.Region, pe.Node, pe.Proxy)
|
||
}
|
||
var warnings []string
|
||
var lastErr *contract.ErrBody
|
||
for _, name := range chain {
|
||
a, ok := c.reg.Get(name)
|
||
if !ok || !a.Health().OK {
|
||
warnings = append(warnings, "adapter_unhealthy:"+name)
|
||
continue
|
||
}
|
||
c.attachSession(ctx, &job, name, egress)
|
||
raw, eb := a.Execute(ctx, job)
|
||
c.commitSession(ctx, job, raw, eb)
|
||
if eb == nil {
|
||
if job.Session != nil {
|
||
warnings = append(warnings, "fp:"+job.Session.TemplateID)
|
||
}
|
||
return raw, nil, routeResult{adapter: name, proxy: proxyExitLabel(job.Read.Region, proxy),
|
||
warnings: warnings, tookMs: ms(t0)}
|
||
}
|
||
lastErr = eb
|
||
warnings = append(warnings, name+" 失败("+eb.Code+"): "+eb.Message)
|
||
// denied（合规拦截）不降级——换引擎同样被拦，直接终态。
|
||
if eb.Code == contract.CodeDenied {
|
||
return nil, eb, routeResult{warnings: warnings, tookMs: ms(t0)}
|
||
}
|
||
}
|
||
if lastErr == nil {
|
||
lastErr = &contract.ErrBody{Code: contract.CodeUpstream, Message: "read 降级链全部不可用"}
|
||
}
|
||
return nil, lastErr, routeResult{warnings: warnings, tookMs: ms(t0)}
|
||
}
|
||
|
||
// execOne 单适配器执行（shell 直达路径用）。
|
||
func (c *Core) execOne(ctx context.Context, job contract.JobEnvelope, name, proxy string, t0 time.Time) (*contract.RawResult, *contract.ErrBody, routeResult) {
|
||
a, ok := c.reg.Get(name)
|
||
if !ok || !a.Health().OK {
|
||
return nil, &contract.ErrBody{Code: contract.CodeUpstream, Message: "适配器不可用: " + name}, routeResult{tookMs: ms(t0)}
|
||
}
|
||
egress := "direct"
|
||
if job.Read != nil {
|
||
egress = session.EgressKey(job.Read.Region, "", proxy)
|
||
}
|
||
c.attachSession(ctx, &job, name, egress)
|
||
raw, eb := a.Execute(ctx, job)
|
||
c.commitSession(ctx, job, raw, eb)
|
||
return raw, eb, routeResult{adapter: name, proxy: proxy, tookMs: ms(t0)}
|
||
}
|
||
|
||
// proxyExitLabel proxy_exit 字段归一（"direct"|"pool:<name>"|"none"）。
|
||
func proxyExitLabel(region, proxy string) string {
|
||
if region != contract.RegionOverseas {
|
||
return "direct"
|
||
}
|
||
if proxy == "" {
|
||
return "none"
|
||
}
|
||
return proxy
|
||
}
|
||
|
||
// overseasSearchExitURL 海外 search 无目标页：出口按 D2 Bing-only 钉搜索域
|
||
// （空串会让 ProxyClient.hostOf 失败，现网 overseas search 整条 fail-closed）。
|
||
const overseasSearchExitURL = "https://www.bing.com/"
|
||
|
||
// jobSearchURL 海外 search 的出口路由键。search 任务没有 Read.URL，必须给
|
||
// 可解析 host；read 任务仍用目标 URL（本函数只给 search 路径调用）。
|
||
func jobSearchURL(job contract.JobEnvelope) string {
|
||
if job.Read != nil && job.Read.URL != "" {
|
||
return job.Read.URL
|
||
}
|
||
return overseasSearchExitURL
|
||
}
|
||
|
||
// ms 起始到现在的毫秒。
|
||
func ms(t time.Time) int64 { return time.Since(t).Milliseconds() }
|
||
|
||
// isBlockedLikeUpstream 判定 upstream 消息里的 empty_extract 类（trafilatura 空正文）。
|
||
func isEmptyExtract(msg string) bool { return strings.Contains(msg, "empty_extract") }
|
||
|
||
// retryPtr int → *int（RetryAfterS 可选字段辅助）。
|
||
func retryPtr(n int) *int { return &n }
|