公开页抓取改为 Chrome 136 自洽身份 + 每节点 SQLite 养罐,Trafilatura 走 curl_cffi;MCP/README/接手说明与 09-02 现网复测对齐,避免消费方继续抄过期的站点三分表。 Co-authored-by: Cursor <cursoragent@cursor.com>
204 lines
6.9 KiB
Go
204 lines
6.9 KiB
Go
// browser_adapters.go:CDP 浏览器类适配器(lightpanda / headless-shell 共用)。
|
||
//
|
||
// 槽位纪律(plan-final §2.5 实测锚点):lightpanda 4 槽(2.13 jobs/s)、
|
||
// headless-shell 槽=1(0.81 jobs/s FIFO 零拒绝)。shell 空闲回收计数:
|
||
// 连续空闲满 idleTTL 由 Health/巡检触发「回收标记」(本版不做 Swarm scale——
|
||
// TODO 部署轮做:scale 0→1 拉起 + 10min 空闲回收,design §A3)。
|
||
package dock
|
||
|
||
import (
|
||
"context"
|
||
"sync"
|
||
"sync/atomic"
|
||
"time"
|
||
|
||
"onesvm.com/onesvm/browser-server/internal/contract"
|
||
)
|
||
|
||
// 槽位常量(plan-final §2.5)。
|
||
const (
|
||
pandaSlots = 4
|
||
shellSlots = 1
|
||
shellIdleTTL = 10 * time.Minute // 空闲回收阈值(design §A3;本版仅计数)
|
||
)
|
||
|
||
// cdpBrowserAdapter CDP 浏览器适配器公共骨架(Execute 由具体实例注入)。
|
||
type cdpBrowserAdapter struct {
|
||
name string // lightpanda | headless-shell
|
||
host string // "host:9222"(CDP 调试端点)
|
||
|
||
sem chan struct{}
|
||
execFn func(ctx context.Context, host, url string, sess *contract.SessionAttach) (*contract.RawResult, *contract.ErrBody)
|
||
startupMs int64 // Init 探活耗时
|
||
healthy atomic.Bool
|
||
lastErr string
|
||
errMu sync.Mutex
|
||
recycles atomic.Int64 // 空闲回收计数(本版仅计数,不做 scale)
|
||
lastActiv atomic.Int64 // unix 秒;shell 空闲回收计时
|
||
startedAt time.Time
|
||
}
|
||
|
||
// markHealth 线程安全健康位。
|
||
func (a *cdpBrowserAdapter) markHealth(ok bool, msg string) {
|
||
a.errMu.Lock()
|
||
a.healthy.Store(ok)
|
||
a.lastErr = msg
|
||
a.errMu.Unlock()
|
||
}
|
||
|
||
// snapshotHealth 读健康位与消息。
|
||
func (a *cdpBrowserAdapter) snapshotHealth() (bool, string) {
|
||
a.errMu.Lock()
|
||
defer a.errMu.Unlock()
|
||
return a.healthy.Load(), a.lastErr
|
||
}
|
||
|
||
// Capabilities render 分级:lightpanda=light、headless-shell=full。
|
||
func (a *cdpBrowserAdapter) Capabilities() contract.Caps {
|
||
render := contract.RenderLight
|
||
if a.name == "headless-shell" {
|
||
render = contract.RenderFull
|
||
}
|
||
slots := pandaSlots
|
||
if a.name == "headless-shell" {
|
||
slots = shellSlots
|
||
}
|
||
return contract.Caps{
|
||
Intents: []string{"read"},
|
||
Render: render,
|
||
Regions: []string{contract.RegionDomestic, contract.RegionOverseas},
|
||
MaxConcurrent: slots,
|
||
ProxyRequired: true, // 代理由引擎容器 env/flags 配(bench compose 同构),适配器不设 CDP 代理
|
||
Formats: []string{"markdown", "text"},
|
||
}
|
||
}
|
||
|
||
// Init 探活:GET /json/version 计 startup_ms(cdp_fetch.mjs pickWs 第一步)。
|
||
func (a *cdpBrowserAdapter) Init(ctx context.Context) error {
|
||
t0 := time.Now()
|
||
ok, msg := cdpProbe(ctx, a.host)
|
||
a.startupMs = time.Since(t0).Milliseconds()
|
||
a.startedAt = time.Now()
|
||
a.markHealth(ok, msg)
|
||
return nil // 探活失败不阻塞启动;Health() 摘除走降级
|
||
}
|
||
|
||
// Health 健康上报(RSS 进程内无法读远端,上报 0 并注明;协议字段在位)。
|
||
func (a *cdpBrowserAdapter) Health() contract.Health {
|
||
ok, msg := a.snapshotHealth()
|
||
slots := pandaSlots
|
||
if a.name == "headless-shell" {
|
||
slots = shellSlots
|
||
// shell 空闲回收计数:超 TTL 标记回收(本版仅记录 + TODO Swarm scale)。
|
||
if ok && time.Since(a.lastActive()) > shellIdleTTL {
|
||
msg = msg + "; idle>10min(回收标记,Swarm scale 回收属部署轮 TODO)"
|
||
}
|
||
}
|
||
return contract.Health{
|
||
OK: ok,
|
||
RSSBytes: 0, // 进程内无法读远端引擎 RSS,恒 0(TODO 部署轮接 cgroup 采集)
|
||
StartupMs: a.startupMs,
|
||
SlotsFree: slots - a.inFlight(),
|
||
Details: msg,
|
||
}
|
||
}
|
||
|
||
// inFlight 在途数(信号量占用)。
|
||
func (a *cdpBrowserAdapter) inFlight() int {
|
||
if a.sem == nil {
|
||
return 0
|
||
}
|
||
return len(a.sem)
|
||
}
|
||
|
||
// sem 槽位(由构造器赋值;放结构外字段避免骨架循环依赖)。
|
||
// cdpBrowserAdapter 内嵌槽位字段。
|
||
// (Go 结构体无继承,这里直接加字段。)
|
||
|
||
// lastActive 最近活动时间。
|
||
func (a *cdpBrowserAdapter) lastActive() time.Time {
|
||
return time.Unix(a.lastActiv.Load(), 0)
|
||
}
|
||
|
||
// touchActive 标记活动。
|
||
func (a *cdpBrowserAdapter) touchActive() { a.lastActiv.Store(time.Now().Unix()) }
|
||
|
||
// Teardown 停新活 + 关闭在途(等待在途清零,最多 grace 5s)。
|
||
func (a *cdpBrowserAdapter) Teardown(ctx context.Context) error {
|
||
a.markHealth(false, "teardown")
|
||
deadline := time.Now().Add(5 * time.Second)
|
||
for a.inFlight() > 0 && time.Now().Before(deadline) {
|
||
select {
|
||
case <-ctx.Done():
|
||
return ctx.Err()
|
||
case <-time.After(100 * time.Millisecond):
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// executeCommon 浏览器适配器 Execute 公共路径:槽位 → cdpFetchPage → 活动标记。
|
||
func (a *cdpBrowserAdapter) executeCommon(ctx context.Context, job contract.JobEnvelope) (*contract.RawResult, *contract.ErrBody) {
|
||
if job.Read == nil {
|
||
return nil, &contract.ErrBody{Code: contract.CodeUpstream, Message: "read 任务缺 ReadInput"}
|
||
}
|
||
select {
|
||
case a.sem <- struct{}{}:
|
||
defer func() { <-a.sem }()
|
||
case <-ctx.Done():
|
||
return nil, &contract.ErrBody{Code: contract.CodeTimeout, Message: a.name + " 槽位等待取消"}
|
||
}
|
||
a.touchActive()
|
||
res, eb := a.execFn(ctx, a.host, job.Read.URL, job.Session)
|
||
if eb != nil {
|
||
// 单页失败(含目标站 403/空正文)不当整实例不健康。
|
||
return nil, eb
|
||
}
|
||
a.markHealth(true, "")
|
||
if res.Engine == "cdp" {
|
||
res.Engine = a.name
|
||
}
|
||
res.Extra["recycles_total"] = a.recycles.Load()
|
||
return res, nil
|
||
}
|
||
|
||
// LightpandaAdapter 轻渲染适配器(render=light,4 槽)。
|
||
type LightpandaAdapter struct{ *cdpBrowserAdapter }
|
||
|
||
// NewLightpanda 构造。host 形如 "lightpanda:9222"。
|
||
func NewLightpanda(host string) *LightpandaAdapter {
|
||
core := &cdpBrowserAdapter{name: "lightpanda", host: host, sem: make(chan struct{}, pandaSlots)}
|
||
core.execFn = cdpFetchPage
|
||
core.healthy.Store(true)
|
||
return &LightpandaAdapter{cdpBrowserAdapter: core}
|
||
}
|
||
|
||
// Execute 精读执行。
|
||
func (l *LightpandaAdapter) Execute(ctx context.Context, job contract.JobEnvelope) (*contract.RawResult, *contract.ErrBody) {
|
||
return l.executeCommon(ctx, job)
|
||
}
|
||
|
||
// HeadlessShellAdapter 保真适配器(render=full,槽=1)。
|
||
type HeadlessShellAdapter struct{ *cdpBrowserAdapter }
|
||
|
||
// NewHeadlessShell 构造。host 形如 "headless-shell:9222"。
|
||
func NewHeadlessShell(host string) *HeadlessShellAdapter {
|
||
core := &cdpBrowserAdapter{name: "headless-shell", host: host, sem: make(chan struct{}, shellSlots)}
|
||
core.execFn = cdpFetchPage
|
||
core.healthy.Store(true)
|
||
return &HeadlessShellAdapter{cdpBrowserAdapter: core}
|
||
}
|
||
|
||
// Execute 精读执行。
|
||
func (h *HeadlessShellAdapter) Execute(ctx context.Context, job contract.JobEnvelope) (*contract.RawResult, *contract.ErrBody) {
|
||
return h.executeCommon(ctx, job)
|
||
}
|
||
|
||
// MarkRecycle 空闲回收计数 +1(调度器巡检触发时调用;Swarm scale 属部署轮)。
|
||
func (h *HeadlessShellAdapter) MarkRecycle() { h.recycles.Add(1) }
|
||
|
||
// Name 适配器展示名(registry 键;内嵌骨架透出)。
|
||
func (l *LightpandaAdapter) Name() string { return l.name }
|
||
|
||
// Name 适配器展示名(registry 键;内嵌骨架透出)。
|
||
func (h *HeadlessShellAdapter) Name() string { return h.name }
|