公开页抓取改为 Chrome 136 自洽身份 + 每节点 SQLite 养罐,Trafilatura 走 curl_cffi;MCP/README/接手说明与 09-02 现网复测对齐,避免消费方继续抄过期的站点三分表。 Co-authored-by: Cursor <cursoragent@cursor.com>
223 lines
6.3 KiB
Go
223 lines
6.3 KiB
Go
// cdp_client.go:CDP WebSocket 客户端(JSON-RPC id 递增 + pending map)。
|
||
//
|
||
// 复用声明:调用面与超时语义移植自 bench/site-matrix/cdp_fetch.mjs 的 Cdp 类
|
||
// (json/version → webSocketDebuggerUrl → dial → send/recv;方法序列
|
||
// Target.createTarget/attachToTarget(flatten) → Page/Runtime/Network.enable →
|
||
// Network.setUserAgentOverride → Page.navigate → 等 loadEventFired →
|
||
// Runtime.evaluate 提取;detectVendor 正则同源)。
|
||
package dock
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"fmt"
|
||
"net"
|
||
"net/http"
|
||
"net/url"
|
||
"strings"
|
||
"sync"
|
||
"time"
|
||
)
|
||
|
||
// cdpHTTPTimeout json/version 探测超时(cdp_fetch.mjs:AbortSignal 5s)。
|
||
const cdpHTTPTimeout = 5 * time.Second
|
||
|
||
// cdpClient 单连接 CDP 会话(一次任务一连接,与 cdp_fetch.mjs 同模式)。
|
||
type cdpClient struct {
|
||
ws *wsConn
|
||
nextID int64
|
||
idMu sync.Mutex
|
||
pending sync.Map // int64 → chan cdpResp
|
||
once sync.Once
|
||
// eventCh 事件流(Page.loadEventFired / Network.responseReceived 等消费方)。
|
||
eventCh chan cdpEvent
|
||
}
|
||
|
||
// cdpResp JSON-RPC 响应。
|
||
type cdpResp struct {
|
||
ID int64 `json:"id"`
|
||
Result json.RawMessage `json:"result"`
|
||
Error *cdpErrResp `json:"error"`
|
||
}
|
||
|
||
// cdpEvent JSON-RPC 事件。
|
||
type cdpEvent struct {
|
||
Method string `json:"method"`
|
||
Params json.RawMessage `json:"params"`
|
||
}
|
||
|
||
// cdpVersionGet GET {host}/json/version → webSocketDebuggerUrl。
|
||
// host 形如 "127.0.0.1:9222"。
|
||
func cdpVersionGet(host string) (wsURL, debuggerURL string, err error) {
|
||
hc := &http.Client{Timeout: cdpHTTPTimeout}
|
||
resp, err := hc.Get("http://" + host + "/json/version")
|
||
if err != nil {
|
||
return "", "", fmt.Errorf("dock: GET /json/version: %w", err)
|
||
}
|
||
defer resp.Body.Close()
|
||
var v struct {
|
||
WebSocketDebuggerURL string `json:"webSocketDebuggerUrl"`
|
||
}
|
||
if err := json.NewDecoder(resp.Body).Decode(&v); err != nil {
|
||
return "", "", fmt.Errorf("dock: /json/version 解码: %w", err)
|
||
}
|
||
if v.WebSocketDebuggerURL == "" {
|
||
return "", "", fmt.Errorf("dock: /json/version 缺 webSocketDebuggerUrl")
|
||
}
|
||
// rewriteWs(cdp_fetch.mjs 同语义):把 ws host:port 对齐 HTTP 端点
|
||
//(调试端点可能回环地址不同)。Host 头仍用引擎自报的回环地址——
|
||
// Lightpanda 对 Host: lightpanda:9222 回 403 Host not allowed。
|
||
return rewriteWsURL(v.WebSocketDebuggerURL, host), v.WebSocketDebuggerURL, nil
|
||
}
|
||
|
||
// wsHandshakeHost 从引擎自报的 debugger URL 取 Host 头。回环地址保留,
|
||
// 以便 TCP 拨 Docker 服务名时握手仍用 127.0.0.1:port。
|
||
func wsHandshakeHost(debuggerURL string) string {
|
||
u, err := url.Parse(debuggerURL)
|
||
if err != nil || u.Hostname() == "" {
|
||
return ""
|
||
}
|
||
host := u.Hostname()
|
||
if host != "127.0.0.1" && host != "localhost" && host != "::1" {
|
||
return ""
|
||
}
|
||
port := u.Port()
|
||
if port == "" {
|
||
port = "9222"
|
||
}
|
||
return net.JoinHostPort(host, port)
|
||
}
|
||
|
||
// rewriteWsURL 把 ws URL 的 host:port 替换为 HTTP 端点的 host:port。
|
||
func rewriteWsURL(wsURL, httpHost string) string {
|
||
i := strings.Index(wsURL, "://")
|
||
if i < 0 {
|
||
return wsURL
|
||
}
|
||
rest := wsURL[i+3:]
|
||
if j := strings.Index(rest, "/"); j >= 0 {
|
||
return "ws://" + httpHost + rest[j:]
|
||
}
|
||
return "ws://" + httpHost
|
||
}
|
||
|
||
// newCdpClient 拨号并启动读泵。
|
||
func newCdpClient(host string, dialTimeout time.Duration) (*cdpClient, error) {
|
||
wsURL, debuggerURL, err := cdpVersionGet(host)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
ws, err := wsDial(wsURL, dialTimeout, wsHandshakeHost(debuggerURL))
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
c := &cdpClient{ws: ws, eventCh: make(chan cdpEvent, 256)}
|
||
go c.readLoop()
|
||
return c, nil
|
||
}
|
||
|
||
// readLoop 读泵:分发响应到 pending / 事件到 eventCh(cdp_fetch.mjs message 监听同语义)。
|
||
func (c *cdpClient) readLoop() {
|
||
defer c.closeOnce()
|
||
for {
|
||
raw, err := c.ws.readFrame()
|
||
if err != nil {
|
||
return
|
||
}
|
||
var env struct {
|
||
ID *int64 `json:"id"`
|
||
Method string `json:"method"`
|
||
Params json.RawMessage `json:"params"`
|
||
Error *cdpErrResp `json:"error"`
|
||
Result json.RawMessage `json:"result"`
|
||
}
|
||
if json.Unmarshal(raw, &env) != nil {
|
||
continue
|
||
}
|
||
if env.ID != nil {
|
||
if chp, ok := c.pending.Load(*env.ID); ok {
|
||
c.pending.Delete(*env.ID)
|
||
ch := *chp.(*chan cdpResp)
|
||
ch <- cdpResp{ID: *env.ID, Result: env.Result, Error: errOf(env.Error)}
|
||
}
|
||
continue
|
||
}
|
||
if env.Method != "" {
|
||
select {
|
||
case c.eventCh <- cdpEvent{Method: env.Method, Params: env.Params}:
|
||
default: // 事件缓冲满丢弃(读泵不能阻塞)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// Send 发送命令等响应(cdp_fetch.mjs Cdp.send 同语义:id 递增 + 超时)。
|
||
func (c *cdpClient) Send(ctx context.Context, method string, params any, sessionID string, timeout time.Duration) (json.RawMessage, error) {
|
||
c.idMu.Lock()
|
||
id := c.nextID
|
||
c.nextID++
|
||
c.idMu.Unlock()
|
||
req := struct {
|
||
ID int64 `json:"id"`
|
||
Method string `json:"method"`
|
||
Params any `json:"params,omitempty"`
|
||
SessionID string `json:"sessionId,omitempty"`
|
||
}{ID: id, Method: method, Params: params, SessionID: sessionID}
|
||
raw, err := json.Marshal(req)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("dock: cdp 请求序列化: %w", err)
|
||
}
|
||
ch := make(chan cdpResp, 2)
|
||
c.pending.Store(id, &ch)
|
||
defer c.pending.Delete(id)
|
||
if err := c.ws.writeFrame(0x1, raw); err != nil {
|
||
return nil, fmt.Errorf("dock: cdp 发送 %s: %w", method, err)
|
||
}
|
||
timer := time.NewTimer(timeout)
|
||
defer timer.Stop()
|
||
select {
|
||
case resp := <-ch:
|
||
if resp.Error != nil {
|
||
return nil, fmt.Errorf("dock: cdp %s: %s", method, resp.errMsg())
|
||
}
|
||
return resp.Result, nil
|
||
case <-timer.C:
|
||
return nil, fmt.Errorf("dock: cdp timeout %s", method)
|
||
case <-ctx.Done():
|
||
return nil, fmt.Errorf("dock: cdp %s: %w", method, ctx.Err())
|
||
}
|
||
}
|
||
|
||
// cdpErrResp JSON-RPC error 对象形状。
|
||
type cdpErrResp struct {
|
||
Code int `json:"code"`
|
||
Message string `json:"message"`
|
||
}
|
||
|
||
// errOf 把 wire 错误转为类型化指针。
|
||
func errOf(e *cdpErrResp) *cdpErrResp {
|
||
if e == nil {
|
||
return nil
|
||
}
|
||
cp := *e
|
||
return &cp
|
||
}
|
||
|
||
// errMsg 错误消息。
|
||
func (r cdpResp) errMsg() string {
|
||
if r.Error == nil {
|
||
return ""
|
||
}
|
||
return r.Error.Message
|
||
}
|
||
|
||
// Events 事件通道。
|
||
func (c *cdpClient) Events() <-chan cdpEvent { return c.eventCh }
|
||
|
||
// closeOnce 幂等关连接。
|
||
func (c *cdpClient) closeOnce() {
|
||
c.once.Do(func() { c.ws.Close() })
|
||
}
|
||
|
||
// Close 关闭连接。
|
||
func (c *cdpClient) Close() { c.closeOnce() }
|