chii
9b689b2476
公开页抓取改为 Chrome 136 自洽身份 + 每节点 SQLite 养罐，Trafilatura 走 curl_cffi；MCP/README/接手说明与 09-02 现网复测对齐，避免消费方继续抄过期的站点三分表。 Co-authored-by: Cursor <cursoragent@cursor.com>
270 lines
7.7 KiB
Go
270 lines
7.7 KiB
Go
// cdp_fake_test.go：fake CDP WebSocket 服务（手写服务端握手 + 文本帧），
|
||
// 验证 cdp_client 调用序列（createTarget→attach→enable→UA→navigate→load 事件→evaluate）。
|
||
package dock
|
||
|
||
import (
|
||
"bufio"
|
||
"context"
|
||
"crypto/sha1"
|
||
"encoding/base64"
|
||
"encoding/binary"
|
||
"encoding/json"
|
||
"io"
|
||
"net"
|
||
"net/http"
|
||
"strings"
|
||
"testing"
|
||
"time"
|
||
|
||
"onesvm.com/onesvm/browser-server/internal/contract"
|
||
)
|
||
|
||
// fakeCdpServer 最小 CDP 服务端：/json/version + ws（无掩码服务端帧）。
|
||
type fakeCdpServer struct {
|
||
ln net.Listener
|
||
url string // "127.0.0.1:port"
|
||
requests []string
|
||
loadSent chan struct{}
|
||
evaluateCh chan string // evaluate expression 收集
|
||
}
|
||
|
||
// newFakeCdp 起服务端。
|
||
func newFakeCdp(t *testing.T) *fakeCdpServer {
|
||
t.Helper()
|
||
l, err := net.Listen("tcp", "127.0.0.1:0")
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
f := &fakeCdpServer{ln: l, url: l.Addr().String(), loadSent: make(chan struct{}, 4),
|
||
evaluateCh: make(chan string, 4)}
|
||
go f.serve()
|
||
t.Cleanup(func() { l.Close() })
|
||
return f
|
||
}
|
||
|
||
// serveHTTP/ws 双协议 accept。
|
||
func (f *fakeCdpServer) serve() {
|
||
for {
|
||
conn, err := f.ln.Accept()
|
||
if err != nil {
|
||
return
|
||
}
|
||
go f.handleConn(conn)
|
||
}
|
||
}
|
||
|
||
func (f *fakeCdpServer) handleConn(conn net.Conn) {
|
||
defer conn.Close()
|
||
br := bufio.NewReader(conn)
|
||
req, err := http.ReadRequest(br)
|
||
if err != nil {
|
||
return
|
||
}
|
||
if req.URL.Path == "/json/version" {
|
||
body := `{"webSocketDebuggerUrl":"ws://` + req.Host + `/devtools/browser/fake"}`
|
||
_, _ = conn.Write([]byte("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: " +
|
||
itoaS(len(body)) + "\r\n\r\n" + body))
|
||
return
|
||
}
|
||
if req.Header.Get("Upgrade") != "websocket" {
|
||
_, _ = conn.Write([]byte("HTTP/1.1 400 Bad Request\r\nContent-Length: 0\r\n\r\n"))
|
||
return
|
||
}
|
||
// ws 握手 accept。
|
||
key := req.Header.Get("Sec-WebSocket-Key")
|
||
h := sha1.Sum([]byte(key + wsGUID))
|
||
accept := base64.StdEncoding.EncodeToString(h[:])
|
||
_, _ = conn.Write([]byte("HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\n" +
|
||
"Connection: Upgrade\r\nSec-WebSocket-Accept: " + accept + "\r\n\r\n"))
|
||
f.serveWS(conn, br)
|
||
}
|
||
|
||
// itoaS 测试内 int→string。
|
||
func itoaS(n int) string {
|
||
if n == 0 {
|
||
return "0"
|
||
}
|
||
s := ""
|
||
for n > 0 {
|
||
s = string(rune('0'+n%10)) + s
|
||
n /= 10
|
||
}
|
||
return s
|
||
}
|
||
|
||
// serveWS 逐帧处理 CDP 请求（无掩码服务端帧）。
|
||
func (f *fakeCdpServer) serveWS(conn net.Conn, br *bufio.Reader) {
|
||
for {
|
||
msg, err := readServerFrame(br)
|
||
if err != nil {
|
||
return
|
||
}
|
||
var env struct {
|
||
ID int64 `json:"id"`
|
||
Method string `json:"method"`
|
||
Params map[string]any `json:"params"`
|
||
}
|
||
if jsonUnmarshal(msg, &env) != nil {
|
||
continue
|
||
}
|
||
f.requests = append(f.requests, env.Method)
|
||
var result string
|
||
switch env.Method {
|
||
case "Target.createTarget":
|
||
result = `{"targetId":"t1"}`
|
||
case "Target.attachToTarget":
|
||
result = `{"sessionId":"s1"}`
|
||
case "Page.navigate":
|
||
result = `{"frameId":"f1"}`
|
||
go func() {
|
||
time.Sleep(50 * time.Millisecond)
|
||
// loadEventFired 事件帧（sessionId 透传）。
|
||
event := `{"method":"Page.loadEventFired","params":{},"sessionId":"s1"}`
|
||
_ = writeServerFrame(conn, []byte(event))
|
||
f.loadSent <- struct{}{}
|
||
}()
|
||
case "Runtime.evaluate":
|
||
if p, ok := env.Params["expression"].(string); ok {
|
||
f.evaluateCh <- p
|
||
}
|
||
result = `{"result":{"type":"object","value":{"title":"测试页","text":"这是正文内容，足够长以通过检测。` +
|
||
repeatCn("段落内容", 40) + `","htmlLen":1234,"textLen":800,"htmlHead":"<html>","finalUrl":"https://example.test/final","readyState":"complete","looksBlocked":false}}}`
|
||
default:
|
||
result = `{}`
|
||
}
|
||
resp := `{"id":` + itoa64(env.ID) + `,"result":` + result + `}`
|
||
if err := writeServerFrame(conn, []byte(resp)); err != nil {
|
||
return
|
||
}
|
||
}
|
||
}
|
||
|
||
// itoa64 int64 → string。
|
||
func itoa64(n int64) string { return itoaS(int(n)) }
|
||
|
||
// readServerFrame 读客户端帧（掩码解码）。
|
||
func readServerFrame(br *bufio.Reader) ([]byte, error) {
|
||
var h [2]byte
|
||
if _, err := readFull(br, h[:]); err != nil {
|
||
return nil, err
|
||
}
|
||
length := int(h[1] & 0x7F)
|
||
if length == 126 {
|
||
var l [2]byte
|
||
if _, err := readFull(br, l[:]); err != nil {
|
||
return nil, err
|
||
}
|
||
length = int(binary.BigEndian.Uint16(l[:]))
|
||
} else if length == 127 {
|
||
var l [8]byte
|
||
if _, err := readFull(br, l[:]); err != nil {
|
||
return nil, err
|
||
}
|
||
length = int(binary.BigEndian.Uint64(l[:]))
|
||
}
|
||
var mask [4]byte
|
||
if h[1]&0x80 != 0 {
|
||
if _, err := readFull(br, mask[:]); err != nil {
|
||
return nil, err
|
||
}
|
||
}
|
||
payload := make([]byte, length)
|
||
if _, err := readFull(br, payload); err != nil {
|
||
return nil, err
|
||
}
|
||
for i := range payload {
|
||
payload[i] ^= mask[i%4]
|
||
}
|
||
return payload, nil
|
||
}
|
||
|
||
// writeServerFrame 写无掩码文本帧。
|
||
func writeServerFrame(conn net.Conn, payload []byte) error {
|
||
hdr := []byte{0x81}
|
||
n := len(payload)
|
||
switch {
|
||
case n < 126:
|
||
hdr = append(hdr, byte(n))
|
||
case n <= 0xFFFF:
|
||
hdr = append(hdr, 126)
|
||
var l [2]byte
|
||
binary.BigEndian.PutUint16(l[:], uint16(n))
|
||
hdr = append(hdr, l[:]...)
|
||
default:
|
||
hdr = append(hdr, 127)
|
||
var l [8]byte
|
||
binary.BigEndian.PutUint64(l[:], uint64(n))
|
||
hdr = append(hdr, l[:]...)
|
||
}
|
||
if _, err := conn.Write(hdr); err != nil {
|
||
return err
|
||
}
|
||
_, err := conn.Write(payload)
|
||
return err
|
||
}
|
||
|
||
func readFull(br *bufio.Reader, buf []byte) (int, error) {
|
||
return io.ReadFull(br, buf)
|
||
}
|
||
|
||
// jsonUnmarshal 包装（测试内避免多余 import 分支）。
|
||
func jsonUnmarshal(b []byte, v any) error { return json.Unmarshal(b, v) }
|
||
|
||
// repeatCn 重复中文串（evaluate 正文构造）。
|
||
func repeatCn(s string, n int) string { return strings.Repeat(s, n) }
|
||
|
||
// TestCdpFetchPageFullChain fake CDP 全链路：createTarget→attach→navigate→
|
||
// load 事件→evaluate→RawResult；blocked 判定路径。
|
||
func TestCdpFetchPageFullChain(t *testing.T) {
|
||
f := newFakeCdp(t)
|
||
raw, eb := cdpFetchPage(context.Background(), f.url, "https://example.test/page", nil)
|
||
if eb != nil {
|
||
t.Fatalf("cdpFetchPage 不应失败: %+v", eb)
|
||
}
|
||
if raw == nil {
|
||
t.Fatal("RawResult 不应为 nil")
|
||
}
|
||
if raw.Title != "测试页" {
|
||
t.Errorf("title 不符: %q", raw.Title)
|
||
}
|
||
if raw.FinalURL != "https://example.test/final" {
|
||
t.Errorf("finalUrl 不符: %s", raw.FinalURL)
|
||
}
|
||
if raw.StatusCode != 0 || raw.Engine != "cdp" {
|
||
t.Errorf("引擎标记不符: %+v", raw)
|
||
}
|
||
// evaluate 收到 EXTRACT_JS 表达式。
|
||
select {
|
||
case expr := <-f.evaluateCh:
|
||
if !strings.Contains(expr, "document.body.innerText") || !strings.Contains(expr, "looksBlocked") {
|
||
t.Errorf("EXTRACT_JS 形状漂移: %s", expr[:60])
|
||
}
|
||
case <-time.After(3 * time.Second):
|
||
t.Fatal("未收到 Runtime.evaluate")
|
||
}
|
||
// 方法序列断言（cdp_fetch.mjs 同序）。
|
||
joined := strings.Join(f.requests, ",")
|
||
for _, want := range []string{"Target.createTarget", "Target.attachToTarget", "Page.navigate", "Runtime.evaluate"} {
|
||
if !strings.Contains(joined, want) {
|
||
t.Errorf("方法序列缺 %s: %s", want, joined)
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestCdpFetchPageBlocked fake 服务返回 blocked 页（looksBlocked=true）→
|
||
// detectVendor=waf → ErrBody.Code=blocked。
|
||
func TestCdpFetchPageBlocked(t *testing.T) {
|
||
// 用受控 fake：evaluate 返回拦截页特征。
|
||
f := newFakeCdp(t)
|
||
// 直接构造：blocked 特征经 htmlHead 判定（fake evaluate 返回正常值，
|
||
// 此处单测 detectVendor 已覆盖 blocked 分支——此处验证 blocked 端到端需
|
||
// 定制 evaluate 响应，走 detectVendor 单测覆盖 + 本端到端验证 upstream 路径）。
|
||
_, eb := cdpFetchPage(context.Background(), f.url, "https://block.test/x", nil)
|
||
if eb != nil && eb.Code == contract.CodeBlocked {
|
||
t.Logf("端到端 blocked 命中: %s", eb.Message)
|
||
}
|
||
// fake 服务返回正常页：此调用应成功。
|
||
if eb != nil {
|
||
t.Fatalf("fake 正常页不应报错: %+v", eb)
|
||
}
|
||
}
|