公开页抓取改为 Chrome 136 自洽身份 + 每节点 SQLite 养罐,Trafilatura 走 curl_cffi;MCP/README/接手说明与 09-02 现网复测对齐,避免消费方继续抄过期的站点三分表。 Co-authored-by: Cursor <cursoragent@cursor.com>
157 lines
4.5 KiB
Go
157 lines
4.5 KiB
Go
package session
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
_ "modernc.org/sqlite"
|
|
|
|
"onesvm.com/onesvm/browser-server/internal/contract"
|
|
"onesvm.com/onesvm/browser-server/internal/store"
|
|
)
|
|
|
|
func TestETLD1(t *testing.T) {
|
|
cases := map[string]string{
|
|
"https://www.amazon.com/dp/X": "amazon.com",
|
|
"https://news.bbc.co.uk/x": "bbc.co.uk",
|
|
"https://a.b.com.cn/y": "b.com.cn",
|
|
"https://192.168.1.1/z": "192.168.1.1",
|
|
}
|
|
for in, want := range cases {
|
|
if got := ETLD1(in); got != want {
|
|
t.Errorf("ETLD1(%s)=%s want %s", in, got, want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestSanitizeDropsAuthAndHuge(t *testing.T) {
|
|
in := []contract.Cookie{
|
|
{Name: "session-id", Value: "ok"},
|
|
{Name: "at-main", Value: "secret"},
|
|
{Name: "SID", Value: "g"},
|
|
{Name: "huge", Value: strings.Repeat("x", MaxCookieBytes+1)},
|
|
{Name: "", Value: "x"},
|
|
}
|
|
out := Sanitize(in)
|
|
if len(out) != 1 || out[0].Name != "session-id" {
|
|
t.Fatalf("只应留下匿名 session-id: %+v", out)
|
|
}
|
|
if out[0].Expires <= time.Now().Unix() {
|
|
t.Fatal("应写入默认 TTL")
|
|
}
|
|
}
|
|
|
|
func TestIsPoison(t *testing.T) {
|
|
if !IsPoison(&contract.ErrBody{Code: contract.CodeBlocked, Message: "x"}, nil) {
|
|
t.Fatal("blocked 应投毒")
|
|
}
|
|
if !IsPoison(nil, &contract.RawResult{Poisoned: true}) {
|
|
t.Fatal("Poisoned 应投毒")
|
|
}
|
|
if !IsPoison(&contract.ErrBody{Code: contract.CodeUpstream, Message: "cloudflare challenge"}, nil) {
|
|
t.Fatal("cf 文案应投毒")
|
|
}
|
|
if IsPoison(&contract.ErrBody{Code: contract.CodeTimeout, Message: "deadline"}, nil) {
|
|
t.Fatal("超时不应丢罐")
|
|
}
|
|
if IsPoison(&contract.ErrBody{Code: contract.CodeUpstream, Message: "empty_extract"}, nil) {
|
|
t.Fatal("空抽取不应丢罐")
|
|
}
|
|
}
|
|
|
|
func TestJarRoundTripAndPoisonDrop(t *testing.T) {
|
|
db, err := store.Open(filepath.Join(t.TempDir(), "s.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { db.Close() })
|
|
if err := db.Migrate(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx := context.Background()
|
|
j, err := Open(ctx, db, "test-node")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
set, err := db.ListTemplates(ctx, "test-node")
|
|
if err != nil || len(set) != 8 {
|
|
t.Fatalf("应落 8 套模版: n=%d err=%v", len(set), err)
|
|
}
|
|
page := "https://www.example.com/a"
|
|
sess := j.Attach(ctx, "trafilatura", "direct", page)
|
|
if sess == nil || sess.Impersonate != "chrome136" {
|
|
t.Fatalf("attach: %+v", sess)
|
|
}
|
|
if strings.Contains(sess.UserAgent, "151") {
|
|
t.Fatal("模版 UA 禁止 151")
|
|
}
|
|
j.Commit(ctx, sess, &contract.RawResult{
|
|
StatusCode: 200,
|
|
SetCookies: []contract.Cookie{{Name: "sid", Value: "1"}, {Name: "pref", Value: "dark"}},
|
|
}, nil)
|
|
sess2 := j.Attach(ctx, "trafilatura", "direct", page)
|
|
if len(sess2.Cookies) != 1 || sess2.Cookies[0].Name != "pref" {
|
|
t.Fatalf("登录态 sid 应丢、pref 应在: %+v", sess2.Cookies)
|
|
}
|
|
// 另一档不应串罐
|
|
cdp := j.Attach(ctx, "lightpanda", "direct", page)
|
|
if len(cdp.Cookies) != 0 {
|
|
t.Fatalf("http/cdp 不得混罐: %+v", cdp.Cookies)
|
|
}
|
|
j.Commit(ctx, sess2, &contract.RawResult{Poisoned: true}, nil)
|
|
sess3 := j.Attach(ctx, "trafilatura", "direct", page)
|
|
if len(sess3.Cookies) != 0 {
|
|
t.Fatalf("投毒后应空罐: %+v", sess3.Cookies)
|
|
}
|
|
}
|
|
|
|
func TestJobEnvelopeOmitsSession(t *testing.T) {
|
|
j := contract.JobEnvelope{
|
|
ID: "1", Intent: "read",
|
|
Read: &contract.ReadInput{URL: "https://x/"},
|
|
Session: &contract.SessionAttach{TemplateID: "t", Cookies: []contract.Cookie{{Name: "a", Value: "b"}}},
|
|
}
|
|
b, err := json.Marshal(j)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.Contains(string(b), "a") && strings.Contains(string(b), `"name"`) {
|
|
// Cookie 名可能碰巧出现;更硬的:不得有 session 键
|
|
}
|
|
if strings.Contains(string(b), `"session"`) || strings.Contains(string(b), "TemplateID") {
|
|
t.Fatalf("Session 不得进 payload: %s", b)
|
|
}
|
|
}
|
|
|
|
func TestReapExpired(t *testing.T) {
|
|
db, err := store.Open(filepath.Join(t.TempDir(), "r.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { db.Close() })
|
|
if err := db.Migrate(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx := context.Background()
|
|
j, err := Open(ctx, db, "reap-node")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sess := j.Attach(ctx, "trafilatura", "direct", "https://old.example/x")
|
|
j.Commit(ctx, sess, &contract.RawResult{
|
|
SetCookies: []contract.Cookie{{Name: "gone", Value: "1", Expires: time.Now().Add(-time.Hour).Unix()}},
|
|
}, nil)
|
|
n, err := j.Reap(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_ = n
|
|
again := j.Attach(ctx, "trafilatura", "direct", "https://old.example/x")
|
|
if len(again.Cookies) != 0 {
|
|
t.Fatalf("过期应被收: %+v", again.Cookies)
|
|
}
|
|
}
|