公开页抓取改为 Chrome 136 自洽身份 + 每节点 SQLite 养罐,Trafilatura 走 curl_cffi;MCP/README/接手说明与 09-02 现网复测对齐,避免消费方继续抄过期的站点三分表。 Co-authored-by: Cursor <cursoragent@cursor.com>
140 lines
4.2 KiB
Go
140 lines
4.2 KiB
Go
// httpx_test.go:守卫 hook / 类型白名单 / 错误归类(环回测试,无外网)。
|
||
package httpx
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"io"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"strings"
|
||
"testing"
|
||
)
|
||
|
||
// fakeGuard 测试用守卫(可编程拒绝)。
|
||
type fakeGuard struct{ deny map[string]bool }
|
||
|
||
func (f fakeGuard) RedirectCheck(_ context.Context, url string) error {
|
||
if f.deny[url] {
|
||
return &DeniedError{Reason: "test deny"}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// TestContentTypeWhitelist 类型白名单判定。
|
||
func TestContentTypeWhitelist(t *testing.T) {
|
||
cases := map[string]bool{
|
||
"text/html; charset=utf-8": true,
|
||
"application/json": true,
|
||
"application/pdf": true,
|
||
"text/xml": true,
|
||
"text/csv": true, // text/* 放行
|
||
"application/octet-stream": false,
|
||
"image/png": false,
|
||
"video/mp4": false,
|
||
"": true, // 空 Content-Type 保守放行
|
||
}
|
||
for ct, want := range cases {
|
||
if got := allowedType(ct); got != want {
|
||
t.Errorf("allowedType(%q)=%v want %v", ct, got, want)
|
||
}
|
||
}
|
||
}
|
||
|
||
// TestMaxBytesGuard 5MB 上限守卫。
|
||
func TestMaxBytesGuard(t *testing.T) {
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||
w.Header().Set("Content-Type", "text/html")
|
||
_, _ = io.Copy(w, strings.NewReader(strings.Repeat("a", 4096)))
|
||
}))
|
||
defer srv.Close()
|
||
c := New(nil, 1024) // 上限 1KB,响应 4KB → blocked
|
||
resp, err := c.Get(context.Background(), srv.URL)
|
||
if err == nil {
|
||
t.Fatalf("超限应拒绝: %+v", resp)
|
||
}
|
||
if _, ok := err.(*BlockedError); !ok {
|
||
t.Fatalf("应 BlockedError: %v", err)
|
||
}
|
||
}
|
||
|
||
// TestRedirectGuard 重定向每跳重验。
|
||
func TestRedirectGuard(t *testing.T) {
|
||
var calls int
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
calls++
|
||
if calls == 1 {
|
||
http.Redirect(w, r, "/step2", http.StatusFound)
|
||
return
|
||
}
|
||
w.Header().Set("Content-Type", "text/plain")
|
||
_, _ = w.Write([]byte("ok"))
|
||
}))
|
||
defer srv.Close()
|
||
// 守卫放行
|
||
c := New(fakeGuard{deny: map[string]bool{}}, 0)
|
||
resp, err := c.Get(context.Background(), srv.URL)
|
||
if err != nil {
|
||
t.Fatalf("放行路径应成功: %v", err)
|
||
}
|
||
if resp.StatusCode != 200 || resp.FinalURL == "" {
|
||
t.Fatalf("响应不符: %+v", resp)
|
||
}
|
||
// 守卫拒绝目标跳:CheckRedirect 对相对 Location 以 req.URL.String() 传入,
|
||
// Go 会先解析为绝对 URL,但错误信息显示传入形态含相对路径;deny 表同时覆盖两种形态。
|
||
calls = 0 // 重置:让 c2 的首跳重新 302 到 /step2
|
||
denyTarget := srv.URL + "/step2"
|
||
c2 := New(fakeGuard{deny: map[string]bool{denyTarget: true, "/step2": true}}, 0)
|
||
_, err = c2.Get(context.Background(), srv.URL)
|
||
if err == nil {
|
||
t.Fatal("守卫拒绝重定向目标应报错")
|
||
}
|
||
if !strings.Contains(err.Error(), "denied") {
|
||
t.Fatalf("拒绝错误应含 denied: %v", err)
|
||
}
|
||
}
|
||
|
||
// TestGuardNil 允许无守卫构造(环回测试场景)。
|
||
func TestGuardNil(t *testing.T) {
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||
w.Header().Set("Content-Type", "application/json")
|
||
_, _ = w.Write([]byte(`{"ok":true}`))
|
||
}))
|
||
defer srv.Close()
|
||
c := New(nil, 0)
|
||
resp, err := c.Get(context.Background(), srv.URL)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if resp.StatusCode != 200 || !strings.Contains(string(resp.Body), "ok") {
|
||
t.Fatalf("响应不符: %+v", resp)
|
||
}
|
||
}
|
||
|
||
// TestErrorCodeMapping 错误归类。
|
||
func TestErrorCodeMapping(t *testing.T) {
|
||
if got := ErrorCode(&BlockedError{Reason: "x"}); got != "blocked" {
|
||
t.Fatalf("blocked: %s", got)
|
||
}
|
||
if got := ErrorCode(&DeniedError{Reason: "x"}); got != "denied" {
|
||
t.Fatalf("denied: %s", got)
|
||
}
|
||
if got := ErrorCode(&TimeoutError{Reason: "x"}); got != "timeout" {
|
||
t.Fatalf("timeout: %s", got)
|
||
}
|
||
if got := ErrorCode(errors.New("net fail")); got != "upstream" {
|
||
t.Fatalf("upstream: %s", got)
|
||
}
|
||
}
|
||
|
||
// TestUserAgent UA 与 bench 一致。
|
||
func TestUserAgent(t *testing.T) {
|
||
if !strings.HasPrefix(UA, "onesvm-browser-server-httpx/") {
|
||
t.Fatalf("httpx 必须用诚实 UA,禁止 Chrome 冒充: %s", UA)
|
||
}
|
||
if strings.Contains(UA, "Chrome/") {
|
||
t.Fatal("Go TLS 客户端不得声称 Chrome")
|
||
}
|
||
}
|
||
|
||
var _ = io.Discard // 锚点:io 依赖保留
|