公开页抓取改为 Chrome 136 自洽身份 + 每节点 SQLite 养罐,Trafilatura 走 curl_cffi;MCP/README/接手说明与 09-02 现网复测对齐,避免消费方继续抄过期的站点三分表。 Co-authored-by: Cursor <cursoragent@cursor.com>
145 lines
4.4 KiB
Go
145 lines
4.4 KiB
Go
// Package fingerprint 提供每节点稳定的浏览器身份模版(P0)。
|
||
//
|
||
// 自洽:impersonate / User-Agent / sec-ch-ua / platform 钉在同一 Chrome 大版本。
|
||
// 上限跟 curl_cffi 能仿的桌面档(chrome136),禁止再写 Chrome/151 + Go TLS。
|
||
package fingerprint
|
||
|
||
import (
|
||
"crypto/sha256"
|
||
"encoding/binary"
|
||
"fmt"
|
||
"os"
|
||
"strconv"
|
||
)
|
||
|
||
// 钉死与 curl_cffi 0.13 桌面上限对齐(DaaS fp-v2 同代)。
|
||
const (
|
||
ChromeMajor = 136
|
||
DefaultImpersonate = "chrome136"
|
||
TemplateCount = 8 // 每节点落库套数,便于切换;内存只持有 active
|
||
)
|
||
|
||
// Profile 一张自洽脸。字段均可落库。
|
||
type Profile struct {
|
||
ID string `json:"id"`
|
||
OS string `json:"os"` // Windows | macOS
|
||
ChromeMajor int `json:"chrome_major"`
|
||
Impersonate string `json:"impersonate"`
|
||
UserAgent string `json:"user_agent"`
|
||
SecCHUA string `json:"sec_ch_ua"`
|
||
SecCHUAMobile string `json:"sec_ch_ua_mobile"`
|
||
SecCHUAPlatform string `json:"sec_ch_ua_platform"`
|
||
AcceptLanguage string `json:"accept_language"`
|
||
}
|
||
|
||
var (
|
||
viewLangs = []string{
|
||
"en-US,en;q=0.9",
|
||
"en-US,en;q=0.8",
|
||
"en-US,en;q=0.9,zh-CN;q=0.6",
|
||
"zh-CN,zh;q=0.9,en;q=0.8",
|
||
"en-GB,en;q=0.9,en-US;q=0.8",
|
||
}
|
||
winNT = []string{"10.0", "10.0", "11.0"}
|
||
macOSX = []string{"10_15_7", "13_6_1", "14_4_0", "14_2_1"}
|
||
)
|
||
|
||
// NodeID 生产节点稳定种子(Swarm HOSTNAME;缺省本机 hostname)。
|
||
func NodeID() string {
|
||
if v := os.Getenv("HOSTNAME"); v != "" {
|
||
return v
|
||
}
|
||
h, err := os.Hostname()
|
||
if err != nil || h == "" {
|
||
return "unknown-node"
|
||
}
|
||
return h
|
||
}
|
||
|
||
// Allocate 由 seed 确定性生成一张脸(同 seed 永同脸)。
|
||
func Allocate(seed string) Profile {
|
||
s := seedInt(seed)
|
||
osFamily := "Windows"
|
||
if s%2 == 1 {
|
||
osFamily = "macOS"
|
||
}
|
||
major := ChromeMajor
|
||
impersonate := DefaultImpersonate
|
||
lang := viewLangs[(s>>8)%uint64(len(viewLangs))]
|
||
buildPatch := (s >> 16) % 10000
|
||
buildMinor := (s >> 4) % 10000
|
||
var ua, platform, tid string
|
||
if osFamily == "Windows" {
|
||
nt := winNT[(s>>12)%uint64(len(winNT))]
|
||
ua = fmt.Sprintf(
|
||
"Mozilla/5.0 (Windows NT %s; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%d.0.%d.%d Safari/537.36",
|
||
nt, major, buildPatch, buildMinor)
|
||
platform = `"Windows"`
|
||
tid = fmt.Sprintf("win-chrome%d-%06d", major, s%1_000_000)
|
||
} else {
|
||
osx := macOSX[(s>>12)%uint64(len(macOSX))]
|
||
ua = fmt.Sprintf(
|
||
"Mozilla/5.0 (Macintosh; Intel Mac OS X %s) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%d.0.%d.%d Safari/537.36",
|
||
osx, major, buildPatch, buildMinor)
|
||
platform = `"macOS"`
|
||
tid = fmt.Sprintf("mac-chrome%d-%06d", major, s%1_000_000)
|
||
}
|
||
return Profile{
|
||
ID: tid,
|
||
OS: osFamily,
|
||
ChromeMajor: major,
|
||
Impersonate: impersonate,
|
||
UserAgent: ua,
|
||
SecCHUA: secCHUA(major),
|
||
SecCHUAMobile: "?0",
|
||
SecCHUAPlatform: platform,
|
||
AcceptLanguage: lang,
|
||
}
|
||
}
|
||
|
||
// AllocateSet 为节点生成 n 套互异模版。
|
||
func AllocateSet(nodeID string, n int) []Profile {
|
||
if n <= 0 {
|
||
n = TemplateCount
|
||
}
|
||
out := make([]Profile, n)
|
||
for i := 0; i < n; i++ {
|
||
out[i] = Allocate(nodeID + "|" + strconv.Itoa(i))
|
||
}
|
||
return out
|
||
}
|
||
|
||
// PickActive 从集合里按节点 hash 选主模版(重启不变)。
|
||
func PickActive(nodeID string, set []Profile) Profile {
|
||
if len(set) == 0 {
|
||
return Allocate(nodeID)
|
||
}
|
||
return set[int(seedInt(nodeID+"|active")%uint64(len(set)))]
|
||
}
|
||
|
||
// Headers 与 impersonate 对齐的导航头(顺序接近桌面 Chrome)。
|
||
func Headers(p Profile) map[string]string {
|
||
return map[string]string{
|
||
"User-Agent": p.UserAgent,
|
||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
|
||
"Accept-Language": p.AcceptLanguage,
|
||
"Accept-Encoding": "gzip, deflate, br",
|
||
"Upgrade-Insecure-Requests": "1",
|
||
"Sec-Fetch-Dest": "document",
|
||
"Sec-Fetch-Mode": "navigate",
|
||
"Sec-Fetch-Site": "none",
|
||
"Sec-Fetch-User": "?1",
|
||
"sec-ch-ua": p.SecCHUA,
|
||
"sec-ch-ua-mobile": p.SecCHUAMobile,
|
||
"sec-ch-ua-platform": p.SecCHUAPlatform,
|
||
}
|
||
}
|
||
|
||
func secCHUA(major int) string {
|
||
return fmt.Sprintf(`"Google Chrome";v="%d", "Chromium";v="%d", "Not_A Brand";v="24"`, major, major)
|
||
}
|
||
|
||
func seedInt(s string) uint64 {
|
||
sum := sha256.Sum256([]byte("fp-v2|" + s))
|
||
return binary.BigEndian.Uint64(sum[:8])
|
||
}
|