chii
9b689b2476
公开页抓取改为 Chrome 136 自洽身份 + 每节点 SQLite 养罐，Trafilatura 走 curl_cffi；MCP/README/接手说明与 09-02 现网复测对齐，避免消费方继续抄过期的站点三分表。 Co-authored-by: Cursor <cursoragent@cursor.com>
340 lines
11 KiB
Go
340 lines
11 KiB
Go
package store
|
||
|
||
import (
|
||
"context"
|
||
"database/sql"
|
||
"fmt"
|
||
"time"
|
||
|
||
"onesvm.com/onesvm/browser-server/internal/config"
|
||
"onesvm.com/onesvm/browser-server/internal/contract"
|
||
"onesvm.com/onesvm/browser-server/internal/fingerprint"
|
||
)
|
||
|
||
// schemaV2 每节点模版 + Cookie 罐（P0/P1）。幂等 CREATE。
|
||
var schemaV2 = []string{
|
||
`CREATE TABLE IF NOT EXISTS fp_templates (
|
||
id TEXT PRIMARY KEY,
|
||
node_id TEXT NOT NULL,
|
||
impersonate TEXT NOT NULL,
|
||
user_agent TEXT NOT NULL,
|
||
sec_ch_ua TEXT NOT NULL,
|
||
sec_ch_ua_platform TEXT NOT NULL,
|
||
accept_language TEXT NOT NULL,
|
||
os TEXT NOT NULL,
|
||
chrome_major INTEGER NOT NULL,
|
||
created_at TEXT NOT NULL
|
||
)`,
|
||
`CREATE INDEX IF NOT EXISTS idx_fp_templates_node ON fp_templates(node_id)`,
|
||
`CREATE TABLE IF NOT EXISTS fp_state (
|
||
node_id TEXT PRIMARY KEY,
|
||
active_id TEXT NOT NULL,
|
||
updated_at TEXT NOT NULL
|
||
)`,
|
||
`CREATE TABLE IF NOT EXISTS session_cookies (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
template_id TEXT NOT NULL,
|
||
adapter_class TEXT NOT NULL,
|
||
egress TEXT NOT NULL,
|
||
etld TEXT NOT NULL,
|
||
name TEXT NOT NULL,
|
||
value TEXT NOT NULL,
|
||
domain TEXT NOT NULL,
|
||
path TEXT NOT NULL DEFAULT '/',
|
||
expires_at TEXT NOT NULL,
|
||
last_used_at TEXT NOT NULL,
|
||
bytes INTEGER NOT NULL,
|
||
UNIQUE(template_id, adapter_class, egress, etld, name, domain, path)
|
||
)`,
|
||
`CREATE INDEX IF NOT EXISTS idx_session_cookies_lookup
|
||
ON session_cookies(template_id, adapter_class, egress, etld)`,
|
||
`CREATE INDEX IF NOT EXISTS idx_session_cookies_expiry ON session_cookies(expires_at)`,
|
||
`CREATE INDEX IF NOT EXISTS idx_session_cookies_lru ON session_cookies(last_used_at)`,
|
||
}
|
||
|
||
func migrateV2(d *DB) error {
|
||
for _, stmt := range schemaV2 {
|
||
if _, err := d.raw.Exec(stmt); err != nil {
|
||
return fmt.Errorf("store: migration v2: %w (stmt=%s)", err, firstLine(stmt))
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// EnsureNodeTemplates 首次为节点写入 TemplateCount 套模版并选定 active。
|
||
func (d *DB) EnsureNodeTemplates(ctx context.Context, nodeID string) (fingerprint.Profile, error) {
|
||
if err := ctx.Err(); err != nil {
|
||
return fingerprint.Profile{}, err
|
||
}
|
||
var n int
|
||
if err := d.raw.QueryRowContext(ctx,
|
||
`SELECT COUNT(*) FROM fp_templates WHERE node_id=?`, nodeID).Scan(&n); err != nil {
|
||
return fingerprint.Profile{}, fmt.Errorf("store: 数模版: %w", err)
|
||
}
|
||
now := config.FormatTime(config.Now())
|
||
if n == 0 {
|
||
set := fingerprint.AllocateSet(nodeID, fingerprint.TemplateCount)
|
||
for _, p := range set {
|
||
if _, err := d.raw.ExecContext(ctx, `INSERT INTO fp_templates
|
||
(id, node_id, impersonate, user_agent, sec_ch_ua, sec_ch_ua_platform,
|
||
accept_language, os, chrome_major, created_at)
|
||
VALUES (?,?,?,?,?,?,?,?,?,?)`,
|
||
p.ID, nodeID, p.Impersonate, p.UserAgent, p.SecCHUA, p.SecCHUAPlatform,
|
||
p.AcceptLanguage, p.OS, p.ChromeMajor, now); err != nil {
|
||
return fingerprint.Profile{}, fmt.Errorf("store: 写模版: %w", err)
|
||
}
|
||
}
|
||
active := fingerprint.PickActive(nodeID, set)
|
||
if _, err := d.raw.ExecContext(ctx,
|
||
`INSERT INTO fp_state (node_id, active_id, updated_at) VALUES (?,?,?)`,
|
||
nodeID, active.ID, now); err != nil {
|
||
return fingerprint.Profile{}, fmt.Errorf("store: 写 active: %w", err)
|
||
}
|
||
return active, nil
|
||
}
|
||
var activeID string
|
||
err := d.raw.QueryRowContext(ctx, `SELECT active_id FROM fp_state WHERE node_id=?`, nodeID).Scan(&activeID)
|
||
if err == sql.ErrNoRows || activeID == "" {
|
||
set, err := d.ListTemplates(ctx, nodeID)
|
||
if err != nil {
|
||
return fingerprint.Profile{}, err
|
||
}
|
||
active := fingerprint.PickActive(nodeID, set)
|
||
if _, err := d.raw.ExecContext(ctx,
|
||
`INSERT INTO fp_state (node_id, active_id, updated_at) VALUES (?,?,?)
|
||
ON CONFLICT(node_id) DO UPDATE SET active_id=excluded.active_id, updated_at=excluded.updated_at`,
|
||
nodeID, active.ID, now); err != nil {
|
||
return fingerprint.Profile{}, fmt.Errorf("store: 补 active: %w", err)
|
||
}
|
||
return active, nil
|
||
}
|
||
if err != nil {
|
||
return fingerprint.Profile{}, fmt.Errorf("store: 读 active: %w", err)
|
||
}
|
||
return d.GetTemplate(ctx, activeID)
|
||
}
|
||
|
||
// ListTemplates 节点全部模版。
|
||
func (d *DB) ListTemplates(ctx context.Context, nodeID string) ([]fingerprint.Profile, error) {
|
||
rows, err := d.raw.QueryContext(ctx,
|
||
`SELECT id, impersonate, user_agent, sec_ch_ua, sec_ch_ua_platform,
|
||
accept_language, os, chrome_major
|
||
FROM fp_templates WHERE node_id=? ORDER BY id`, nodeID)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("store: 列模版: %w", err)
|
||
}
|
||
defer rows.Close()
|
||
var out []fingerprint.Profile
|
||
for rows.Next() {
|
||
var p fingerprint.Profile
|
||
if err := rows.Scan(&p.ID, &p.Impersonate, &p.UserAgent, &p.SecCHUA, &p.SecCHUAPlatform,
|
||
&p.AcceptLanguage, &p.OS, &p.ChromeMajor); err != nil {
|
||
return nil, fmt.Errorf("store: 扫模版: %w", err)
|
||
}
|
||
p.SecCHUAMobile = "?0"
|
||
out = append(out, p)
|
||
}
|
||
return out, rows.Err()
|
||
}
|
||
|
||
// GetTemplate 按 id 取模版。
|
||
func (d *DB) GetTemplate(ctx context.Context, id string) (fingerprint.Profile, error) {
|
||
var p fingerprint.Profile
|
||
err := d.raw.QueryRowContext(ctx,
|
||
`SELECT id, impersonate, user_agent, sec_ch_ua, sec_ch_ua_platform,
|
||
accept_language, os, chrome_major
|
||
FROM fp_templates WHERE id=?`, id).Scan(
|
||
&p.ID, &p.Impersonate, &p.UserAgent, &p.SecCHUA, &p.SecCHUAPlatform,
|
||
&p.AcceptLanguage, &p.OS, &p.ChromeMajor)
|
||
if err != nil {
|
||
return p, fmt.Errorf("store: 读模版 %s: %w", id, err)
|
||
}
|
||
p.SecCHUAMobile = "?0"
|
||
return p, nil
|
||
}
|
||
|
||
// SwitchActive 快速切到已落库的另一套模版。
|
||
func (d *DB) SwitchActive(ctx context.Context, nodeID, templateID string) error {
|
||
now := config.FormatTime(config.Now())
|
||
_, err := d.raw.ExecContext(ctx,
|
||
`UPDATE fp_state SET active_id=?, updated_at=? WHERE node_id=?`,
|
||
templateID, now, nodeID)
|
||
if err != nil {
|
||
return fmt.Errorf("store: 切模版: %w", err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// CookieKey 罐主键（模版 × 适配器档 × 出口 × 域）。
|
||
type CookieKey struct {
|
||
TemplateID string
|
||
AdapterClass string
|
||
Egress string
|
||
ETLD string
|
||
}
|
||
|
||
// LoadCookies 读一罐（未过期）。
|
||
func (d *DB) LoadCookies(ctx context.Context, k CookieKey) ([]contract.Cookie, error) {
|
||
now := config.FormatTime(config.Now())
|
||
rows, err := d.raw.QueryContext(ctx,
|
||
`SELECT name, value, domain, path, expires_at FROM session_cookies
|
||
WHERE template_id=? AND adapter_class=? AND egress=? AND etld=? AND expires_at>?`,
|
||
k.TemplateID, k.AdapterClass, k.Egress, k.ETLD, now)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("store: 读罐: %w", err)
|
||
}
|
||
defer rows.Close()
|
||
var out []contract.Cookie
|
||
for rows.Next() {
|
||
var c contract.Cookie
|
||
var exp string
|
||
if err := rows.Scan(&c.Name, &c.Value, &c.Domain, &c.Path, &exp); err != nil {
|
||
return nil, fmt.Errorf("store: 扫罐: %w", err)
|
||
}
|
||
if t, err := config.ParseRFC3339(exp); err == nil {
|
||
c.Expires = t.Unix()
|
||
}
|
||
out = append(out, c)
|
||
}
|
||
if err := rows.Err(); err != nil {
|
||
return nil, err
|
||
}
|
||
if len(out) > 0 {
|
||
_, _ = d.raw.ExecContext(ctx,
|
||
`UPDATE session_cookies SET last_used_at=?
|
||
WHERE template_id=? AND adapter_class=? AND egress=? AND etld=?`,
|
||
now, k.TemplateID, k.AdapterClass, k.Egress, k.ETLD)
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
// ReplaceCookies 覆盖一域的罐（先删后插）。调用方已 Sanitize。
|
||
func (d *DB) ReplaceCookies(ctx context.Context, k CookieKey, cookies []contract.Cookie) error {
|
||
nowS := config.FormatTime(config.Now())
|
||
tx, err := d.raw.BeginTx(ctx, nil)
|
||
if err != nil {
|
||
return fmt.Errorf("store: 开事务: %w", err)
|
||
}
|
||
defer tx.Rollback() // Commit 后幂等
|
||
if _, err := tx.ExecContext(ctx,
|
||
`DELETE FROM session_cookies WHERE template_id=? AND adapter_class=? AND egress=? AND etld=?`,
|
||
k.TemplateID, k.AdapterClass, k.Egress, k.ETLD); err != nil {
|
||
return fmt.Errorf("store: 清罐: %w", err)
|
||
}
|
||
for _, c := range cookies {
|
||
expS := nowS
|
||
if c.Expires > 0 {
|
||
expS = config.FormatTime(time.Unix(c.Expires, 0))
|
||
}
|
||
bytes := len(c.Name) + len(c.Value) + len(c.Domain) + len(c.Path)
|
||
if _, err := tx.ExecContext(ctx, `INSERT INTO session_cookies
|
||
(template_id, adapter_class, egress, etld, name, value, domain, path,
|
||
expires_at, last_used_at, bytes)
|
||
VALUES (?,?,?,?,?,?,?,?,?,?,?)`,
|
||
k.TemplateID, k.AdapterClass, k.Egress, k.ETLD,
|
||
c.Name, c.Value, c.Domain, c.Path, expS, nowS, bytes); err != nil {
|
||
return fmt.Errorf("store: 插 Cookie: %w", err)
|
||
}
|
||
}
|
||
if err := tx.Commit(); err != nil {
|
||
return fmt.Errorf("store: 提交罐: %w", err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// DropCookies 整域丢罐（投毒 / 换出口）。
|
||
func (d *DB) DropCookies(ctx context.Context, k CookieKey) error {
|
||
_, err := d.raw.ExecContext(ctx,
|
||
`DELETE FROM session_cookies WHERE template_id=? AND adapter_class=? AND egress=? AND etld=?`,
|
||
k.TemplateID, k.AdapterClass, k.Egress, k.ETLD)
|
||
if err != nil {
|
||
return fmt.Errorf("store: 丢罐: %w", err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// ReapCookies 过期删除 + 总字节/域数硬顶（LRU）。
|
||
func (d *DB) ReapCookies(ctx context.Context, maxETLD, maxBytes int) (int, error) {
|
||
now := config.FormatTime(config.Now())
|
||
res, err := d.raw.ExecContext(ctx, `DELETE FROM session_cookies WHERE expires_at<=?`, now)
|
||
if err != nil {
|
||
return 0, fmt.Errorf("store: 收过期罐: %w", err)
|
||
}
|
||
n, _ := res.RowsAffected()
|
||
cut, err := d.enforceJarCaps(ctx, maxETLD, maxBytes)
|
||
return int(n) + cut, err
|
||
}
|
||
|
||
func (d *DB) enforceJarCaps(ctx context.Context, maxETLD, maxBytes int) (int, error) {
|
||
cut := 0
|
||
// 按 (template,adapter,egress) 组限制域数。
|
||
rows, err := d.raw.QueryContext(ctx,
|
||
`SELECT template_id, adapter_class, egress, etld, MAX(last_used_at) AS lu
|
||
FROM session_cookies
|
||
GROUP BY template_id, adapter_class, egress, etld
|
||
ORDER BY template_id, adapter_class, egress, lu ASC`)
|
||
if err != nil {
|
||
return 0, fmt.Errorf("store: 域统计: %w", err)
|
||
}
|
||
type grp struct{ tpl, cls, eg, etld string }
|
||
counts := map[string][]grp{}
|
||
for rows.Next() {
|
||
var g grp
|
||
var lu string
|
||
if err := rows.Scan(&g.tpl, &g.cls, &g.eg, &g.etld, &lu); err != nil {
|
||
rows.Close()
|
||
return cut, err
|
||
}
|
||
key := g.tpl + "\x00" + g.cls + "\x00" + g.eg
|
||
counts[key] = append(counts[key], g)
|
||
}
|
||
if err := rows.Err(); err != nil {
|
||
rows.Close()
|
||
return cut, err
|
||
}
|
||
rows.Close()
|
||
for _, list := range counts {
|
||
if len(list) <= maxETLD {
|
||
continue
|
||
}
|
||
overflow := list[:len(list)-maxETLD] // 已按 lu ASC，最旧在前
|
||
for _, g := range overflow {
|
||
r, err := d.raw.ExecContext(ctx,
|
||
`DELETE FROM session_cookies WHERE template_id=? AND adapter_class=? AND egress=? AND etld=?`,
|
||
g.tpl, g.cls, g.eg, g.etld)
|
||
if err != nil {
|
||
return cut, fmt.Errorf("store: LRU 域: %w", err)
|
||
}
|
||
c, _ := r.RowsAffected()
|
||
cut += int(c)
|
||
}
|
||
}
|
||
var total int
|
||
if err := d.raw.QueryRowContext(ctx, `SELECT COALESCE(SUM(bytes),0) FROM session_cookies`).Scan(&total); err != nil {
|
||
return cut, fmt.Errorf("store: 字节合计: %w", err)
|
||
}
|
||
for total > maxBytes {
|
||
var tpl, cls, eg, etld string
|
||
err := d.raw.QueryRowContext(ctx,
|
||
`SELECT template_id, adapter_class, egress, etld FROM session_cookies
|
||
ORDER BY last_used_at ASC LIMIT 1`).Scan(&tpl, &cls, &eg, &etld)
|
||
if err == sql.ErrNoRows {
|
||
break
|
||
}
|
||
if err != nil {
|
||
return cut, fmt.Errorf("store: 找最旧域: %w", err)
|
||
}
|
||
r, err := d.raw.ExecContext(ctx,
|
||
`DELETE FROM session_cookies WHERE template_id=? AND adapter_class=? AND egress=? AND etld=?`,
|
||
tpl, cls, eg, etld)
|
||
if err != nil {
|
||
return cut, err
|
||
}
|
||
c, _ := r.RowsAffected()
|
||
cut += int(c)
|
||
if err := d.raw.QueryRowContext(ctx, `SELECT COALESCE(SUM(bytes),0) FROM session_cookies`).Scan(&total); err != nil {
|
||
return cut, err
|
||
}
|
||
}
|
||
return cut, nil
|
||
}
|