chii
eb972dfa93
单二进制三角色 + Dock 适配器 + Swarm stack 达到可部署态；mgr1 实测订阅经 central-proxy bootstrap，探活 alive=41/52。 Co-authored-by: Cursor <cursoragent@cursor.com>
247 lines
7.8 KiB
Go
247 lines
7.8 KiB
Go
// robots.go：robots.txt 缓存与 Disallow 校验（普通 key 遵守 robots，
|
||
// design-arch §5.4 域名策略行）。缓存落 SQLite robots_cache 表，TTL 24h。
|
||
package policy
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"io"
|
||
"net/http"
|
||
"net/url"
|
||
"strings"
|
||
"time"
|
||
|
||
"onesvm.com/onesvm/browser-server/internal/config"
|
||
"onesvm.com/onesvm/browser-server/internal/store"
|
||
)
|
||
|
||
// RobotsTTL 缓存有效期（design 值 24h）。
|
||
const RobotsTTL = 24 * time.Hour
|
||
|
||
// RobotsChecker robots 缓存 + 校验器。
|
||
type RobotsChecker struct {
|
||
db *store.DB
|
||
cli *http.Client
|
||
redir func(ctx context.Context, rawURL string) *DeniedError // 每跳重验（ITER-3 FIX-3；可 nil）
|
||
}
|
||
|
||
// NewRobotsChecker 构造；client 超时建议 ≤10s（拉 robots 专用）。redir 非 nil 时挂
|
||
// CheckRedirect 每跳重验（ITER-3 FIX-3 接线：robots 拉取的 3xx 目标同样过 SSRF 门闩）。
|
||
func NewRobotsChecker(db *store.DB, cli *http.Client) *RobotsChecker {
|
||
return &RobotsChecker{db: db, cli: cli}
|
||
}
|
||
|
||
// SetRedirectCheck 挂每跳重验钩子（Engine 组装后注入，避免构造环依赖）。
|
||
func (r *RobotsChecker) SetRedirectCheck(fn func(ctx context.Context, rawURL string) *DeniedError) {
|
||
r.redir = fn
|
||
}
|
||
|
||
// Allowed 判断普通 key 是否可抓取 targetURL（ITER-3 FIX-4：路径级判定——
|
||
// 从 targetURL 提取 u.Path（缺省 /），matchRobots 按 UA 段 + AllowedPath 前缀规则判定）。
|
||
// 1. 查缓存（SQLite，TTL 内直接用）；
|
||
// 2. 未命中拉 https://<host>/robots.txt（超时/非 200/无 body → 视为允许，fail-open 仅限 robots 拉取失败；
|
||
// 本判定不影响 SSRF/域名黑名单等 fail-closed 层）；
|
||
// 3. User-agent 匹配段 + path 前缀规则（最长匹配，Allow 同长胜）。
|
||
func (r *RobotsChecker) Allowed(ctx context.Context, targetURL, userAgent string) (bool, error) {
|
||
host := parseHost(targetURL)
|
||
if host == "" {
|
||
return false, fmt.Errorf("policy: robots: 空 host")
|
||
}
|
||
body, err := r.cachedBody(ctx, host)
|
||
if err != nil {
|
||
return true, nil // 拉取失败不阻塞（见函数注释），仅记 warning 由调用方处理
|
||
}
|
||
return matchRobots(body, userAgent, pathOfURL(targetURL)), nil
|
||
}
|
||
|
||
// cachedBody 读缓存或拉取并写缓存。
|
||
func (r *RobotsChecker) cachedBody(ctx context.Context, host string) (string, error) {
|
||
now := config.Now()
|
||
var body string
|
||
var expires string
|
||
err := r.db.Raw().QueryRow(
|
||
`SELECT body, expires_at FROM robots_cache WHERE host = ?`, host).Scan(&body, &expires)
|
||
if err == nil {
|
||
if exp, perr := time.Parse(time.RFC3339, expires); perr == nil && now.Before(exp) {
|
||
return body, nil // TTL 内
|
||
}
|
||
}
|
||
fetched, ferr := r.fetch(ctx, host)
|
||
if ferr != nil {
|
||
// stale-on-error：TTL 过期但有旧值时容忍旧值（design §5.1 同思路）。
|
||
if err == nil && body != "" {
|
||
return body, nil
|
||
}
|
||
return "", ferr
|
||
}
|
||
_, _ = r.db.Raw().Exec(
|
||
`INSERT INTO robots_cache(host, body, fetched_at, expires_at) VALUES(?, ?, ?, ?)
|
||
ON CONFLICT(host) DO UPDATE SET body=excluded.body, fetched_at=excluded.fetched_at, expires_at=excluded.expires_at`,
|
||
host, fetched, now.Format(time.RFC3339), now.Add(RobotsTTL).Format(time.RFC3339))
|
||
return fetched, nil
|
||
}
|
||
|
||
// fetch 拉 robots.txt（≤256KB，10s 超时；重定向每跳重验——ITER-3 FIX-3）。
|
||
func (r *RobotsChecker) fetch(ctx context.Context, host string) (string, error) {
|
||
scheme := "https"
|
||
raw := scheme + "://" + host + "/robots.txt"
|
||
cli := r.cli
|
||
if r.redir != nil {
|
||
// 包一层每跳重验：3xx 目标 URL 过 CheckURL+ResolveCheck（fail-closed）。
|
||
cli = &http.Client{
|
||
Timeout: r.cli.Timeout,
|
||
CheckRedirect: func(req *http.Request, via []*http.Request) error {
|
||
if len(via) >= 5 {
|
||
return fmt.Errorf("policy: robots 重定向超 5 跳")
|
||
}
|
||
if d := r.redir(req.Context(), req.URL.String()); d != nil {
|
||
return fmt.Errorf("policy: robots 重定向拦截（%s）: %s", d.RuleID, d.Reason)
|
||
}
|
||
return nil
|
||
},
|
||
}
|
||
}
|
||
cctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||
defer cancel()
|
||
req, err := http.NewRequestWithContext(cctx, http.MethodGet, raw, nil)
|
||
if err != nil {
|
||
return "", err
|
||
}
|
||
req.Header.Set("User-Agent", "onesvm-browser-server-robots/1.0")
|
||
resp, err := cli.Do(req)
|
||
if err != nil {
|
||
return "", fmt.Errorf("policy: robots 拉取 %s: %w", raw, err)
|
||
}
|
||
defer resp.Body.Close()
|
||
if resp.StatusCode != http.StatusOK {
|
||
return "", fmt.Errorf("policy: robots %s 状态 %d", raw, resp.StatusCode)
|
||
}
|
||
limited := io.LimitReader(resp.Body, 256*1024)
|
||
b, err := io.ReadAll(limited)
|
||
if err != nil {
|
||
return "", fmt.Errorf("policy: robots 读取: %w", err)
|
||
}
|
||
return string(b), nil
|
||
}
|
||
|
||
// parseHost 提取 host（含端口归一到 authority）。
|
||
func parseHost(targetURL string) string {
|
||
s := strings.TrimPrefix(targetURL, "https://")
|
||
s = strings.TrimPrefix(s, "http://")
|
||
if i := strings.IndexAny(s, "/?#"); i >= 0 {
|
||
s = s[:i]
|
||
}
|
||
return s
|
||
}
|
||
|
||
// matchRobots 解析 robots.txt：取 * 段（或与 userAgent 匹配段）规则，
|
||
// 对 path 执行前缀匹配（ITER-3 FIX-4：路径级判定走 AllowedPath 语义）。
|
||
// 返回 true = 允许。实现按 Google robots 规范简化：最长匹配优先，Allow 胜同长 Disallow。
|
||
func matchRobots(body, userAgent, path string) bool {
|
||
ua := strings.ToLower(strings.TrimSpace(userAgent))
|
||
starRules := [][2]string{} // [prefix, allow|disallow]
|
||
uaRules := [][2]string{}
|
||
cur := -1 // -1=none, 0=star, 1=ua
|
||
for _, line := range strings.Split(body, "\n") {
|
||
line = strings.TrimSpace(line)
|
||
if i := strings.Index(line, "#"); i >= 0 {
|
||
line = line[:i]
|
||
}
|
||
if line == "" {
|
||
continue
|
||
}
|
||
k, v, ok := cutKV(line)
|
||
if !ok {
|
||
continue
|
||
}
|
||
k = strings.ToLower(k)
|
||
switch k {
|
||
case "user-agent":
|
||
agent := strings.ToLower(strings.TrimSpace(v))
|
||
if agent == "*" {
|
||
cur = 0
|
||
} else if ua != "" && strings.Contains(agent, ua) {
|
||
cur = 1
|
||
} else {
|
||
cur = -1
|
||
}
|
||
case "disallow":
|
||
switch cur {
|
||
case 0:
|
||
starRules = append(starRules, [2]string{v, "disallow"})
|
||
case 1:
|
||
uaRules = append(uaRules, [2]string{v, "disallow"})
|
||
}
|
||
case "allow":
|
||
switch cur {
|
||
case 0:
|
||
starRules = append(starRules, [2]string{v, "allow"})
|
||
case 1:
|
||
uaRules = append(uaRules, [2]string{v, "allow"})
|
||
}
|
||
}
|
||
}
|
||
rules := uaRules
|
||
if len(rules) == 0 {
|
||
rules = starRules
|
||
}
|
||
return evalRules(rules, path)
|
||
}
|
||
|
||
// evalRules 规则求值：把 UA 段规则序列重放为 robots 文本后走 AllowedPath
|
||
// （最长匹配前缀，allow 同长胜）——判定语义与缓存命中分支完全同源（ITER-3 FIX-4）。
|
||
func evalRules(rules [][2]string, path string) bool {
|
||
var b strings.Builder
|
||
for _, r := range rules {
|
||
if r[0] == "" && r[1] == "disallow" {
|
||
continue // 空 Disallow = 允许全部（Google 规范），不入规则序列
|
||
}
|
||
b.WriteString(strings.ToUpper(r[1]) + ": " + r[0] + "\n")
|
||
}
|
||
return AllowedPath(b.String(), path)
|
||
}
|
||
|
||
// AllowedPath 前缀级校验（最长匹配优先，allow 同长胜 disallow）。
|
||
func AllowedPath(body, path string) bool {
|
||
var best [2]string
|
||
has := false
|
||
for _, line := range strings.Split(body, "\n") {
|
||
line = strings.TrimSpace(line)
|
||
if i := strings.Index(line, "#"); i >= 0 {
|
||
line = line[:i]
|
||
}
|
||
k, v, ok := cutKV(line)
|
||
if !ok {
|
||
continue
|
||
}
|
||
k = strings.ToLower(k)
|
||
if k != "allow" && k != "disallow" {
|
||
continue
|
||
}
|
||
if v != "" && strings.HasPrefix(path, v) && len(v) >= len(best[0]) {
|
||
best = [2]string{v, k}
|
||
has = true
|
||
}
|
||
}
|
||
if !has {
|
||
return true
|
||
}
|
||
return best[1] == "allow"
|
||
}
|
||
|
||
func cutKV(line string) (string, string, bool) {
|
||
i := strings.Index(line, ":")
|
||
if i < 0 {
|
||
return "", "", false
|
||
}
|
||
return strings.TrimSpace(line[:i]), strings.TrimSpace(line[i+1:]), true
|
||
}
|
||
|
||
// pathOfURL 从 targetURL 提取请求路径（缺省 "/"；ITER-3 FIX-4 替代 host_targetCache 占位）。
|
||
func pathOfURL(targetURL string) string {
|
||
u, err := url.Parse(targetURL)
|
||
if err != nil || u.Path == "" {
|
||
return "/"
|
||
}
|
||
return u.Path
|
||
}
|