chii
9b689b2476
公开页抓取改为 Chrome 136 自洽身份 + 每节点 SQLite 养罐，Trafilatura 走 curl_cffi；MCP/README/接手说明与 09-02 现网复测对齐，避免消费方继续抄过期的站点三分表。 Co-authored-by: Cursor <cursoragent@cursor.com>
307 lines
12 KiB
Go
307 lines
12 KiB
Go
// cmd/browser-server 单二进制多角色入口（design-arch D7）：
|
||
//
|
||
// go run ./cmd/browser-server -role=gateway # :8640 MCP + HTTP + healthz
|
||
// go run ./cmd/browser-server -role=scheduler # :8641 队列/调度
|
||
// go run ./cmd/browser-server -role=proxymanager # :8642 代理控制面
|
||
//
|
||
// 骨架门（本步 W1）：角色实现以 TODO(role) 占位接口挂上，main 可编译运行、
|
||
// healthz 可起；W2–W4 在各自 roleRunXxx 内填实现。
|
||
package main
|
||
|
||
import (
|
||
"context"
|
||
"flag"
|
||
"fmt"
|
||
"log"
|
||
"net/http"
|
||
"os"
|
||
"os/signal"
|
||
"path/filepath"
|
||
"strings"
|
||
"syscall"
|
||
"time"
|
||
|
||
// SQLite driver 注册（modernc 纯 Go；store.Open 依赖，二进制入口统一注册）。
|
||
_ "modernc.org/sqlite"
|
||
|
||
"onesvm.com/onesvm/browser-server/internal/auth"
|
||
"onesvm.com/onesvm/browser-server/internal/config"
|
||
"onesvm.com/onesvm/browser-server/internal/dock"
|
||
"onesvm.com/onesvm/browser-server/internal/fingerprint"
|
||
gateway "onesvm.com/onesvm/browser-server/internal/gateway"
|
||
"onesvm.com/onesvm/browser-server/internal/policy"
|
||
"onesvm.com/onesvm/browser-server/internal/proxymanager"
|
||
"onesvm.com/onesvm/browser-server/internal/scheduler"
|
||
"onesvm.com/onesvm/browser-server/internal/session"
|
||
"onesvm.com/onesvm/browser-server/internal/store"
|
||
)
|
||
|
||
// role 角色名。
|
||
const (
|
||
roleGateway = "gateway"
|
||
roleScheduler = "scheduler"
|
||
roleProxyManager = "proxymanager"
|
||
)
|
||
|
||
func main() {
|
||
role := flag.String("role", roleGateway, "运行角色: gateway|scheduler|proxymanager")
|
||
addr := flag.String("addr", "", "监听地址（缺省按角色默认端口）")
|
||
probe := flag.Bool("healthz-probe", false, "健康探测模式：GET 本服务 /healthz 成功即退出 0（Swarm healthcheck 用，distroless 无 curl）")
|
||
flag.Parse()
|
||
|
||
// 健康探测模式：distroless 运行镜像无 shell 工具，Swarm healthcheck 直接调
|
||
// 二进制探测自身 /healthz（stack 内 ["CMD","/browser-server","-healthz-probe"]）。
|
||
if *probe {
|
||
os.Exit(healthzProbe(listenAddr(*role, *addr)))
|
||
}
|
||
|
||
logger := log.New(os.Stdout, fmt.Sprintf("[browser-server/%s] ", *role), log.LstdFlags)
|
||
logger.Printf("启动 role=%s tz=%s", *role, config.Timezone)
|
||
|
||
var run func(ctx context.Context, addr string, log *log.Logger) error
|
||
switch *role {
|
||
case roleGateway:
|
||
run = roleRunGateway
|
||
case roleScheduler:
|
||
run = roleRunScheduler
|
||
case roleProxyManager:
|
||
run = roleRunProxyManager
|
||
default:
|
||
logger.Fatalf("未知 role=%q（可选 gateway|scheduler|proxymanager）", *role)
|
||
return
|
||
}
|
||
|
||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||
defer stop()
|
||
|
||
if err := run(ctx, listenAddr(*role, *addr), logger); err != nil {
|
||
logger.Fatalf("角色 %s 退出: %v", *role, err)
|
||
}
|
||
logger.Printf("role=%s 已优雅退出", *role)
|
||
}
|
||
|
||
// listenAddr 角色默认端口（D3 锁定：8640/8641/8642；env 可覆盖，非密钥）。
|
||
func listenAddr(role, override string) string {
|
||
if override != "" {
|
||
return override
|
||
}
|
||
def := map[string]string{
|
||
roleGateway: ":8640",
|
||
roleScheduler: ":8641",
|
||
roleProxyManager: ":8642",
|
||
}
|
||
return config.EnvDefault("BROWSER_SERVER_LISTEN_"+role, def[role])
|
||
}
|
||
|
||
// serveHealth 统一启动 HTTP 服务：healthz + 角色挂载的 handler，阻塞至 ctx 取消。
|
||
// 注意：healthz 挂载在 gateway.Handler 内部已含（gateway 路由含 GET /healthz）；
|
||
// 其它角色传 *http.ServeMux 时此处补挂 healthz。
|
||
func serveHealth(ctx context.Context, addr string, logger *log.Logger, handler http.Handler) error {
|
||
if mux, ok := handler.(*http.ServeMux); ok && muxPatternRegistered(mux, "/healthz") == nil {
|
||
// V3 arch-review D-1：PM/scheduler Handler 已自挂 /healthz；ServeMux 同
|
||
// pattern 双注册启动即 panic（生产 stack PM 起不来）。此处先探测已注册
|
||
// 则跳过——根治（healthz 挂载权全归角色 Handler）留拆分轮。
|
||
mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) {
|
||
w.Header().Set("Content-Type", "application/json")
|
||
w.WriteHeader(http.StatusOK)
|
||
_, _ = w.Write([]byte(`{"ok":true}`))
|
||
})
|
||
}
|
||
srv := &http.Server{Addr: addr, Handler: handler, ReadHeaderTimeout: 5 * time.Second}
|
||
errCh := make(chan error, 1)
|
||
go func() {
|
||
logger.Printf("healthz listening on %s", addr)
|
||
errCh <- srv.ListenAndServe()
|
||
}()
|
||
select {
|
||
case <-ctx.Done():
|
||
shutCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||
defer cancel()
|
||
_ = srv.Shutdown(shutCtx)
|
||
return nil
|
||
case err := <-errCh:
|
||
if err == http.ErrServerClosed {
|
||
return nil
|
||
}
|
||
return err
|
||
}
|
||
}
|
||
|
||
// roleRunGateway（W2 实装，design §2）：MCP /mcp + /v1/search /v1/read + admin 面
|
||
// + 搜索短缓存 + policy 合规预检 + scheduler 入队/等待。
|
||
//
|
||
// env（密钥走 MustEnv fail-closed，非密钥可默认）：
|
||
// - BROWSER_SERVER_KEYS_SEED 必填（MustEnv，key 哈希全局盐）
|
||
// - BROWSER_SERVER_ADMIN_TOKEN 必填（MustEnv，admin 面静态 token）
|
||
// - BROWSER_SERVER_DB_PATH 默认 /data/browser-server.db（gateway 与 scheduler
|
||
// 共库：gateway 只读 consumers/api_keys + 写 audit/quota）
|
||
// - BROWSER_SERVER_SCHEDULER_URL 默认 http://scheduler:8641
|
||
// - BROWSER_SERVER_LISTEN 默认 :8640（listenAddr 已按角色处理）
|
||
func roleRunGateway(ctx context.Context, addr string, logger *log.Logger) error {
|
||
adminToken := config.MustEnv("BROWSER_SERVER_ADMIN_TOKEN")
|
||
seedSalt := config.MustEnv("BROWSER_SERVER_KEYS_SEED")
|
||
|
||
dbPath := config.EnvDefault("BROWSER_SERVER_DB_PATH", "/data/browser-server.db")
|
||
st, err := store.Open(dbPath)
|
||
if err != nil {
|
||
return fmt.Errorf("gateway: SQLite 打开失败: %w", err)
|
||
}
|
||
defer st.Close()
|
||
if err := st.Migrate(); err != nil {
|
||
return fmt.Errorf("gateway: migration: %w", err)
|
||
}
|
||
|
||
verifier := auth.NewVerifier(st, seedSalt)
|
||
// policy 引擎：robots 拉取走 httpx 共享 client（守卫挂接见 httpx.RedirectCheck）。
|
||
robotsCli := &http.Client{Timeout: 10 * time.Second}
|
||
pol := policy.NewEngine(st, robotsCli)
|
||
if n, err := pol.Reload(); err != nil {
|
||
logger.Printf("警告：域名规则加载失败（fail-closed 生效）: %v", err)
|
||
} else {
|
||
logger.Printf("域名规则已加载 %d 条", n)
|
||
}
|
||
|
||
deps := gateway.Deps{
|
||
DB: st,
|
||
Glue: gateway.NewDBGlue(st),
|
||
Verifier: verifier,
|
||
Policy: pol,
|
||
Scheduler: gateway.NewSchedulerClient(config.EnvDefault("BROWSER_SERVER_SCHEDULER_URL", "http://scheduler:8641")),
|
||
Cache: gateway.NewSearchCache(),
|
||
Limiter: gateway.NewRateLimiter(),
|
||
AdminToken: adminToken,
|
||
SeedSalt: seedSalt,
|
||
Logger: logger,
|
||
}
|
||
logger.Printf("gateway listening on %s", addr)
|
||
return serveHealth(ctx, addr, logger, gateway.NewServer(deps).Handler())
|
||
}
|
||
|
||
// roleRunScheduler W3 实现：env → store open → 适配器注册表 init（并发探活）→
|
||
// worker 池 + reaper + proxy 探活 → HTTP mux（:8641）。
|
||
// env（均非密钥，可默认）：
|
||
//
|
||
// BROWSER_SERVER_DB_PATH（默认 /data/browser-server.db；测试可给临时路径）
|
||
// BROWSER_SERVER_ADMIT_MAX（默认 60）、BROWSER_SERVER_WORKERS（默认 8）
|
||
// BROWSER_SERVER_SEARXNG_CN_URL / _GLOBAL_URL / TRAFILATURA_URL
|
||
// BROWSER_SERVER_LIGHTPANDA_CDP_HOST / SHELL_CDP_HOST、PROXYMANAGER_URL
|
||
func roleRunScheduler(ctx context.Context, addr string, logger *log.Logger) error {
|
||
dbPath := config.EnvDefault("BROWSER_SERVER_DB_PATH", "/data/browser-server.db")
|
||
// 环境探查：目录不可建时落内存库（healthz 可起，入队 503——fail-closed）。
|
||
if err := os.MkdirAll(filepath.Dir(dbPath), 0o755); err != nil {
|
||
logger.Printf("警告：DB 目录 %s 不可建（%v），退内存库（重启即失）", filepath.Dir(dbPath), err)
|
||
dbPath = ":memory:"
|
||
}
|
||
st, err := store.Open(dbPath)
|
||
if err != nil {
|
||
return fmt.Errorf("scheduler: SQLite 打开失败: %w", err)
|
||
}
|
||
defer st.Close()
|
||
if err := st.Migrate(); err != nil {
|
||
return fmt.Errorf("scheduler: migration: %w", err)
|
||
}
|
||
|
||
// 五适配器（design §4.4 路由表；URL env 可覆盖）。
|
||
reg := dock.NewRegistry()
|
||
reg.Register(dock.NewSearx("searxng-cn",
|
||
config.EnvDefault("BROWSER_SERVER_SEARXNG_CN_URL", "http://searxng-cn:8080"), "zh-CN"))
|
||
reg.Register(dock.NewSearx("searxng-global",
|
||
config.EnvDefault("BROWSER_SERVER_SEARXNG_GLOBAL_URL", "http://searxng-global:8080"), "en"))
|
||
reg.Register(dock.NewTrafilatura(
|
||
config.EnvDefault("BROWSER_SERVER_TRAFILATURA_URL", "http://trafilatura:8080")))
|
||
reg.Register(dock.NewLightpanda(config.EnvDefault("BROWSER_SERVER_LIGHTPANDA_CDP_HOST", "lightpanda:9222")))
|
||
reg.Register(dock.NewHeadlessShell(config.EnvDefault("BROWSER_SERVER_SHELL_CDP_HOST", "headless-shell:9222")))
|
||
initErrs := reg.InitAll(ctx)
|
||
for name, e := range initErrs {
|
||
logger.Printf("适配器 %s init 错误（Health 摘除走降级）: %v", name, e)
|
||
}
|
||
|
||
tmpl, err := scheduler.NewTemplate()
|
||
if err != nil {
|
||
return fmt.Errorf("scheduler: 模版层构造: %w", err)
|
||
}
|
||
workers := config.EnvDefaultInt("BROWSER_SERVER_WORKERS", 8)
|
||
admitMax := config.EnvDefaultInt("BROWSER_SERVER_ADMIT_MAX", 60)
|
||
core := scheduler.NewCore(st, reg, tmpl,
|
||
config.EnvDefault("BROWSER_SERVER_PROXYMANAGER_URL", "http://proxymanager:8642"),
|
||
workers, admitMax, logger)
|
||
jar, jarErr := session.Open(ctx, st, fingerprint.NodeID())
|
||
if jarErr != nil {
|
||
logger.Printf("警告：会话罐初始化失败（本轮不养 Cookie）: %v", jarErr)
|
||
} else {
|
||
core.SetJar(jar)
|
||
p := jar.Active()
|
||
logger.Printf("指纹模版 active=%s impersonate=%s", p.ID, p.Impersonate)
|
||
}
|
||
core.Start(ctx)
|
||
defer core.Stop()
|
||
|
||
logger.Printf("scheduler listening on %s (workers=%d admit_max=%d)", addr, workers, admitMax)
|
||
return serveHealth(ctx, addr, logger, core.Handler())
|
||
}
|
||
|
||
// roleRunProxyManager（W4 实装，design §5）：订阅解析（lib.py Go 移植）→
|
||
// mihomo provider 生成/热载 → 探活状态机 → P2C/sticky → /api/proxies /api/exit。
|
||
//
|
||
// env（非密钥走 EnvDefault，密钥走 MustEnv fail-closed）：
|
||
// - PROXY_SUB_URLS 必填（MustEnv，逗号分隔多订阅容灾；URL 永不入日志）
|
||
// - MIHOMO_CONTROLLER_URL 默认 http://mihomo:19090
|
||
// - BROWSER_SERVER_MIHOMO_SECRET 可选（mihomo controller 自身 bearer，分轨说明见 controller.go）
|
||
// - BROWSER_SERVER_MIHOMO_CONFIG_DIR 默认 /data/mihomo（目录 0700/文件 0600）
|
||
// - BROWSER_SERVER_DB_PATH 可选（SQLite rules 表共享；缺省不挂路由表）
|
||
func roleRunProxyManager(ctx context.Context, addr string, logger *log.Logger) error {
|
||
subURLsRaw := config.MustEnv("PROXY_SUB_URLS")
|
||
var subURLs []string
|
||
for _, u := range strings.Split(subURLsRaw, ",") {
|
||
if u = strings.TrimSpace(u); u != "" {
|
||
subURLs = append(subURLs, u)
|
||
}
|
||
}
|
||
if len(subURLs) == 0 {
|
||
return fmt.Errorf("PROXY_SUB_URLS 为空（fail-closed）")
|
||
}
|
||
ctrlURL := config.EnvDefault("MIHOMO_CONTROLLER_URL", "http://mihomo:19090")
|
||
// mihomo controller secret：属 mihomo 自身 REST 认证（下游契约允许 Bearer），
|
||
// 非空则启用；缺省不认证（controller 仅 overlay 可达）。
|
||
ctrlSecret := os.Getenv("BROWSER_SERVER_MIHOMO_SECRET")
|
||
cfgDir := config.EnvDefault("BROWSER_SERVER_MIHOMO_CONFIG_DIR", "/data/mihomo")
|
||
|
||
var db *store.DB
|
||
if dbPath := os.Getenv("BROWSER_SERVER_DB_PATH"); dbPath != "" {
|
||
d, err := store.Open(dbPath)
|
||
if err != nil {
|
||
return fmt.Errorf("proxymanager: 打开 SQLite 失败: %w", err)
|
||
}
|
||
defer d.Close()
|
||
if err := d.Migrate(); err != nil {
|
||
return fmt.Errorf("proxymanager: migration 失败: %w", err)
|
||
}
|
||
db = d
|
||
}
|
||
|
||
mgr := proxymanager.NewManager(db, cfgDir, subURLs, ctrlURL, ctrlSecret, logger)
|
||
go mgr.Run(ctx)
|
||
|
||
mux := http.NewServeMux()
|
||
mgr.Routes(mux)
|
||
return serveHealth(ctx, addr, logger, mux)
|
||
}
|
||
|
||
// healthzProbe 探测本服务 /healthz（Swarm healthcheck 用；distroless 无 curl）。
|
||
// 返回进程退出码：0 = 健康；1 = 不健康（未监听/非 200）。
|
||
func healthzProbe(addr string) int {
|
||
host := addr
|
||
if strings.HasPrefix(host, ":") {
|
||
host = "127.0.0.1" + host
|
||
}
|
||
c := &http.Client{Timeout: 2 * time.Second}
|
||
resp, err := c.Get("http://" + host + "/healthz")
|
||
if err != nil {
|
||
return 1
|
||
}
|
||
defer resp.Body.Close()
|
||
if resp.StatusCode != http.StatusOK {
|
||
return 1
|
||
}
|
||
return 0
|
||
}
|