// pipelinecore.go:请求管线核心(认证后 → 校验 → 限流 → 配额 → 合规 → 入队 → 等待)。 // MCP 与 HTTP 兜底共用本内核(design §2.4:一套重试策略通吃)。 package gateway import ( "context" "encoding/json" "errors" "net/http" "strings" "time" "onesvm.com/onesvm/browser-server/internal/contract" "onesvm.com/onesvm/browser-server/internal/policy" ) // Intent 常量别名(pipeline 侧引用)。 const ( IntentSearch = "search" IntentRead = "read" ) // result 管线结果:完整信封字节(200)或前置错误(HTTPStatus + Body2)。 type result struct { Body []byte // 200 完整信封 HTTPStatus int // 前置错误状态码 Body2 []byte // 前置错误响应体 RetryAfterS *int // Retry-After Headers map[string]string // 附加头(X-RateLimit-*) fromCache bool } // pipeline 单请求管线(runCtx 在 server.go 定义)。 func (s *Server) pipeline(rc runCtx) result { // 1. 入参校验 var err error if rc.Intent == IntentSearch { err = validateSearch(rc.Search) } else { err = validateRead(rc.Read) } if err != nil { return s.preErr(rc, http.StatusBadRequest, contract.CodeUnavailable, err.Error(), nil, nil) } // 2. scope 校验(search/read 默认;extract/screenshot/rawHtml 特权) if sc, ok := s.scopeDenied(rc); ok { s.auditDenied(rc, "scope_denied", "") return s.preErr(rc, http.StatusForbidden, contract.CodeDenied, "缺少特权 scope: "+sc, nil, nil) } // 3. 429 令牌桶(rpm) if ok, remaining, reset := s.deps.Limiter.Allow(rc.Auth.Key.ID, rc.Auth.Key.RPM); !ok { hdr := map[string]string{ "X-RateLimit-Limit": itoa(rc.Auth.Key.RPM), "X-RateLimit-Remaining": "0", "X-RateLimit-Reset": itoa(reset), } ra := reset return s.preErr(rc, http.StatusTooManyRequests, contract.CodeRateLimited, "超 rpm 限速(每分钟 "+itoa(rc.Auth.Key.RPM)+" 次)", &ra, hdr) } else if remaining >= 0 && rc.Auth.Key.RPM > 0 { _ = remaining // 成功路径不带头(Brave 惯例仅错误带) } // 4. 402 配额预扣(日窗;月窗字段保留由管理员签发时设 0=不限) if _, aerr := s.deps.Verifier.ReserveDaily(rc.Auth.Key.ID, rc.Auth.Key.DailyQuota, time.Now()); aerr != nil { return s.preErr(rc, http.StatusPaymentRequired, aerr.Code, aerr.Message, nil, nil) } // 5. 403 合规预检(SSRF + 域名 deny + robots) if d := s.policyCheck(rc); d != nil { s.auditDenied(rc, d.RuleID, policyURL(rc)) _ = s.deps.Verifier.Release(rc.Auth.Key.ID, time.Now()) // 拒绝不扣额度 return s.preErr(rc, http.StatusForbidden, contract.CodeDenied, d.Reason+"(rule_id="+d.RuleID+")", nil, nil) } // 6. 搜索短缓存(命中直接回,credits=0) if rc.Intent == IntentSearch { if body, ok := s.deps.Cache.Get(rc.Search); ok { return result{Body: body, fromCache: true} } } // 7. 在途会话计数(X-Session-Remaining) s.beginRequest(rc.Auth) defer s.finishRequest(rc.Auth) // 8. 入队(3s 超时 + 1 次重试;不可达 503+Retry-After 零落盘) env := s.buildJob(rc) enc, err := s.deps.Scheduler.Enqueue(ctxOf(rc.R), env) if err != nil { _ = s.deps.Verifier.Release(rc.Auth.Key.ID, time.Now()) return s.schedulerDown(rc, err) } if enc.JobID == 0 && enc.RequestID == "" { _ = s.deps.Verifier.Release(rc.Auth.Key.ID, time.Now()) return s.preErr(rc, http.StatusBadGateway, contract.CodeUpstream, "enqueue 响应缺 job 标识", nil, nil) } // 9. 同步等待结果(200ms 轮询,120s 硬顶) rawEnv, err := waitResult(ctxOf(rc.R), s.deps.Scheduler, enc.RequestID) if err != nil { // 等待失败(ITER-3 FIX-1):gateway 侧结算——预扣在本进程(步骤 4), // 终态结算也归本进程,同进程 Reserve→Settle/Release 闭环(不给 scheduler // 加第三写点)。超时任务若 scheduler 侧后续完成,该次用量不计(首版口径: // 超时不计费),waitFail 内释放预扣并记审计。 return s.waitFail(rc, err) } // 9bis 终态结算(ITER-3 FIX-1):信封 ok=true → Settle(reserved→used); // ok=false → Release(失败回收预扣)。scheduler 不碰 quota 表(单写者纪律)。 s.settleQuota(rc, isOKEnvelope(rawEnv)) // 10. 缓存写入(仅 search 成功且非空结果)。零结果不缓存:引擎抖一次会把空信封冻 300s。 if rc.Intent == IntentSearch && isOKEnvelope(rawEnv) && !isEmptySearchEnvelope(rawEnv) { s.deps.Cache.Put(rc.Search, rawEnv) } return result{Body: rawEnv} } // settleQuota 配额终态结算(ITER-3 FIX-1):成功 Settle / 失败 Release。 // 结算失败仅记日志不回滚业务响应(额度窗口自愈:次日过期回收)。 func (s *Server) settleQuota(rc runCtx, ok bool) { var qerr error if ok { qerr = s.deps.Verifier.Settle(rc.Auth.Key.ID, time.Now()) } else { qerr = s.deps.Verifier.Release(rc.Auth.Key.ID, time.Now()) } if qerr != nil { s.deps.Logger.Printf("quota_settle_failed key=%d ok=%v err=%v", rc.Auth.Key.ID, ok, qerr) } } // preErr 前置错误(信封形状 + HTTP 状态)。 func (s *Server) preErr(rc runCtx, status int, code, msg string, retryAfter *int, hdr map[string]string) result { env := contract.Envelope{ OK: false, Kind: rc.Intent, Error: &contract.ErrBody{Code: code, Message: msg, RetryAfterS: retryAfter}, } b, _ := json.Marshal(env) return result{HTTPStatus: status, Body2: b, RetryAfterS: retryAfter, Headers: hdr} } // schedulerDown scheduler 不可达(503 + Retry-After: 2)。 func (s *Server) schedulerDown(rc runCtx, err error) result { ae := &APIError{Code: contract.CodeUnavailable} if errors.As(err, &ae) || true { // 归一:任何 enqueue 失败都按 503 fail-closed _ = ae } ra := 2 return s.preErr(rc, http.StatusServiceUnavailable, contract.CodeUnavailable, "scheduler 不可达(排队失败,gateway 未落盘,稍后重试): "+errMsg(err), &ra, nil) } // waitFail 等待结果失败(120s 硬顶 / scheduler 半途不可达)。 // ITER-3 FIX-1:gateway 侧结算——120s 超时路径释放预扣并记审计「quota_unsettled_timeout」 // (超时任务若后续 scheduler 侧完成,该次用量不计——首版口径:超时不计费); // 其余等待中断(scheduler 不可达)同口径释放。gateway 返回 timeout/upstream 200 信封。 func (s *Server) waitFail(rc runCtx, err error) result { if errors.Is(err, errWaitTimeout) || errors.Is(err, context.DeadlineExceeded) { // 超时不计费:释放预扣 + 审计(rule_id=quota_unsettled_timeout,url 记 request 语义)。 if rerr := s.deps.Verifier.Release(rc.Auth.Key.ID, time.Now()); rerr != nil { s.deps.Logger.Printf("quota_unsettled_timeout release 失败 key=%d err=%v", rc.Auth.Key.ID, rerr) } s.auditDenied(rc, "quota_unsettled_timeout", policyURL(rc)) env := envelopeWithErr(rc, contract.CodeTimeout, "任务执行超时(120s 硬顶)", 30) b, _ := json.Marshal(env) return result{Body: b, HTTPStatus: http.StatusOK} } // 半途不可达:预扣同样回收(任务可能仍在 scheduler 侧执行,首版口径:gateway 未见终态不计费)。 if rerr := s.deps.Verifier.Release(rc.Auth.Key.ID, time.Now()); rerr != nil { s.deps.Logger.Printf("quota_unsettled_interrupt release 失败 key=%d err=%v", rc.Auth.Key.ID, rerr) } env := envelopeWithErr(rc, contract.CodeUpstream, "scheduler 等待中断: "+errMsg(err), 5) b, _ := json.Marshal(env) return result{Body: b, HTTPStatus: http.StatusOK} } // envelopeWithErr 构造 200 信封错误。 func envelopeWithErr(rc runCtx, code, msg string, retryAfter int) contract.Envelope { env := contract.Envelope{ OK: false, Kind: rc.Intent, Error: &contract.ErrBody{Code: code, Message: msg, RetryAfterS: &retryAfter}, } return env } // scopeDenied 特权 scope 检查;返回缺失的 scope 名。 func (s *Server) scopeDenied(rc runCtx) (string, bool) { switch rc.Intent { case IntentSearch: if !authHasScope(rc.Auth, "search") { return "search", true } case IntentRead: if !authHasScope(rc.Auth, "read") { return "read", true } for _, need := range privilegedRead(rc.Read) { if !authHasScope(rc.Auth, need) { return need, true } } } return "", false } // authHasScope 消费 auth.Error 形状无关的 scope 判定(镜像 auth.HasScope)。 func authHasScope(a *consumerAuth, scope string) bool { for _, s := range a.Key.Scopes { if strings.TrimSpace(s) == scope { return true } } return false } // policyCheck 合规预检:read 走 SSRF+域名+robots;search 只过域名(query 无 URL)。 // 返回 nil = 放行。 func (s *Server) policyCheck(rc runCtx) *policy.DeniedError { if rc.Intent == IntentRead { u := rc.Read.URL if d := s.deps.Policy.Authorize(ctxOf(rc.R), u); d != nil { return d } // robots(普通 key;特权覆盖首版未开放 scope,恒 false) if rc.Read.Extract == nil || !authHasScope(rc.Auth, "extract") { allowed, perr := s.deps.Policy.RobotsAllowed(ctxOf(rc.R), u, false) if perr == nil && !allowed { return &policy.DeniedError{RuleID: "robots_disallow", Reason: "robots.txt 禁止抓取该路径"} } // robots 拉取失败(perr!=nil)不阻塞:fail-open 仅限 robots 层(见 robots.go 注释) } } return nil } // auditDenied denied 审计落库(consumer_id, url, rule_id, ts+08:00)。 func (s *Server) auditDenied(rc runCtx, ruleID, url string) { cid := rc.Auth.Key.ConsumerID s.deps.Policy.Audit(cid, url, ruleID) } // policyURL 合规审计 URL 提取(search 无 URL 记 query 摘要)。 func policyURL(rc runCtx) string { if rc.Intent == IntentRead { return rc.Read.URL } q := "" if rc.Search != nil { q = rc.Search.Query } return "query:" + q } // buildJob 组装 JobEnvelopeExt(内部契约见回执 §4)。 func (s *Server) buildJob(rc runCtx) *contract.JobEnvelopeExt { reqID := newRequestID() now := contract.NowTime() env := contract.JobEnvelopeExt{ JobEnvelope: contract.JobEnvelope{ ID: reqID, RequestID: reqID, Intent: rc.Intent, KeyID: itoa64(rc.Auth.Key.ID), ConsumerID: itoa64(rc.Auth.Key.ConsumerID), Priority: 100, SubmittedAt: now, Status: contract.JobQueued, }, TimeoutS: jobTimeoutS(rc.Intent), } if rc.Intent == IntentSearch { e := *rc.Search env.Search = &e } else { e := *rc.Read env.Read = &e } return &env }