chii
eb972dfa93
单二进制三角色 + Dock 适配器 + Swarm stack 达到可部署态；mgr1 实测订阅经 central-proxy bootstrap，探活 alive=41/52。 Co-authored-by: Cursor <cursoragent@cursor.com>
261 lines
9 KiB
Go
261 lines
9 KiB
Go
// jobs.go：任务队列持久层。原子抢单用单条 UPDATE...WHERE id=(SELECT...RETURNING)
|
||
// 语义（SQLite 3.35+；design-arch §4.2），配合 WAL 与单写者连接。
|
||
package store
|
||
|
||
import (
|
||
"database/sql"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"time"
|
||
|
||
"onesvm.com/onesvm/browser-server/internal/config"
|
||
"onesvm.com/onesvm/browser-server/internal/contract"
|
||
)
|
||
|
||
// Job 队列任务行。
|
||
type Job struct {
|
||
ID int64
|
||
RequestID string
|
||
Intent string
|
||
Payload []byte // JobEnvelope JSON
|
||
Priority int
|
||
Status string
|
||
AvailableAt time.Time
|
||
LeaseUntil *time.Time
|
||
Attempts int
|
||
Worker string
|
||
Error string
|
||
CreatedAt time.Time
|
||
UpdatedAt time.Time
|
||
}
|
||
|
||
// EnqueueJob 插入任务（status=queued），落 WAL 才 ACK。
|
||
func (d *DB) EnqueueJob(reqID, intent string, payload []byte, priority int) (int64, error) {
|
||
now := config.Now().Format(time.RFC3339)
|
||
res, err := d.raw.Exec(
|
||
`INSERT INTO jobs(request_id, intent, payload, priority, status, available_at, created_at, updated_at)
|
||
VALUES(?, ?, ?, ?, 'queued', ?, ?, ?)`,
|
||
reqID, intent, string(payload), priority, now, now, now)
|
||
if err != nil {
|
||
return 0, fmt.Errorf("store: EnqueueJob: %w", err)
|
||
}
|
||
return res.LastInsertId()
|
||
}
|
||
|
||
// ClaimNext 原子抢单：挑一条可执行任务并置 running + 租约，事务内完成。
|
||
// 返回 ErrNotFound 表示队列空。worker 需在 leaseUntil 前续租或完成。
|
||
func (d *DB) ClaimNext(worker string, leaseFor time.Duration) (*Job, error) {
|
||
now := config.Now()
|
||
nowStr := now.Format(time.RFC3339)
|
||
leaseStr := now.Add(leaseFor).Format(time.RFC3339)
|
||
tx, err := d.raw.Begin()
|
||
if err != nil {
|
||
return nil, fmt.Errorf("store: ClaimNext begin: %w", err)
|
||
}
|
||
defer tx.Rollback() //nolint:errcheck // Commit 后幂等
|
||
var id int64
|
||
// 原子选单：最高优先级（数值小者优先）→ available_at 最早。
|
||
err = tx.QueryRow(
|
||
`SELECT id FROM jobs
|
||
WHERE status = 'queued' AND available_at <= ?
|
||
ORDER BY priority ASC, available_at ASC, id ASC LIMIT 1`,
|
||
nowStr).Scan(&id)
|
||
if errors.Is(err, sql.ErrNoRows) {
|
||
return nil, ErrNotFound
|
||
}
|
||
if err != nil {
|
||
return nil, fmt.Errorf("store: ClaimNext select: %w", err)
|
||
}
|
||
if _, err := tx.Exec(
|
||
`UPDATE jobs SET status='running', worker=?, lease_until=?, attempts=attempts+1, updated_at=?
|
||
WHERE id = ? AND status = 'queued'`, worker, leaseStr, nowStr, id); err != nil {
|
||
return nil, fmt.Errorf("store: ClaimNext update: %w", err)
|
||
}
|
||
job, err := scanJob(tx.QueryRow(
|
||
`SELECT id, request_id, intent, payload, priority, status, available_at,
|
||
lease_until, attempts, worker, error, created_at, updated_at FROM jobs WHERE id = ?`, id))
|
||
if err != nil {
|
||
return nil, fmt.Errorf("store: ClaimNext scan: %w", err)
|
||
}
|
||
if err := tx.Commit(); err != nil {
|
||
return nil, fmt.Errorf("store: ClaimNext commit: %w", err)
|
||
}
|
||
return job, nil
|
||
}
|
||
|
||
// RenewLease 续租。
|
||
func (d *DB) RenewLease(id int64, until time.Time) error {
|
||
_, err := d.raw.Exec(
|
||
`UPDATE jobs SET lease_until = ?, updated_at = ? WHERE id = ? AND status = 'running'`,
|
||
until.In(config.TZ).Format(time.RFC3339), config.Now().Format(time.RFC3339), id)
|
||
if err != nil {
|
||
return fmt.Errorf("store: RenewLease: %w", err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// FinishJob 任务完成（done/failed），error 仅瞬时错误带退避。
|
||
func (d *DB) FinishJob(id int64, status, errMsg string) error {
|
||
now := config.Now().Format(time.RFC3339)
|
||
_, err := d.raw.Exec(
|
||
`UPDATE jobs SET status = ?, error = ?, lease_until = NULL, updated_at = ? WHERE id = ?`,
|
||
status, errMsg, now, id)
|
||
if err != nil {
|
||
return fmt.Errorf("store: FinishJob: %w", err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// RetryWithBackoff 瞬时错误重试：attempts≤maxAttempts 时回队列并指数退避，
|
||
// 否则入死信表。返回是否进入重试。
|
||
func (d *DB) RetryWithBackoff(id int64, lastErr string, maxAttempts int, backoffBase time.Duration) (bool, error) {
|
||
var attempts int
|
||
var payload []byte
|
||
var reqID, intent string
|
||
err := d.raw.QueryRow(
|
||
`SELECT attempts, request_id, intent, payload FROM jobs WHERE id = ?`, id).
|
||
Scan(&attempts, &reqID, &intent, &payload)
|
||
if err != nil {
|
||
return false, fmt.Errorf("store: RetryWithBackoff scan: %w", err)
|
||
}
|
||
now := config.Now()
|
||
if attempts >= maxAttempts {
|
||
if derr := d.DeadLetter(id, reqID, intent, payload, attempts, lastErr); derr != nil {
|
||
return false, derr
|
||
}
|
||
if ferr := d.FinishJob(id, "dead", lastErr); ferr != nil {
|
||
return false, ferr
|
||
}
|
||
return false, nil
|
||
}
|
||
// 指数退避：base * 2^(attempts-1)。RFC3339 只有秒精度，不足 1s 的退避会因
|
||
// 同秒截断失效，故退避值向上取整到秒（+999ms 保证至少隔 1 秒可抢）。
|
||
delay := backoffBase * (1 << (attempts - 1))
|
||
avail := now.Add(delay).Truncate(time.Second).Add(time.Second).Format(time.RFC3339)
|
||
if _, err := d.raw.Exec(
|
||
`UPDATE jobs SET status='queued', available_at=?, lease_until=NULL, worker=NULL,
|
||
error=?, updated_at=? WHERE id=?`, avail, lastErr, now.Format(time.RFC3339), id); err != nil {
|
||
return false, fmt.Errorf("store: RetryWithBackoff update: %w", err)
|
||
}
|
||
return true, nil
|
||
}
|
||
|
||
// ReapExpired 收割租约过期任务：回队列或入死信（attempts 已尽）。返回处理数量。
|
||
func (d *DB) ReapExpired(maxAttempts int) (int, error) {
|
||
nowStr := config.Now().Format(time.RFC3339)
|
||
rows, err := d.raw.Query(
|
||
`SELECT id, request_id, intent, payload, attempts FROM jobs
|
||
WHERE status = 'running' AND lease_until IS NOT NULL AND lease_until < ?`, nowStr)
|
||
if err != nil {
|
||
return 0, fmt.Errorf("store: ReapExpired select: %w", err)
|
||
}
|
||
type reaped struct {
|
||
id int64
|
||
reqID, intent string
|
||
payload []byte
|
||
attempts int
|
||
}
|
||
var list []reaped
|
||
for rows.Next() {
|
||
var r reaped
|
||
if err := rows.Scan(&r.id, &r.reqID, &r.intent, &r.payload, &r.attempts); err != nil {
|
||
rows.Close()
|
||
return 0, fmt.Errorf("store: ReapExpired scan: %w", err)
|
||
}
|
||
list = append(list, r)
|
||
}
|
||
rows.Close()
|
||
if err := rows.Err(); err != nil {
|
||
return 0, fmt.Errorf("store: ReapExpired rows: %w", err)
|
||
}
|
||
for _, r := range list {
|
||
ok, err := d.RetryWithBackoff(r.id, "lease expired (reaped)", maxAttempts, 30*time.Second)
|
||
if err != nil {
|
||
return len(list), err
|
||
}
|
||
_ = ok
|
||
}
|
||
return len(list), nil
|
||
}
|
||
|
||
// DeadLetter 落死信表。
|
||
func (d *DB) DeadLetter(jobID int64, reqID, intent string, payload []byte, attempts int, lastErr string) error {
|
||
_, err := d.raw.Exec(
|
||
`INSERT INTO dead_letters(job_id, request_id, intent, payload, attempts, last_error, dead_at)
|
||
VALUES(?, ?, ?, ?, ?, ?, ?)`,
|
||
jobID, reqID, intent, string(payload), attempts, lastErr, config.Now().Format(time.RFC3339))
|
||
if err != nil {
|
||
return fmt.Errorf("store: DeadLetter: %w", err)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// CountStatus 统计 running/queued 数量（背压 /pressure 用）。
|
||
func (d *DB) CountRunning() (int, error) { return countByStatus(d, "running") }
|
||
|
||
// CountQueued 统计 queued 数量。
|
||
func (d *DB) CountQueued() (int, error) { return countByStatus(d, "queued") }
|
||
|
||
func countByStatus(d *DB, status string) (int, error) {
|
||
var n int
|
||
if err := d.raw.QueryRow(
|
||
`SELECT COUNT(id) FROM jobs WHERE status = ?`, status).Scan(&n); err != nil {
|
||
return 0, fmt.Errorf("store: CountByStatus(%s): %w", status, err)
|
||
}
|
||
return n, nil
|
||
}
|
||
|
||
// JobByID 查询单任务（gateway 转告消费者排位用）。
|
||
func (d *DB) JobByID(id int64) (*Job, error) {
|
||
job, err := scanJob(d.raw.QueryRow(
|
||
`SELECT id, request_id, intent, payload, priority, status, available_at,
|
||
lease_until, attempts, worker, error, created_at, updated_at FROM jobs WHERE id = ?`, id))
|
||
if errors.Is(err, sql.ErrNoRows) {
|
||
return nil, ErrNotFound
|
||
}
|
||
return job, err
|
||
}
|
||
|
||
// JobEnvelopeFromPayload 反序列化任务 payload 为契约 JobEnvelope。
|
||
func (d *DB) JobEnvelopeFromPayload(payload []byte) (contract.JobEnvelope, error) {
|
||
var env contract.JobEnvelope
|
||
if err := json.Unmarshal(payload, &env); err != nil {
|
||
return env, fmt.Errorf("store: payload 反序列化: %w", err)
|
||
}
|
||
return env, nil
|
||
}
|
||
|
||
func scanJob(row interface{ Scan(...any) error }) (*Job, error) {
|
||
var j Job
|
||
var payload, avail, created, updated string
|
||
var lease, worker, jobErr sql.NullString // lease_until 可空（回队/完成后置 NULL）
|
||
if err := row.Scan(&j.ID, &j.RequestID, &j.Intent, &payload, &j.Priority, &j.Status,
|
||
&avail, &lease, &j.Attempts, &worker, &jobErr, &created, &updated); err != nil {
|
||
return nil, err
|
||
}
|
||
j.Payload = []byte(payload)
|
||
j.Worker = worker.String
|
||
j.Error = jobErr.String
|
||
var err error
|
||
if j.AvailableAt, err = time.Parse(time.RFC3339, avail); err != nil {
|
||
return nil, fmt.Errorf("store: jobs.available_at 解析: %w", err)
|
||
}
|
||
if lease.Valid && lease.String != "" {
|
||
t, perr := time.Parse(time.RFC3339, lease.String)
|
||
if perr != nil {
|
||
return nil, fmt.Errorf("store: jobs.lease_until 解析: %w", perr)
|
||
}
|
||
j.LeaseUntil = &t
|
||
}
|
||
if j.CreatedAt, err = time.Parse(time.RFC3339, created); err != nil {
|
||
return nil, fmt.Errorf("store: jobs.created_at 解析: %w", err)
|
||
}
|
||
if j.UpdatedAt, err = time.Parse(time.RFC3339, updated); err != nil {
|
||
return nil, fmt.Errorf("store: jobs.updated_at 解析: %w", err)
|
||
}
|
||
return &j, nil
|
||
}
|
||
|
||
// payloadOf 供 Scan 目标取址。
|
||
func payloadOf(p *[]byte) any { return p }
|