onesvm-browser-server/bench/trafilatura-http/run.sh

#!/usr/bin/env bash
# phases: build | up | wait | inspect | idle | t2 | t5 | down | summarize
set -euo pipefail
unset HTTP_PROXY HTTPS_PROXY ALL_PROXY http_proxy https_proxy all_proxy NO_PROXY no_proxy

ROOT="$(cd "$(dirname "$0")" && pwd)"
cd "$ROOT"
NAME="bench-s3b-trafilatura"
COMPOSE=(docker compose -p bench-s3b-trafilatura -f "$ROOT/compose.yml")
RAW="$ROOT/raw"
SAMPLES="$ROOT/samples"
mkdir -p "$RAW/bodies" "$SAMPLES"
LOG="$RAW/commands.log"
phase="${1:-}"
FALLBACK="https://www.gov.cn/zhengce/2021-03/13/content_5592681.htm"

log() {
  local ts
  ts=$(date -u +%Y-%m-%dT%H:%M:%SZ)
  echo "[$ts] $*" | tee -a "$LOG"
}

wait_ready() {
  local i code
  for i in $(seq 1 60); do
    set +e
    code=$(curl -sS -o /dev/null -w '%{http_code}' --connect-timeout 2 --max-time 3 -A 'onesvm-bench/s3b' "http://127.0.0.1:18881/health" || true)
    set -e
    if [ "$code" = "200" ]; then
      log "ready /health=$code after ${i}s"
      return 0
    fi
    sleep 1
  done
  log "ERROR: trafilatura not ready"
  docker logs "$NAME" 2>&1 | tail -80 | tee -a "$LOG" || true
  return 1
}

inspect_image() {
  python3 - <<PY
import json, subprocess, pathlib
raw = pathlib.Path("$RAW")
meta = {}
mp = raw / "meta.json"
if mp.exists():
    meta = json.loads(mp.read_text())
line = subprocess.check_output(["docker","inspect","--format","{{.Name}} {{.Image}} {{.Config.Image}}","$NAME"], text=True).strip()
arch = subprocess.check_output(["docker","image","inspect","--format","{{.Architecture}}","bench-s3b-trafilatura:local"], text=True).strip()
base_arch = subprocess.check_output(["docker","image","inspect","--format","{{.Architecture}}","docker.io/library/python:3.12-slim-bookworm"], text=True).strip()
base_dig = subprocess.check_output(["docker","image","inspect","--format","{{json .RepoDigests}}","docker.io/library/python:3.12-slim-bookworm"], text=True).strip()
base_id = subprocess.check_output(["docker","image","inspect","--format","{{.Id}}","docker.io/library/python:3.12-slim-bookworm"], text=True).strip()
local_id = subprocess.check_output(["docker","image","inspect","--format","{{.Id}}","bench-s3b-trafilatura:local"], text=True).strip()
meta["image"] = {
    "ref": "bench-s3b-trafilatura:local",
    "digest": local_id,
    "base": "docker.io/library/python:3.12-slim-bookworm",
    "base_digest": (json.loads(base_dig) or [base_id])[0],
    "base_id": base_id,
    "architecture": arch,
    "base_architecture": base_arch,
    "trafilatura": "2.2.0",
    "inspect_line": line,
}
mp.write_text(json.dumps(meta, ensure_ascii=False, indent=2))
print(json.dumps(meta["image"], indent=2))
PY
}

case "$phase" in
  build)
    log "compose build (no cache pip layer if needed)"
    "${COMPOSE[@]}" build
    ;;
  up)
    log "compose up $NAME"
    "${COMPOSE[@]}" up -d --build
    wait_ready
    inspect_image
    ;;
  wait)
    wait_ready
    ;;
  inspect)
    inspect_image
    ;;
  idle)
    log "idle sample 5min interval=5 container=$NAME"
    STATS_DIR="$RAW" bash "$ROOT/lib/sample_stats.sh" trafilatura-http-idle 5 300 "$NAME"
    log "idle done lines=$(wc -l < "$RAW/trafilatura-http-idle.tsv")"
    ;;
  t2)
    T2_URL="${T2_URL:-}"
    T2_CJK="${T2_TITLE_CJK4:-}"
    if [ -z "$T2_URL" ] && [ -f "$ROOT/../searxng-cn/raw/meta.json" ]; then
      T2_URL=$(python3 -c "import json; d=json.load(open('$ROOT/../searxng-cn/raw/meta.json')); print((d.get('t2_pick') or {}).get('url') or '')")
      T2_CJK=$(python3 -c "import json; d=json.load(open('$ROOT/../searxng-cn/raw/meta.json')); print((d.get('t2_pick') or {}).get('title_cjk4') or '')")
      T2_SRC=$(python3 -c "import json; d=json.load(open('$ROOT/../searxng-cn/raw/meta.json')); print((d.get('t2_pick') or {}).get('source') or '')")
    fi
    if [ -z "${T2_URL:-}" ]; then
      T2_URL="$FALLBACK"
      T2_CJK=""
      T2_SRC="fallback"
    fi
    export T2_URL
    log "T2 x5 url=$T2_URL src=${T2_SRC:-} cjk4=$T2_CJK"
    : > "$RAW/t2.jsonl"
    STATS_DIR="$RAW" bash "$ROOT/lib/sample_stats.sh" trafilatura-http-session 1 0 "$NAME" &
    spid=$!
    sleep 10
    for i in 1 2 3 4 5; do
      log "T2 request $i"
      T2_URL="$T2_URL" BODY_OUT="$RAW/bodies/t2-${i}.json" RAW_DIR="$RAW" bash "$ROOT/request.sh" t2 "$i" > "$RAW/t2-${i}.metrics.json"
      verdict=$(python3 "$ROOT/eval_t2.py" "$RAW/bodies/t2-${i}.json" "$T2_URL" "$T2_CJK")
      python3 - "$RAW/t2-${i}.metrics.json" "$verdict" "$RAW/t2.jsonl" <<'PY'
import json, sys
m = json.loads(open(sys.argv[1], encoding="utf-8").read())
v = json.loads(sys.argv[2])
m.update(v)
open(sys.argv[3], "a", encoding="utf-8").write(json.dumps(m, ensure_ascii=False) + "\n")
print(json.dumps({"idx": m.get("idx"), "http": m.get("http_code"), "assert_ok": m.get("assert_ok"), "chars": m.get("char_count"), "title": m.get("title"), "t_total": m.get("t_total"), "note": m.get("note")}, ensure_ascii=False))
PY
      if [ "$i" -eq 1 ]; then
        sleep 10
        kill "$spid" 2>/dev/null || true
        wait "$spid" 2>/dev/null || true
      fi
      if [ "$i" -lt 5 ]; then
        sleep 1
      fi
    done
    kill "$spid" 2>/dev/null || true
    wait "$spid" 2>/dev/null || true
    python3 - "$RAW/meta.json" "$T2_URL" "${T2_SRC:-}" "$T2_CJK" <<'PY'
import json, sys, pathlib
p = pathlib.Path(sys.argv[1])
meta = json.loads(p.read_text()) if p.exists() else {}
meta["t2_url"] = sys.argv[2]
meta["t2_source"] = sys.argv[3]
meta["t2_title_cjk4"] = sys.argv[4]
p.write_text(json.dumps(meta, ensure_ascii=False, indent=2))
PY
    python3 - <<PY
import json, pathlib
raw = pathlib.Path("$RAW/bodies")
samples = pathlib.Path("$SAMPLES")
for i in (1,2,3):
    src = raw / f"t2-{i}.json"
    if not src.exists():
        continue
    data = json.loads(src.read_text(encoding="utf-8"))
    md = data.get("markdown") or ""
    slim = {k: data.get(k) for k in ("ok", "title", "char_count", "truncated", "url", "error", "fail_class")}
    slim["markdown_head"] = md[:1800]
    slim["markdown_tail"] = md[-400:] if len(md) > 1800 else ""
    (samples / f"t2-{i}.excerpt.json").write_text(json.dumps(slim, ensure_ascii=False, indent=2), encoding="utf-8")
print("samples written")
PY
    log "T2 done"
    ;;
  t5)
    T2_URL="${T2_URL:-}"
    T2_CJK="${T2_TITLE_CJK4:-}"
    if [ -z "$T2_URL" ] && [ -f "$RAW/meta.json" ]; then
      T2_URL=$(python3 -c "import json; d=json.load(open('$RAW/meta.json')); print(d.get('t2_url') or '')")
      T2_CJK=$(python3 -c "import json; d=json.load(open('$RAW/meta.json')); print(d.get('t2_title_cjk4') or '')")
    fi
    if [ -z "${T2_URL:-}" ]; then
      T2_URL="$FALLBACK"
    fi
    export T2_URL
    log "T5 burst 60 tasks worker=8 url=$T2_URL"
    : > "$RAW/t5.jsonl"
    STATS_DIR="$RAW" bash "$ROOT/lib/sample_stats.sh" trafilatura-http-t5 1 0 "$NAME" &
    spid=$!
    t5start=$(date +%s)
    seq 1 60 | xargs -P 8 -I{} bash -c '
      unset HTTP_PROXY HTTPS_PROXY ALL_PROXY http_proxy https_proxy all_proxy NO_PROXY no_proxy
      ROOT="'"$ROOT"'"
      RAW="'"$RAW"'"
      i="$1"
      T2_URL="'"$T2_URL"'" BODY_OUT="$RAW/bodies/t5-${i}.json" RAW_DIR="$RAW" bash "$ROOT/request.sh" t5-once "$i" > "$RAW/t5-${i}.metrics.json"
    ' _ {}
    t5end=$(date +%s)
    log "T5 curls finished wall=$((t5end-t5start))s; cooling 15s"
    sleep 15
    kill "$spid" 2>/dev/null || true
    wait "$spid" 2>/dev/null || true
    python3 - <<PY
import json, pathlib, subprocess
root = pathlib.Path("$ROOT")
raw = pathlib.Path("$RAW")
url = "$T2_URL"
cjk = "$T2_CJK"
out = raw / "t5.jsonl"
lines = []
for i in range(1, 61):
    mp = raw / f"t5-{i}.metrics.json"
    bp = raw / "bodies" / f"t5-{i}.json"
    if not mp.exists():
        continue
    m = json.loads(mp.read_text(encoding="utf-8"))
    if bp.exists():
        v = json.loads(subprocess.check_output(["python3", str(root/"eval_t2.py"), str(bp), url, cjk], text=True))
        m.update(v)
    else:
        m["assert_ok"] = False
        m["fail_class"] = "timeout"
        m["note"] = "no_body"
    lines.append(m)
out.write_text("".join(json.dumps(x, ensure_ascii=False)+"\n" for x in lines), encoding="utf-8")
ok = sum(1 for x in lines if x.get("assert_ok"))
print(f"t5 n={len(lines)} ok={ok}")
PY
    oom=$(docker inspect --format '{{.State.OOMKilled}} {{.State.Status}}' "$NAME" 2>/dev/null || echo "unknown unknown")
    log "T5 oom/status=$oom"
    python3 - "$RAW/meta.json" "$oom" <<'PY'
import json, sys, pathlib
p = pathlib.Path(sys.argv[1])
meta = json.loads(p.read_text()) if p.exists() else {}
parts = sys.argv[2].split()
meta["oom"] = parts[0].lower() == "true"
meta["container_status_after_t5"] = sys.argv[2]
p.write_text(json.dumps(meta, ensure_ascii=False, indent=2))
PY
    ;;
  down)
    log "compose down"
    "${COMPOSE[@]}" down --remove-orphans || true
    docker rm -f "$NAME" 2>/dev/null || true
    ;;
  summarize)
    python3 "$ROOT/summarize.py" "$ROOT"
    ;;
  *)
    echo "usage: $0 build|up|wait|inspect|idle|t2|t5|down|summarize" >&2
    exit 2
    ;;
esac