onesvm-browser-server/bench/site-matrix/run.sh

#!/usr/bin/env bash
# PROBE-1 site-matrix: trafilatura-http vs lightpanda on 20 sites.
# Writes only under this dir. No subscription plaintext on disk.
set -euo pipefail

ROOT="$(cd "$(dirname "$0")" && pwd)"
REPO="$(cd "$ROOT/../.." && pwd)"
RAW="$ROOT/raw"
PROXY_UP="$REPO/bench/proxy/up.sh"
PROXY_DOWN="$REPO/bench/proxy/down.sh"
SUB_MD="$REPO/.dsh/artifacts/proxy-subscriptions.local.md"
IMG_TRAF="bench-s3b-trafilatura:local"
CDP_PROXY="http://127.0.0.1:19223"
CDP_DIRECT="http://127.0.0.1:19224"
TIMEOUT_S=20

mkdir -p "$RAW"
: > "$RAW/run.log"

log() {
  local ts
  ts="$(date '+%Y-%m-%dT%H:%M:%S%z')"
  printf '[%s] %s\n' "$ts" "$*" | tee -a "$RAW/run.log"
}

cleanup() {
  local rc=$?
  trap - EXIT INT TERM
  set +e
  log "cleanup start rc=$rc"
  docker rm -f bench-probe-trafilatura-proxy bench-probe-trafilatura-direct \
    bench-probe-lightpanda-proxy bench-probe-lightpanda-direct >/dev/null 2>&1
  if [[ -x "$PROXY_DOWN" ]]; then
    bash "$PROXY_DOWN" >>"$RAW/run.log" 2>&1 || true
  fi
  docker rm -f $(docker ps -aq --filter 'name=^bench-probe-' 2>/dev/null) >/dev/null 2>&1
  docker rm -f $(docker ps -aq --filter 'name=^bench-proxy-' 2>/dev/null) >/dev/null 2>&1
  log "cleanup done leftover_probe=$(docker ps -a --filter 'name=^bench-probe-' --format '{{.Names}}' | tr '\n' ' ')"
  log "cleanup done leftover_proxy=$(docker ps -a --filter 'name=^bench-proxy-' --format '{{.Names}}' | tr '\n' ' ')"
  exit "$rc"
}
trap cleanup EXIT INT TERM

wait_http() {
  local url="$1" tries="${2:-40}"
  local i
  for i in $(seq 1 "$tries"); do
    if curl -sf --max-time 2 "$url" >/dev/null; then
      return 0
    fi
    sleep 1
  done
  return 1
}

count_probe() {
  docker ps -q --filter 'name=^bench-probe-' | wc -l | tr -d ' '
}

assert_probe_cap() {
  local n
  n="$(count_probe)"
  if [[ "$n" -gt 3 ]]; then
    log "ERROR: bench-probe-* count=$n exceeds 3"
    exit 1
  fi
}

if [[ -z "${PROXY_SUB_URL:-}" ]]; then
  PROXY_SUB_URL="$(python3 - "$SUB_MD" <<'PY'
import re, sys
from pathlib import Path
t = Path(sys.argv[1]).read_text(encoding="utf-8")
m = re.search(r"https://[^\s|]+", t)
if not m:
    raise SystemExit("PROXY_SUB_URL missing in subscription file")
print(m.group(0).strip())
PY
)"
  export PROXY_SUB_URL
fi
if [[ -z "${PROXY_SUB_URL:-}" ]]; then
  echo "error: PROXY_SUB_URL empty" >&2
  exit 2
fi
log "PROXY_SUB_URL loaded from env (value not logged)"

log "proxy up"
bash "$PROXY_UP" >>"$RAW/run.log" 2>&1
PROXY_CODE="$(curl -sS -o /dev/null -w '%{http_code}' --connect-timeout 10 --max-time 20 \
  -x http://127.0.0.1:17890 -A onesvm-bench/probe-1 \
  https://www.google.com/generate_204 || true)"
log "proxy generate_204 http=$PROXY_CODE"
if [[ "$PROXY_CODE" != "204" && "$PROXY_CODE" != "200" ]]; then
  log "WARN: proxy probe unexpected HTTP $PROXY_CODE; continue"
fi

# --- foreign: proxy channels ---
log "up lightpanda-proxy"
docker rm -f bench-probe-lightpanda-proxy >/dev/null 2>&1 || true
(cd "$ROOT" && docker compose -p bench-probe-lp-proxy -f "$ROOT/compose-lightpanda-proxy.yml" up -d --force-recreate) >>"$RAW/run.log" 2>&1
assert_probe_cap
if ! wait_http "$CDP_PROXY/json/version" 40; then
  log "WARN: lightpanda-proxy json/version not ready; try /json/list"
  wait_http "$CDP_PROXY/json/list" 15 || true
  docker logs --tail 40 bench-probe-lightpanda-proxy >>"$RAW/run.log" 2>&1 || true
fi
curl -sS --max-time 5 "$CDP_PROXY/json/version" >"$RAW/cdp-version-proxy.json" || true

log "run trafilatura-proxy (foreign HTTP batch)"
docker rm -f bench-probe-trafilatura-proxy >/dev/null 2>&1 || true
docker run -d --name bench-probe-trafilatura-proxy --platform linux/arm64 \
  --add-host=host.docker.internal:host-gateway \
  -e PYTHONUNBUFFERED=1 \
  -e PYTHONPATH=/probe \
  -e HTTP_PROXY=http://host.docker.internal:17890 \
  -e HTTPS_PROXY=http://host.docker.internal:17890 \
  -e http_proxy=http://host.docker.internal:17890 \
  -e https_proxy=http://host.docker.internal:17890 \
  -e NO_PROXY=127.0.0.1,localhost \
  -e no_proxy=127.0.0.1,localhost \
  -v "$ROOT:/probe:ro" \
  -v "$RAW:/probe-raw" \
  --memory=256m --restart=no \
  "$IMG_TRAF" \
  python -u /probe/http_probe.py --group foreign --targets /probe/targets.json \
    --out /probe-raw --proxy http://host.docker.internal:17890 >>"$RAW/run.log"
assert_probe_cap
# wait one-shot container
if ! docker wait bench-probe-trafilatura-proxy >>"$RAW/run.log"; then
  log "WARN: trafilatura-proxy wait failed"
fi
docker logs bench-probe-trafilatura-proxy >>"$RAW/http-foreign.container.log" 2>&1 || true
docker rm -f bench-probe-trafilatura-proxy >/dev/null 2>&1 || true
log "trafilatura-proxy done"

log "run lightpanda-proxy (foreign CDP batch)"
python3 "$ROOT/lp_probe.py" \
  --targets "$ROOT/targets.json" \
  --group foreign \
  --cdp "$CDP_PROXY" \
  --cdp-fetch "$ROOT/cdp_fetch.mjs" \
  --resolved "$RAW/resolved-foreign.json" \
  --timeout "$TIMEOUT_S" \
  --out "$RAW" | tee -a "$RAW/run.log"
docker rm -f bench-probe-lightpanda-proxy >/dev/null 2>&1 || true
(cd "$ROOT" && docker compose -p bench-probe-lp-proxy -f "$ROOT/compose-lightpanda-proxy.yml" down --remove-orphans) >>"$RAW/run.log" 2>&1 || true
log "foreign phase done probe_count=$(count_probe)"

# --- domestic: direct channels ---
log "up lightpanda-direct"
docker rm -f bench-probe-lightpanda-direct >/dev/null 2>&1 || true
(cd "$ROOT" && docker compose -p bench-probe-lp-direct -f "$ROOT/compose-lightpanda-direct.yml" up -d --force-recreate) >>"$RAW/run.log" 2>&1
assert_probe_cap
if ! wait_http "$CDP_DIRECT/json/version" 40; then
  log "WARN: lightpanda-direct json/version not ready"
  wait_http "$CDP_DIRECT/json/list" 15 || true
  docker logs --tail 40 bench-probe-lightpanda-direct >>"$RAW/run.log" 2>&1 || true
fi
curl -sS --max-time 5 "$CDP_DIRECT/json/version" >"$RAW/cdp-version-direct.json" || true

log "run trafilatura-direct (domestic HTTP batch)"
docker rm -f bench-probe-trafilatura-direct >/dev/null 2>&1 || true
docker run -d --name bench-probe-trafilatura-direct --platform linux/arm64 \
  --add-host=host.docker.internal:host-gateway \
  -e PYTHONUNBUFFERED=1 \
  -e PYTHONPATH=/probe \
  -e NO_PROXY='*' \
  -e no_proxy='*' \
  -v "$ROOT:/probe:ro" \
  -v "$RAW:/probe-raw" \
  --memory=256m --restart=no \
  "$IMG_TRAF" \
  python -u /probe/http_probe.py --group domestic --targets /probe/targets.json \
    --out /probe-raw --proxy "" >>"$RAW/run.log"
assert_probe_cap
if ! docker wait bench-probe-trafilatura-direct >>"$RAW/run.log"; then
  log "WARN: trafilatura-direct wait failed"
fi
docker logs bench-probe-trafilatura-direct >>"$RAW/http-domestic.container.log" 2>&1 || true
docker rm -f bench-probe-trafilatura-direct >/dev/null 2>&1 || true
log "trafilatura-direct done"

log "run lightpanda-direct (domestic CDP batch)"
python3 "$ROOT/lp_probe.py" \
  --targets "$ROOT/targets.json" \
  --group domestic \
  --cdp "$CDP_DIRECT" \
  --cdp-fetch "$ROOT/cdp_fetch.mjs" \
  --resolved "$RAW/resolved-domestic.json" \
  --timeout "$TIMEOUT_S" \
  --out "$RAW" | tee -a "$RAW/run.log"
docker rm -f bench-probe-lightpanda-direct >/dev/null 2>&1 || true
(cd "$ROOT" && docker compose -p bench-probe-lp-direct -f "$ROOT/compose-lightpanda-direct.yml" down --remove-orphans) >>"$RAW/run.log" 2>&1 || true
log "domestic phase done"

log "summarize"
python3 "$ROOT/summarize.py" "$ROOT"
log "run complete"