onesvm-browser-server/bench/searxng-global/summarize.py

#!/usr/bin/env python3
"""Build searxng-global/results.json from raw artifacts. No secrets."""
import json, math, os, statistics
from pathlib import Path

ROOT = Path(__file__).resolve().parent
RAW = ROOT / "raw"


def percentile(xs, p):
    if not xs:
        return None
    s = sorted(xs)
    if len(s) == 1:
        return round(s[0], 4)
    k = (len(s) - 1) * p / 100.0
    f = math.floor(k)
    c = math.ceil(k)
    if f == c:
        return round(s[int(k)], 4)
    return round(s[f] + (s[c] - s[f]) * (k - f), 4)


def load_jsonl(p):
    rows = []
    if not p.exists():
        return rows
    for line in p.read_text(encoding="utf-8").splitlines():
        line = line.strip()
        if not line:
            continue
        rows.append(json.loads(line))
    return rows


def tsv_mibs(p, col="mem_mib"):
    vals = []
    if not p.exists():
        return vals
    lines = p.read_text(encoding="utf-8").splitlines()
    if len(lines) < 2:
        return vals
    hdr = lines[0].split("\t")
    try:
        i = hdr.index(col)
    except ValueError:
        i = 4 if col == "mem_mib" else 5
    for line in lines[1:]:
        parts = line.split("\t")
        if len(parts) <= i:
            continue
        try:
            v = float(parts[i])
        except ValueError:
            continue
        if v > 0:
            vals.append(v)
    return vals


def median(xs):
    if not xs:
        return None
    return round(statistics.median(xs), 3)


def pmax(xs):
    return round(max(xs), 3) if xs else None


def mem_score(idle, delta, burst):
    def band(v, cuts):
        if v is None:
            return None
        for score, lim in cuts:
            if v <= lim:
                return score
        return 0
    idle_s = band(idle, [(100, 60), (80, 100), (60, 200), (40, 400)])
    delta_s = band(delta, [(100, 30), (80, 80), (60, 200), (40, 350)])
    burst_s = band(burst, [(100, 200), (80, 400), (60, 700), (40, 1000)])
    parts = [x for x in (idle_s, delta_s, burst_s) if x is not None]
    if not parts:
        return None
    return round(sum(parts) / 3, 1)


def quality_search(t3_rows):
    if not t3_rows:
        return 0
    oks = [r for r in t3_rows if r.get("assert_ok")]
    if any(r.get("assert_ok") for r in t3_rows):
        # 100 if >=5 and >=2 engines
        best = max(t3_rows, key=lambda r: (r.get("assert_ok"), r.get("n_results", 0), len(r.get("engines") or [])))
        if best.get("assert_ok"):
            return 100
        if best.get("n_results", 0) >= 5 and len(best.get("engines") or []) == 1:
            return 70
        if best.get("n_results", 0) >= 1:
            return 40
    # maybe some results even if assert failed
    if any(r.get("n_results", 0) >= 5 for r in t3_rows):
        if any(len(r.get("engines") or []) >= 2 for r in t3_rows):
            return 100
        return 70
    if any(r.get("n_results", 0) >= 1 for r in t3_rows):
        return 40
    return 0


def stab_score(rate, oom):
    if oom:
        return 0
    if rate is None:
        return None
    if rate >= 0.99:
        return 100
    if rate >= 0.95:
        return 80
    if rate >= 0.85:
        return 60
    if rate >= 0.70:
        return 40
    return 0


def lat_score_t3(p95):
    if p95 is None:
        return None
    if p95 <= 2:
        return 100
    if p95 <= 5:
        return 70
    if p95 <= 12:
        return 40
    return 0


def lat_score_t5(t5_p95, t3_p95):
    if t5_p95 is None:
        return None
    if t3_p95 and t3_p95 > 0:
        ratio = t5_p95 / t3_p95
        if ratio <= 3:
            return 100
        if ratio <= 5:
            return 70
        if ratio <= 8:
            return 40
        return 0
    if t5_p95 <= 12:
        return 40
    return 0


def main():
    t3 = load_jsonl(RAW / "t3.jsonl")
    t3e = load_jsonl(RAW / "t3-engines.jsonl")
    t5 = load_jsonl(RAW / "t5.jsonl")
    idle = tsv_mibs(RAW / "idle.tsv")
    sess = tsv_mibs(RAW / "session.tsv")
    burst = tsv_mibs(RAW / "t5-stats.tsv")
    meta = {}
    mp = RAW / "image-meta.json"
    if mp.exists():
        meta = json.loads(mp.read_text(encoding="utf-8"))

    idle_p50 = median(idle)
    sess_peak = pmax(sess)
    burst_peak = pmax(burst)
    delta = None
    if idle_p50 is not None and sess_peak is not None:
        delta = round(max(0.0, sess_peak - idle_p50), 3)

    t3_ok = [r for r in t3 if r.get("assert_ok")]
    t3_times = [r["t_total"] for r in t3 if r.get("t_total") is not None]
    t5_ok = [r for r in t5 if r.get("assert_ok")]
    t5_times = [r["t_total"] for r in t5 if r.get("t_total") is not None]
    t5_n = len(t5)
    t5_rate = (len(t5_ok) / t5_n) if t5_n else None

    engine_matrix = {}
    for r in t3e:
        name = r.get("engines_param") or "unknown"
        engine_matrix.setdefault(name, {"n": 0, "http_200": 0, "n_results": [], "unresponsive": [], "assert_like": 0})
        e = engine_matrix[name]
        e["n"] += 1
        if r.get("http_code") == 200:
            e["http_200"] += 1
        e["n_results"].append(r.get("n_results") or 0)
        e["unresponsive"].extend(r.get("unresponsive_engines") or [])
        requested = (r.get("engines_param") or "").lower()
        hit = requested in [str(x).lower() for x in (r.get("engines") or [])]
        e["engine_contributed_n"] = e.get("engine_contributed_n", 0) + (1 if hit else 0)
        if hit and r.get("http_code") == 200:
            e["assert_like"] += 1
    for k, v in engine_matrix.items():
        v["success_rate"] = round(v["assert_like"] / v["n"], 3) if v["n"] else 0
        v["results_p50"] = percentile(v["n_results"], 50)

    oom = bool(meta.get("oom"))
    q = quality_search(t3)
    t3_p95 = percentile(t3_times, 95)
    t5_p95 = percentile(t5_times, 95)
    t3_p50 = percentile(t3_times, 50)
    t5_p50 = percentile(t5_times, 50)
    # combined latency: average of T3 and T5 scores if both
    l3 = lat_score_t3(t3_p95)
    l5 = lat_score_t5(t5_p95, t3_p95)
    lat_parts = [x for x in (l3, l5) if x is not None]
    lat = round(sum(lat_parts) / len(lat_parts), 1) if lat_parts else None
    mem = mem_score(idle_p50, delta, burst_peak)
    stab = stab_score(t5_rate if t5_rate is not None else (len(t3_ok) / len(t3) if t3 else None), oom)
    ops = 80
    total = None
    if None not in (mem, q, stab, lat, ops):
        total = round(mem * 0.30 + q * 0.25 + stab * 0.20 + lat * 0.15 + ops * 0.10, 1)

    out = {
        "scheme": "searxng-global",
        "group": "foreign_proxy",
        "image": {
            "ref": meta.get("ref") or "docker.io/searxng/searxng:2026.8.29-d226b78bc",
            "digest": meta.get("digest"),
            "id": meta.get("id"),
            "arch": meta.get("arch"),
            "pull": meta.get("pull"),
        },
        "probed_at": meta.get("probed_at"),
        "commands": meta.get("commands") or [],
        "proxy": meta.get("proxy") or {},
        "rss_source": "docker_stats+cgroup",
        "idle_mb_p50": idle_p50,
        "session_delta_mb": delta,
        "burst_peak_mb": burst_peak,
        "session_peak_mb": sess_peak,
        "templates": {
            "T3": {
                "n": len(t3),
                "success": len(t3_ok),
                "http_codes": [r.get("http_code") for r in t3],
                "p50_s": t3_p50,
                "p95_s": t3_p95,
                "assert_ok": bool(t3_ok),
                "engines_seen": sorted({e for r in t3 for e in (r.get("engines") or [])}),
                "unresponsive_union": sorted({e for r in t3 for e in (r.get("unresponsive_engines") or [])}),
                "reddit_ref_n": sum(1 for r in t3 if r.get("reddit_ref")),
                "note": "; ".join(sorted({r.get("note") for r in t3 if r.get("note")})),
            },
            "T3_engine_matrix": engine_matrix,
        },
        "t5": {
            "n": t5_n,
            "parallelism": 60,
            "queue_impl": "none",
            "success": len(t5_ok),
            "success_rate": t5_rate,
            "p50_s": t5_p50,
            "p95_s": t5_p95,
            "oom": oom,
            "http_429": sum(1 for r in t5 if r.get("http_code") == 429),
            "http_503": sum(1 for r in t5 if r.get("http_code") == 503),
            "first_ttfb_s": min((r.get("ttfb") or 999) for r in t5) if t5 else None,
            "last_t_total_s": max((r.get("t_total") or 0) for r in t5) if t5 else None,
            "wall_s": meta.get("t5_wall_s"),
        },
        "t4_waf": [],
        "scorecard": {
            "mem": mem,
            "quality": q,
            "stability": stab,
            "latency": lat,
            "ops": ops,
            "total": total,
        },
        "notes": meta.get("notes") or [],
    }
    (ROOT / "results.json").write_text(json.dumps(out, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    print(json.dumps({"wrote": str(ROOT / "results.json"), "scorecard": out["scorecard"]}, ensure_ascii=False))


if __name__ == "__main__":
    main()