chii
983259836d
docs: 联网搜索服务架构方案全套（plan-final/design-arch/选型决策/整合导览/MCP文档/部署预设/联调手册） bench: 5 方案 + 代理 + 站点矩阵本机实测工程（无密钥） 部署目标：primary mgr1 先行测试（待批准后执行）
101 lines
3.7 KiB
Python
Executable file
101 lines
3.7 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
"""Assert one trafilatura T2 JSON body. stdout: JSON verdict."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
import sys
|
|
|
|
NAV = re.compile(r"网站地图|设为首页|加入收藏|政务微信|客户端下载|无障碍浏览", re.I)
|
|
CRUMB = re.compile(r"首页\s*[>|/]\s*政策")
|
|
HTML_DIV = re.compile(r"<div", re.I)
|
|
HTML_SCRIPT = re.compile(r"<script", re.I)
|
|
FALLBACK_URL = "https://www.gov.cn/zhengce/2021-03/13/content_5592681.htm"
|
|
|
|
|
|
def eval_body(raw: str, expect_url: str = "", title_cjk4: str = "") -> dict:
|
|
out = {
|
|
"assert_ok": False,
|
|
"fail_class": "",
|
|
"note": "",
|
|
"title": "",
|
|
"char_count": 0,
|
|
"truncated": None,
|
|
"ok_field": None,
|
|
"has_shisiwu": False,
|
|
"has_2035": False,
|
|
"has_cjk4": False,
|
|
"nav_hit": False,
|
|
"crumb_count": 0,
|
|
"html_div": 0,
|
|
"html_script": 0,
|
|
}
|
|
try:
|
|
data = json.loads(raw)
|
|
except json.JSONDecodeError:
|
|
out["fail_class"] = "html_dump"
|
|
out["note"] = "body_not_json"
|
|
return out
|
|
out["ok_field"] = data.get("ok")
|
|
out["title"] = (data.get("title") or "")[:200]
|
|
md = data.get("markdown") or ""
|
|
out["char_count"] = data.get("char_count") if data.get("char_count") is not None else len(md)
|
|
out["truncated"] = data.get("truncated")
|
|
notes = []
|
|
if data.get("fail_class"):
|
|
out["fail_class"] = data["fail_class"]
|
|
notes.append(str(data.get("error") or data["fail_class"]))
|
|
if not data.get("ok"):
|
|
out["fail_class"] = out["fail_class"] or "fetch_fail"
|
|
out["note"] = "; ".join(notes) or "ok!=true"
|
|
return out
|
|
if not md or len(md) < 800:
|
|
out["fail_class"] = "empty_extract"
|
|
notes.append(f"markdown_len={len(md)}")
|
|
if not (data.get("title") or "").strip():
|
|
notes.append("empty_title")
|
|
out["fail_class"] = out["fail_class"] or "empty_extract"
|
|
out["nav_hit"] = bool(NAV.search(md))
|
|
out["crumb_count"] = len(CRUMB.findall(md))
|
|
if out["nav_hit"] or out["crumb_count"] >= 3:
|
|
out["fail_class"] = out["fail_class"] or "nav_residue"
|
|
notes.append("nav_residue")
|
|
out["html_div"] = len(HTML_DIV.findall(md))
|
|
out["html_script"] = len(HTML_SCRIPT.findall(md))
|
|
if out["html_div"] or out["html_script"]:
|
|
out["fail_class"] = out["fail_class"] or "html_dump"
|
|
notes.append("html_dump")
|
|
out["has_shisiwu"] = "十四五" in md
|
|
out["has_2035"] = "2035" in md
|
|
out["has_cjk4"] = bool(title_cjk4) and title_cjk4 in md
|
|
use_fallback = (not expect_url) or expect_url.rstrip("/") == FALLBACK_URL.rstrip("/")
|
|
if use_fallback:
|
|
if not (out["has_shisiwu"] and out["has_2035"]):
|
|
notes.append("missing_十四五_or_2035")
|
|
if not out["fail_class"]:
|
|
out["fail_class"] = "empty_extract"
|
|
elif title_cjk4 and not out["has_cjk4"]:
|
|
notes.append(f"missing_cjk4={title_cjk4}")
|
|
if not out["fail_class"]:
|
|
out["fail_class"] = "empty_extract"
|
|
if data.get("truncated") is True:
|
|
max_chars = 20000
|
|
if int(out["char_count"] or 0) != max_chars:
|
|
notes.append(f"truncated_char_count={out['char_count']}")
|
|
if not out["fail_class"]:
|
|
out["fail_class"] = "empty_extract"
|
|
out["assert_ok"] = out["fail_class"] == ""
|
|
out["note"] = "; ".join(notes)
|
|
return out
|
|
|
|
|
|
def main() -> None:
|
|
path = sys.argv[1]
|
|
expect_url = sys.argv[2] if len(sys.argv) > 2 else ""
|
|
title_cjk4 = sys.argv[3] if len(sys.argv) > 3 else ""
|
|
raw = open(path, "r", encoding="utf-8", errors="replace").read()
|
|
print(json.dumps(eval_body(raw, expect_url, title_cjk4), ensure_ascii=False))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|