chii
983259836d
docs: 联网搜索服务架构方案全套（plan-final/design-arch/选型决策/整合导览/MCP文档/部署预设/联调手册） bench: 5 方案 + 代理 + 站点矩阵本机实测工程（无密钥） 部署目标：primary mgr1 先行测试（待批准后执行）
106 lines
3.4 KiB
Bash
Executable file
106 lines
3.4 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# T4 / T5 CDP request helper. Env: CDP_HTTP SCHEME TIER URL
|
|
set -euo pipefail
|
|
ROOT="$(cd "$(dirname "$0")" && pwd)"
|
|
CDP_HTTP="${CDP_HTTP:-http://127.0.0.1:19222}"
|
|
SCHEME="${SCHEME:-lightpanda}"
|
|
TIER="${1:?tier}"
|
|
URL="${2:?url}"
|
|
TIMEOUT="${3:-25}"
|
|
OUT="$(node "$ROOT/cdp_fetch.mjs" "$CDP_HTTP" "$URL" "$TIMEOUT")"
|
|
python3 - "$OUT" "$SCHEME" "$TIER" "$URL" <<'PY'
|
|
import json, sys, datetime, re
|
|
raw, scheme, tier, url = sys.argv[1:5]
|
|
try:
|
|
d = json.loads(raw)
|
|
except Exception as e:
|
|
d = {"ok": False, "error": f"json {e}", "raw": raw[:500]}
|
|
title = d.get("title") or ""
|
|
text = d.get("text") or ""
|
|
price = d.get("price") or ""
|
|
jprice = d.get("jsonldPrice") or ""
|
|
quote_n = int(d.get("quoteCount") or 0)
|
|
blocked = bool(d.get("blocked"))
|
|
vendor = d.get("challenge_vendor") or "none"
|
|
lc = (title + "\n" + text).lower()
|
|
assert_ok = False
|
|
note = ""
|
|
if tier == "T4-ctrl":
|
|
assert_ok = ("quote" in lc) or quote_n >= 3 or bool(re.search(r"albert einstein|jane austen|marilyn monroe|steve jobs", lc))
|
|
if not assert_ok:
|
|
note = "js_kernel_or_empty"
|
|
elif tier == "T4-shop":
|
|
assert_ok = (("wool runner" in lc) or ("allbirds" in lc)) and (
|
|
bool(re.search(r"\$[0-9]", price or text)) or bool(re.search(r"[0-9]", jprice))
|
|
)
|
|
if not assert_ok:
|
|
note = "shop_fields_missing"
|
|
elif tier == "T4-amz":
|
|
if blocked or re.search(r"robot|captcha|sorry|click the button|continue shopping", lc):
|
|
assert_ok = True # interception is expected; not a scheme fail
|
|
note = "waf"
|
|
blocked = True
|
|
else:
|
|
assert_ok = bool(title.strip()) and not blocked
|
|
elif tier == "T4-medium":
|
|
assert_ok = bool(title.strip()) and not blocked and len(text) >= 200
|
|
if blocked:
|
|
note = "medium_wall_or_waf"
|
|
elif not assert_ok:
|
|
note = "medium_thin"
|
|
elif tier == "T4-t3x":
|
|
assert_ok = bool(title.strip()) and len(text) >= 80
|
|
if not assert_ok:
|
|
note = "serp_thin"
|
|
elif tier == "T5":
|
|
assert_ok = ("quote" in lc) or quote_n >= 3
|
|
if not assert_ok:
|
|
note = "t5_ctrl_fail"
|
|
else:
|
|
assert_ok = bool(d.get("ok")) and bool(title)
|
|
|
|
out = {
|
|
"ts": datetime.datetime.now().astimezone().isoformat(timespec="seconds"),
|
|
"scheme": scheme,
|
|
"template": tier,
|
|
"url": url,
|
|
"final_url": d.get("finalUrl") or "",
|
|
"http_code": 200 if d.get("ok") else 0,
|
|
"http_status_in_page": None,
|
|
"t_total": d.get("t_total") or 0,
|
|
"ttfb": d.get("t_nav_s") or 0,
|
|
"t_namelookup": 0,
|
|
"t_connect": 0,
|
|
"bytes": d.get("htmlLen") or 0,
|
|
"html_len": d.get("htmlLen") or 0,
|
|
"text_len": d.get("textLen") or 0,
|
|
"title": title[:240],
|
|
"price": price,
|
|
"jsonld_price": jprice,
|
|
"quote_count": quote_n,
|
|
"blocked": blocked,
|
|
"challenge_vendor": vendor,
|
|
"networkidle": d.get("networkidle"),
|
|
"assert_ok": bool(assert_ok),
|
|
"note": note or d.get("error") or "",
|
|
"cdp_mode": d.get("cdp_mode"),
|
|
"cdp_errors": d.get("cdp_errors") or [],
|
|
"text_head": text[:500],
|
|
}
|
|
print(json.dumps(out, ensure_ascii=False))
|
|
# excerpt to stderr for samples
|
|
excerpt = {
|
|
"tier": tier,
|
|
"title": title[:200],
|
|
"final_url": out["final_url"],
|
|
"html_len": out["html_len"],
|
|
"text_len": out["text_len"],
|
|
"price": price,
|
|
"jsonld_price": jprice,
|
|
"blocked": blocked,
|
|
"vendor": vendor,
|
|
"text_head": text[:800],
|
|
"html_head": (d.get("htmlHead") or "")[:800],
|
|
}
|
|
sys.stderr.write(json.dumps(excerpt, ensure_ascii=False) + "\n")
|
|
PY
|