onesvm-browser-server/bench/site-matrix/cdp_fetch.mjs

#!/usr/bin/env node
/**
 * CDP navigate + extract for site-matrix. No secrets.
 * Usage: node cdp_fetch.mjs <cdp_http> <url> [nav_timeout_s]
 */
const cdpHttp = process.argv[2] || "http://127.0.0.1:19223";
const targetUrl = process.argv[3];
const navTimeoutS = Number(process.argv[4] || 20);
if (!targetUrl) {
  console.error("usage: node cdp_fetch.mjs <cdp_http> <url> [nav_timeout_s]");
  process.exit(2);
}

const UA =
  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.7922.109 Safari/537.36";

const EXTRACT_JS = `(() => {
  const title = document.title || "";
  const bodyText = (document.body && document.body.innerText) ? document.body.innerText : "";
  const html = document.documentElement ? document.documentElement.outerHTML : "";
  const lc = (title + "\\n" + bodyText).toLowerCase();
  return {
    title,
    text: bodyText.slice(0, 8000),
    htmlLen: html.length,
    textLen: bodyText.length,
    htmlHead: html.slice(0, 4000),
    finalUrl: location.href,
    readyState: document.readyState,
    looksBlocked: /(robot|captcha|sorry|click the button|continue shopping|just a moment|attention required|verify you are human|are you a human|access denied)/i.test(lc),
  };
})()`;

function rewriteWs(wsUrl, httpEndpoint) {
  const u = new URL(wsUrl);
  const h = new URL(httpEndpoint);
  u.protocol = h.protocol === "https:" ? "wss:" : "ws:";
  u.hostname = h.hostname;
  u.port = h.port;
  return u.toString();
}

async function httpJson(url) {
  const r = await fetch(url, { signal: AbortSignal.timeout(5000) });
  if (!r.ok) throw new Error(`GET ${url} ${r.status}`);
  return r.json();
}

class Cdp {
  constructor(wsUrl) {
    this.wsUrl = wsUrl;
    this.ws = null;
    this.nextId = 1;
    this.pending = new Map();
    this.handlers = new Map();
  }
  async open() {
    this.ws = new WebSocket(this.wsUrl);
    await new Promise((resolve, reject) => {
      const t = setTimeout(() => reject(new Error("ws open timeout")), 8000);
      this.ws.addEventListener("open", () => {
        clearTimeout(t);
        resolve();
      });
      this.ws.addEventListener("error", (e) => {
        clearTimeout(t);
        reject(e);
      });
    });
    this.ws.addEventListener("message", (ev) => {
      let msg;
      try {
        msg = JSON.parse(ev.data);
      } catch {
        return;
      }
      if (msg.id && this.pending.has(msg.id)) {
        const { resolve, reject } = this.pending.get(msg.id);
        this.pending.delete(msg.id);
        if (msg.error) reject(new Error(JSON.stringify(msg.error)));
        else resolve(msg.result || {});
        return;
      }
      if (msg.method) {
        const hs = this.handlers.get(msg.method) || [];
        for (const h of hs) h(msg.params || {}, msg.sessionId);
      }
    });
  }
  on(method, fn) {
    const arr = this.handlers.get(method) || [];
    arr.push(fn);
    this.handlers.set(method, arr);
  }
  send(method, params = {}, sessionId) {
    const id = this.nextId++;
    const payload = { id, method, params };
    if (sessionId) payload.sessionId = sessionId;
    return new Promise((resolve, reject) => {
      const t = setTimeout(() => {
        this.pending.delete(id);
        reject(new Error(`cdp timeout ${method}`));
      }, Math.max(8000, navTimeoutS * 1000));
      this.pending.set(id, {
        resolve: (v) => {
          clearTimeout(t);
          resolve(v);
        },
        reject: (e) => {
          clearTimeout(t);
          reject(e);
        },
      });
      this.ws.send(JSON.stringify(payload));
    });
  }
  close() {
    try {
      this.ws && this.ws.close();
    } catch {}
  }
}

function detectVendor(ex, status, headersBlob) {
  const blob = `${ex.title || ""}\n${ex.text || ""}\n${ex.htmlHead || ""}\n${headersBlob || ""}`.toLowerCase();
  if (/cloudflare|cf-challenge|just a moment|cf-turnstile|attention required/.test(blob)) return "cloudflare";
  if (/datadome|captcha-delivery/.test(blob)) return "waf";
  if (/akamai/.test(blob) && /access denied/.test(blob)) return "waf";
  if (/amazon|opfcaptcha|validatecaptcha|enter the characters you see/.test(blob) && /robot|captcha/.test(blob))
    return "waf";
  if (/paywall|subscribe to continue|become a member/.test(blob)) return "paywall";
  if (/recaptcha|hcaptcha|verify you are human|captcha/.test(blob) && /robot|human|verify/.test(blob))
    return "captcha";
  if (ex.looksBlocked) return "waf";
  if (status && status >= 300 && status < 400) return "redirect";
  if (!ex.textLen || ex.textLen < 80) return "empty";
  return "none";
}

async function pickWs() {
  const notes = [];
  try {
    const ver = await httpJson(`${cdpHttp}/json/version`);
    notes.push({ step: "json/version", keys: Object.keys(ver) });
    if (ver.webSocketDebuggerUrl) {
      return { ws: rewriteWs(ver.webSocketDebuggerUrl, cdpHttp), notes, mode: "version" };
    }
  } catch (e) {
    notes.push({ step: "json/version", error: String(e) });
  }
  try {
    const list = await httpJson(`${cdpHttp}/json/list`);
    notes.push({ step: "json/list", n: Array.isArray(list) ? list.length : 0 });
    const page = (list || []).find((t) => t.webSocketDebuggerUrl);
    if (page) return { ws: rewriteWs(page.webSocketDebuggerUrl, cdpHttp), notes, mode: "list" };
  } catch (e) {
    notes.push({ step: "json/list", error: String(e) });
  }
  const u = new URL(cdpHttp);
  return { ws: `ws://${u.hostname}:${u.port}`, notes, mode: "bare" };
}

async function main() {
  const t0 = Date.now();
  const pick = await pickWs();
  const cdp = new Cdp(pick.ws);
  await cdp.open();
  let sessionId;
  let targetId;
  const errors = [];
  try {
    try {
      const created = await cdp.send("Target.createTarget", { url: "about:blank" });
      targetId = created.targetId;
      const attached = await cdp.send("Target.attachToTarget", { targetId, flatten: true });
      sessionId = attached.sessionId;
    } catch (e) {
      errors.push(`Target.create/attach: ${e.message}`);
    }
    const send = (m, p) => cdp.send(m, p, sessionId);
    for (const m of ["Page.enable", "Runtime.enable", "Network.enable"]) {
      try {
        await send(m);
      } catch (e) {
        errors.push(`${m}: ${e.message}`);
      }
    }
    try {
      await send("Network.setUserAgentOverride", { userAgent: UA });
    } catch (e) {
      errors.push(`UA override: ${e.message}`);
    }

    let docStatus = 0;
    let docUrl = targetUrl;
    const headerBits = [];
    const redirectChain = [];
    cdp.on("Network.responseReceived", (params) => {
      const resp = params.response || {};
      const t = (params.type || resp.mimeType || "").toString();
      const url = resp.url || "";
      if (t === "Document" || /html/i.test(resp.mimeType || "") || url === targetUrl || url === docUrl) {
        if (!docStatus || t === "Document") {
          docStatus = resp.status || docStatus;
          docUrl = url || docUrl;
        }
        const h = resp.headers || {};
        for (const [k, v] of Object.entries(h)) {
          if (/cf-|server|x-amzn|x-cache|location|refresh/i.test(k)) headerBits.push(`${k}:${v}`);
        }
      }
    });
    cdp.on("Network.requestWillBeSent", (params) => {
      if (params.redirectResponse) {
        redirectChain.push({
          status: params.redirectResponse.status,
          from: params.redirectResponse.url,
          to: (params.request && params.request.url) || "",
        });
      }
    });

    let loadFired = false;
    cdp.on("Page.loadEventFired", () => {
      loadFired = true;
    });

    let navResult;
    try {
      navResult = await send("Page.navigate", { url: targetUrl });
    } catch (e) {
      throw new Error(`Page.navigate failed: ${e.message}`);
    }
    const navDeadline = Date.now() + navTimeoutS * 1000;
    while (Date.now() < navDeadline && !loadFired) {
      await new Promise((r) => setTimeout(r, 100));
    }
    await new Promise((r) => setTimeout(r, loadFired ? 800 : 1500));

    let extract = {};
    try {
      const ev = await send("Runtime.evaluate", {
        expression: EXTRACT_JS,
        returnByValue: true,
        awaitPromise: true,
      });
      extract = (ev.result && ev.result.value) || {};
      if (ev.exceptionDetails) {
        errors.push(`evaluate exception: ${JSON.stringify(ev.exceptionDetails)}`);
      }
    } catch (e) {
      errors.push(`Runtime.evaluate: ${e.message}`);
    }

    if (targetId) {
      try {
        await cdp.send("Target.closeTarget", { targetId });
      } catch {}
    }

    const vendor = detectVendor(extract, docStatus, headerBits.join("\n"));
    const out = {
      ok: true,
      cdp_mode: pick.mode,
      t_total: (Date.now() - t0) / 1000,
      errorText: navResult && navResult.errorText,
      http_status: docStatus || (extract.finalUrl ? 200 : 0),
      finalUrl: extract.finalUrl || docUrl || "",
      title: extract.title || "",
      text: extract.text || "",
      htmlLen: extract.htmlLen || 0,
      textLen: extract.textLen || 0,
      htmlHead: extract.htmlHead || "",
      looksBlocked: Boolean(extract.looksBlocked),
      challenge_vendor: vendor,
      redirect_chain: redirectChain,
      header_bits: headerBits.slice(0, 20),
      loadFired,
      cdp_errors: errors,
      pick_notes: pick.notes,
    };
    process.stdout.write(JSON.stringify(out) + "\n");
  } finally {
    cdp.close();
  }
}

main().catch((e) => {
  process.stdout.write(
    JSON.stringify({
      ok: false,
      error: String(e && e.message ? e.message : e),
      t_total: 0,
      http_status: 0,
      title: "",
      text: "",
      htmlLen: 0,
      textLen: 0,
      challenge_vendor: "empty",
      finalUrl: "",
    }) + "\n"
  );
  process.exit(0);
});