onesvm-browser-server/bench/lightpanda/cdp_fetch.mjs

#!/usr/bin/env node
/**
 * Minimal CDP navigate+extract via native WebSocket (Node 22+).
 * Usage: node cdp_fetch.mjs <cdp_http> <url> [nav_timeout_s]
 * Prints one JSON object to stdout. No secrets.
 */
const cdpHttp = process.argv[2] || "http://127.0.0.1:19222";
const targetUrl = process.argv[3];
const navTimeoutS = Number(process.argv[4] || 25);
if (!targetUrl) {
  console.error("usage: node cdp_fetch.mjs <cdp_http> <url> [nav_timeout_s]");
  process.exit(2);
}

const UA =
  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.7922.109 Safari/537.36";

const EXTRACT_JS = `(() => {
  const title = document.title || "";
  const bodyText = (document.body && document.body.innerText) ? document.body.innerText : "";
  const html = document.documentElement ? document.documentElement.outerHTML : "";
  let price = "";
  const meta = document.querySelector('meta[itemprop="price"], meta[property="product:price:amount"]');
  if (meta) price = (meta.getAttribute("content") || "").trim();
  if (!price) {
    const el = document.querySelector('[class*="price"], [data-testid*="price"], .product-price, #priceblock_ourprice, .a-price .a-offscreen');
    if (el) price = (el.textContent || "").trim().slice(0, 80);
  }
  let jsonldPrice = "";
  for (const s of document.querySelectorAll('script[type="application/ld+json"]')) {
    try {
      const j = JSON.parse(s.textContent || "null");
      const arr = Array.isArray(j) ? j : [j];
      const walk = (o) => {
        if (!o || typeof o !== "object") return;
        if (o.offers) {
          const off = Array.isArray(o.offers) ? o.offers[0] : o.offers;
          if (off && (off.price || off.lowPrice)) jsonldPrice = String(off.price || off.lowPrice);
        }
        if (Array.isArray(o["@graph"])) o["@graph"].forEach(walk);
      };
      arr.forEach(walk);
    } catch (e) {}
  }
  const quoteCount = document.querySelectorAll(".quote").length;
  const lc = (title + "\\n" + bodyText).toLowerCase();
  return {
    title,
    text: bodyText.slice(0, 8000),
    htmlLen: html.length,
    textLen: bodyText.length,
    price,
    jsonldPrice,
    quoteCount,
    htmlHead: html.slice(0, 1800),
    finalUrl: location.href,
    readyState: document.readyState,
    looksBlocked: /(robot|captcha|sorry|click the button|continue shopping|just a moment|attention required|verify you are human|are you a human)/i.test(lc),
  };
})()`;

function rewriteWs(wsUrl, httpEndpoint) {
  const u = new URL(wsUrl);
  const h = new URL(httpEndpoint);
  u.protocol = h.protocol === "https:" ? "wss:" : "ws:";
  u.hostname = h.hostname;
  u.port = h.port;
  return u.toString();
}

async function httpJson(url) {
  const r = await fetch(url, { signal: AbortSignal.timeout(5000) });
  if (!r.ok) throw new Error(`GET ${url} ${r.status}`);
  return r.json();
}

class Cdp {
  constructor(wsUrl) {
    this.wsUrl = wsUrl;
    this.ws = null;
    this.nextId = 1;
    this.pending = new Map();
    this.events = [];
    this.handlers = new Map();
  }
  async open() {
    this.ws = new WebSocket(this.wsUrl);
    await new Promise((resolve, reject) => {
      const t = setTimeout(() => reject(new Error("ws open timeout")), 8000);
      this.ws.addEventListener("open", () => {
        clearTimeout(t);
        resolve();
      });
      this.ws.addEventListener("error", (e) => {
        clearTimeout(t);
        reject(e);
      });
    });
    this.ws.addEventListener("message", (ev) => {
      let msg;
      try {
        msg = JSON.parse(ev.data);
      } catch {
        return;
      }
      if (msg.id && this.pending.has(msg.id)) {
        const { resolve, reject } = this.pending.get(msg.id);
        this.pending.delete(msg.id);
        if (msg.error) reject(new Error(JSON.stringify(msg.error)));
        else resolve(msg.result || {});
        return;
      }
      if (msg.method) {
        this.events.push(msg);
        const hs = this.handlers.get(msg.method) || [];
        for (const h of hs) h(msg.params || {}, msg.sessionId);
      }
    });
  }
  on(method, fn) {
    const arr = this.handlers.get(method) || [];
    arr.push(fn);
    this.handlers.set(method, arr);
  }
  send(method, params = {}, sessionId) {
    const id = this.nextId++;
    const payload = { id, method, params };
    if (sessionId) payload.sessionId = sessionId;
    return new Promise((resolve, reject) => {
      const t = setTimeout(() => {
        this.pending.delete(id);
        reject(new Error(`cdp timeout ${method}`));
      }, 20000);
      this.pending.set(id, {
        resolve: (v) => {
          clearTimeout(t);
          resolve(v);
        },
        reject: (e) => {
          clearTimeout(t);
          reject(e);
        },
      });
      this.ws.send(JSON.stringify(payload));
    });
  }
  close() {
    try {
      this.ws && this.ws.close();
    } catch {}
  }
}

function detectVendor(ex) {
  const blob = `${ex.title || ""}\n${ex.text || ""}\n${ex.htmlHead || ""}`.toLowerCase();
  if (/cloudflare|cf-challenge|just a moment|cf-turnstile/.test(blob)) return "cloudflare";
  if (/datadome|captcha-delivery/.test(blob)) return "datadome";
  if (/akamai|access denied/.test(blob) && /akamai/.test(blob)) return "akamai";
  if (/amazon|opfcaptcha|validatecaptcha|enter the characters you see/.test(blob) && /robot|captcha/.test(blob))
    return "amazon";
  if (ex.looksBlocked) return "unknown";
  return "none";
}

async function pickWs() {
  const notes = [];
  try {
    const ver = await httpJson(`${cdpHttp}/json/version`);
    notes.push({ step: "json/version", keys: Object.keys(ver) });
    if (ver.webSocketDebuggerUrl) {
      return { ws: rewriteWs(ver.webSocketDebuggerUrl, cdpHttp), notes, mode: "version" };
    }
  } catch (e) {
    notes.push({ step: "json/version", error: String(e) });
  }
  try {
    const list = await httpJson(`${cdpHttp}/json/list`);
    notes.push({ step: "json/list", n: Array.isArray(list) ? list.length : 0 });
    const page = (list || []).find((t) => t.webSocketDebuggerUrl);
    if (page) return { ws: rewriteWs(page.webSocketDebuggerUrl, cdpHttp), notes, mode: "list" };
  } catch (e) {
    notes.push({ step: "json/list", error: String(e) });
  }
  try {
    const created = await httpJson(`${cdpHttp}/json/new?${encodeURIComponent("about:blank")}`);
    if (created.webSocketDebuggerUrl) {
      return { ws: rewriteWs(created.webSocketDebuggerUrl, cdpHttp), notes, mode: "new" };
    }
  } catch (e) {
    notes.push({ step: "json/new", error: String(e) });
  }
  // last resort: ws on same host/port
  const u = new URL(cdpHttp);
  return { ws: `ws://${u.hostname}:${u.port}`, notes, mode: "bare" };
}

async function waitEvent(cdp, method, timeoutMs, pred) {
  return new Promise((resolve, reject) => {
    const t = setTimeout(() => reject(new Error(`wait ${method} timeout`)), timeoutMs);
    const fn = (params, sessionId) => {
      if (pred && !pred(params, sessionId)) return;
      clearTimeout(t);
      resolve({ params, sessionId });
    };
    cdp.on(method, fn);
  });
}

async function main() {
  const t0 = Date.now();
  const pick = await pickWs();
  const cdp = new Cdp(pick.ws);
  await cdp.open();
  let sessionId;
  let targetId;
  const errors = [];
  try {
    try {
      await cdp.send("Browser.getVersion");
    } catch (e) {
      errors.push(`Browser.getVersion: ${e.message}`);
    }
    try {
      const created = await cdp.send("Target.createTarget", { url: "about:blank" });
      targetId = created.targetId;
      const attached = await cdp.send("Target.attachToTarget", { targetId, flatten: true });
      sessionId = attached.sessionId;
    } catch (e) {
      errors.push(`Target.create/attach: ${e.message}`);
      sessionId = undefined;
    }
    const send = (m, p) => cdp.send(m, p, sessionId);
    try {
      await send("Page.enable");
    } catch (e) {
      errors.push(`Page.enable: ${e.message}`);
    }
    try {
      await send("Runtime.enable");
    } catch (e) {
      errors.push(`Runtime.enable: ${e.message}`);
    }
    try {
      await send("Network.enable");
    } catch (e) {
      errors.push(`Network.enable: ${e.message}`);
    }
    try {
      await send("Network.setUserAgentOverride", { userAgent: UA });
    } catch (e) {
      errors.push(`UA override: ${e.message}`);
    }

    let inflight = 0;
    let lastNet = Date.now();
    cdp.on("Network.requestWillBeSent", () => {
      inflight += 1;
      lastNet = Date.now();
    });
    cdp.on("Network.loadingFinished", () => {
      inflight = Math.max(0, inflight - 1);
      lastNet = Date.now();
    });
    cdp.on("Network.loadingFailed", () => {
      inflight = Math.max(0, inflight - 1);
      lastNet = Date.now();
    });

    let loadFired = false;
    cdp.on("Page.loadEventFired", () => {
      loadFired = true;
    });
    cdp.on("Page.domContentEventFired", () => {});

    const navStart = Date.now();
    let navResult;
    try {
      navResult = await send("Page.navigate", { url: targetUrl });
    } catch (e) {
      throw new Error(`Page.navigate failed: ${e.message}`);
    }
    const navDeadline = Date.now() + navTimeoutS * 1000;
    while (Date.now() < navDeadline && !loadFired) {
      await new Promise((r) => setTimeout(r, 100));
    }
    const loadWaitMs = Date.now() - navStart;
    // networkidle up to 8s, else 2s settle
    const idleDeadline = Date.now() + 8000;
    let usedIdle = false;
    while (Date.now() < idleDeadline) {
      if (inflight === 0 && Date.now() - lastNet >= 500 && loadFired) {
        usedIdle = true;
        break;
      }
      await new Promise((r) => setTimeout(r, 100));
    }
    if (!usedIdle) {
      await new Promise((r) => setTimeout(r, 2000));
    }

    let extract = {};
    try {
      const ev = await send("Runtime.evaluate", {
        expression: EXTRACT_JS,
        returnByValue: true,
        awaitPromise: true,
      });
      extract = (ev.result && ev.result.value) || {};
      if (ev.exceptionDetails) {
        errors.push(`evaluate exception: ${JSON.stringify(ev.exceptionDetails)}`);
      }
    } catch (e) {
      errors.push(`Runtime.evaluate: ${e.message}`);
    }

    if (targetId) {
      try {
        await cdp.send("Target.closeTarget", { targetId });
      } catch {}
    }

    const vendor = detectVendor(extract);
    const out = {
      ok: true,
      cdp_mode: pick.mode,
      t_total: (Date.now() - t0) / 1000,
      t_nav_s: loadWaitMs / 1000,
      networkidle: usedIdle,
      errorText: navResult && navResult.errorText,
      loaderId: navResult && navResult.loaderId,
      ...extract,
      challenge_vendor: vendor,
      blocked: Boolean(extract.looksBlocked) || vendor !== "none",
      cdp_errors: errors,
      pick_notes: pick.notes,
    };
    process.stdout.write(JSON.stringify(out) + "\n");
  } finally {
    cdp.close();
  }
}

main().catch((e) => {
  process.stdout.write(
    JSON.stringify({
      ok: false,
      error: String(e && e.message ? e.message : e),
      t_total: 0,
      title: "",
      text: "",
      htmlLen: 0,
      textLen: 0,
      blocked: true,
      challenge_vendor: "unknown",
    }) + "\n"
  );
  process.exit(0);
});