cloakfetch-gateway/test/browser_integration.test.ts

import test from "node:test";
import assert from "node:assert/strict";
import http from "node:http";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { once } from "node:events";
import { Fetcher } from "../src/fetcher.js";
import { NetworkHistory } from "../src/network_history.js";
import { RequestHistory } from "../src/request_history.js";
import { Semaphore } from "../src/semaphore.js";
import { buildServer } from "../src/server.js";
import { SourceSessionManager } from "../src/source_session_manager.js";
import type { AppConfig, NetworkRecord } from "../src/types.js";
import { WebSocketHub } from "../src/websocket_hub.js";

const PNG = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=", "base64");

test("real Chromium source sessions and network observer", { timeout: 120_000 }, async (t) => {
  const target = await startTargetServer();
  const profileRoot = await fs.mkdtemp(path.join(os.tmpdir(), "cloakfetch-test-profile-"));
  const config = createConfig(profileRoot);
  const history = new RequestHistory(config.historyLimit);
  const networkHistory = new NetworkHistory(config.networkHistoryLimit);
  let fetcher: Fetcher;
  const hub = new WebSocketHub(config.websocketHeartbeatMs, () => ({
    server_time: new Date().toISOString(),
    stats: fetcher.stats,
  }));
  const sources = new SourceSessionManager(config, networkHistory, hub);
  fetcher = new Fetcher(config, sources, new Semaphore(config.concurrency));
  const gateway = await buildServer(config, fetcher, history, networkHistory, sources, hub);
  await gateway.listen({ host: "127.0.0.1", port: 0 });
  const gatewayBase = `http://127.0.0.1:${(gateway.server.address() as { port: number }).port}`;
  t.after(async () => {
    await gateway.close();
    await sources.closeAll();
    await target.close();
    if (path.resolve(profileRoot).startsWith(os.tmpdir())) {
      await fs.rm(profileRoot, { recursive: true, force: true });
    }
  });

  const ws = new WebSocket(gatewayBase.replace("http://", "ws://") + "/v1/ws");
  const wsMessages: unknown[] = [];
  ws.onmessage = (event) => {
    wsMessages.push(JSON.parse(event.data));
  };
  await waitFor(() => wsMessages.some((message) => (message as { type?: string }).type === "hello"));

  await postJson(`${gatewayBase}/v1/sources`, {
    id: "s1",
    warmup_url: `${target.base}/set-cookie?value=a`,
    viewport: { width: 1280, height: 720 },
  });
  await postJson(`${gatewayBase}/v1/sources`, {
    id: "s2",
    warmup_url: `${target.base}/set-cookie?value=b`,
  });

  const echo1 = await postFetchText(gatewayBase, `${target.base}/echo-cookie`, "s1");
  const echo2 = await postFetchText(gatewayBase, `${target.base}/echo-cookie`, "s2");
  assert.match(echo1, /session=a/);
  assert.match(echo2, /session=b/);

  await postJson(`${gatewayBase}/v1/diagnostics/fetch`, {
    source_id: "s1",
    url: `${target.base}/`,
    timeout_ms: 30000,
  });
  await waitFor(() => hasPaths(networkHistory.recent(2000), ["/", "/script.js", "/style.css", "/image.png", "/frame", "/api?script=1"]));
  assert.ok(target.hits.some((hit) => hit.pathname === "/api" && hit.searchParams.get("script") === "1"));

  await postJson(`${gatewayBase}/v1/sources/s1/warmup`, {
    url: `${target.base}/localstorage-check`,
    timeout_ms: 30000,
  });
  await waitFor(() => target.hits.some((hit) => hit.pathname === "/api" && hit.searchParams.get("ls") === "a"));

  await postJson(`${gatewayBase}/v1/sources/s2/warmup`, {
    url: `${target.base}/localstorage-check`,
    timeout_ms: 30000,
  });
  await waitFor(() => target.hits.some((hit) => hit.pathname === "/api" && hit.searchParams.get("ls") === "b"));

  await postJson(`${gatewayBase}/v1/diagnostics/fetch`, {
    source_id: "s1",
    url: `${target.base}/redirect`,
    timeout_ms: 30000,
  });
  await waitFor(() => networkHistory.recent(2000).some((record) => record.redirect_to?.endsWith("/image.png")));

  await postJson(`${gatewayBase}/v1/diagnostics/fetch`, {
    source_id: "s1",
    url: `${target.base}/fail-page`,
    timeout_ms: 30000,
  });
  await waitFor(() => networkHistory.recent(2000).some((record) => record.failure_text && record.url.endsWith("/fail.png")));

  await postFetchText(gatewayBase, `${target.base}/cache-page`, "s1");
  await postFetchText(gatewayBase, `${target.base}/cache-page`, "s1");
  await waitFor(() => networkHistory.recent(2000).some((record) => record.url.endsWith("/cache.png") && record.from_cache));

  await postJson(`${gatewayBase}/v1/diagnostics/fetch`, {
    source_id: "s1",
    url: `${target.base}/many`,
    timeout_ms: 30000,
  });
  assert.ok(networkHistory.summary().retained_total > 20);
  await waitFor(() => wsMessages.some((message) => (message as { type?: string }).type === "network.request"));
  await waitFor(() => wsMessages.some((message) => (message as { type?: string }).type === "network.finished"));

  const imageResponse = await fetch(`${gatewayBase}/v1/fetch`, {
    method: "POST",
    headers: { "content-type": "application/json" },
    body: JSON.stringify({ source_id: "s1", url: `${target.base}/image.png`, timeout_ms: 30000 }),
  });
  const imageBody = Buffer.from(await imageResponse.arrayBuffer());
  assert.equal(imageResponse.status, 200);
  assert.equal(imageBody[0], 0x89);

  const pages = await fetchJson<unknown[]>(`${gatewayBase}/v1/sources/s1/pages`);
  const cookies = await fetchJson<unknown[]>(`${gatewayBase}/v1/sources/s1/cookies`);
  assert.ok(pages.length > 0);
  assert.ok(cookies.length > 0);

  await fetch(`${gatewayBase}/v1/sources/s2`, { method: "DELETE" });
  const removed = await fetch(`${gatewayBase}/v1/sources/s2`);
  assert.equal(removed.status, 404);
  ws.close();
});

async function startTargetServer() {
  const hits: URL[] = [];
  const server = http.createServer((req, res) => {
    const url = new URL(req.url ?? "/", "http://127.0.0.1");
    hits.push(url);
    if (url.pathname === "/") {
      send(res, 200, "text/html", `<!doctype html><html><head><title>root</title><link rel="stylesheet" href="/style.css"></head><body><img src="/image.png"><iframe src="/frame"></iframe><script src="/script.js"></script></body></html>`);
      return;
    }
    if (url.pathname === "/script.js") {
      send(res, 200, "application/javascript", `window.scriptLoaded = true; fetch('/api?script=1').catch(() => {});`);
      return;
    }
    if (url.pathname === "/style.css") {
      send(res, 200, "text/css", "body { color: rgb(1, 2, 3); }");
      return;
    }
    if (url.pathname === "/image.png") {
      sendBuffer(res, 200, "image/png", PNG);
      return;
    }
    if (url.pathname === "/frame") {
      send(res, 200, "text/html", "<!doctype html><title>frame</title><p>frame</p>");
      return;
    }
    if (url.pathname === "/api") {
      send(res, 200, "application/json", JSON.stringify({ ok: true }));
      return;
    }
    if (url.pathname === "/redirect") {
      res.writeHead(302, { location: "/image.png" });
      res.end();
      return;
    }
    if (url.pathname === "/set-cookie") {
      const value = url.searchParams.get("value") ?? "x";
      res.writeHead(200, {
        "content-type": "text/html",
        "set-cookie": `session=${value}; Path=/`,
      });
      res.end(`<!doctype html><script>localStorage.setItem('session', ${JSON.stringify(value)});</script>`);
      return;
    }
    if (url.pathname === "/echo-cookie") {
      send(res, 200, "text/plain", req.headers.cookie ?? "");
      return;
    }
    if (url.pathname === "/localstorage-check") {
      send(res, 200, "text/html", "<!doctype html><script>fetch('/api?ls=' + encodeURIComponent(localStorage.getItem('session') || '')).catch(() => {});</script>");
      return;
    }
    if (url.pathname === "/cache-page") {
      send(res, 200, "text/html", "<!doctype html><img src=\"/cache.png\">");
      return;
    }
    if (url.pathname === "/fail-page") {
      send(res, 200, "text/html", "<!doctype html><img src=\"/fail.png\">");
      return;
    }
    if (url.pathname === "/fail.png") {
      req.socket.destroy();
      return;
    }
    if (url.pathname === "/cache.png") {
      if (req.headers["if-none-match"] === "\"cache-test\"") {
        res.writeHead(304, { etag: "\"cache-test\"", "cache-control": "no-cache" });
        res.end();
        return;
      }
      res.writeHead(200, {
        "content-type": "image/png",
        etag: "\"cache-test\"",
        "cache-control": "no-cache",
        "x-cache": "HIT",
      });
      res.end(PNG);
      return;
    }
    if (url.pathname === "/many") {
      const images = Array.from({ length: 40 }, (_, index) => `<img src="/image.png?n=${index}">`).join("");
      send(res, 200, "text/html", `<!doctype html>${images}<script>fetch('/api?many=1')</script>`);
      return;
    }
    send(res, 404, "text/plain", "not found");
  });
  await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
  const address = server.address() as { port: number };
  return {
    base: `http://127.0.0.1:${address.port}`,
    hits,
    close: async () => {
      server.close();
      await once(server, "close").catch(() => undefined);
    },
  };
}

function createConfig(profileDir: string): AppConfig {
  return {
    host: "127.0.0.1",
    port: 0,
    profileDir,
    headless: true,
    concurrency: 4,
    timeoutMs: 30_000,
    maxBodyBytes: 64 * 1024 * 1024,
    historyLimit: 1000,
    websocketHeartbeatMs: 1_000,
    networkHistoryLimit: 5000,
    networkBodyCaptureBytes: 262_144,
  };
}

async function postFetchText(gatewayBase: string, url: string, sourceId: string): Promise<string> {
  const response = await fetch(`${gatewayBase}/v1/fetch`, {
    method: "POST",
    headers: { "content-type": "application/json" },
    body: JSON.stringify({ source_id: sourceId, url, timeout_ms: 30000 }),
  });
  return response.text();
}

async function postJson<T = unknown>(url: string, body: unknown): Promise<T> {
  const response = await fetch(url, {
    method: "POST",
    headers: { "content-type": "application/json" },
    body: JSON.stringify(body),
  });
  if (!response.ok) {
    throw new Error(`${response.status}: ${await response.text()}`);
  }
  return response.json() as Promise<T>;
}

async function fetchJson<T>(url: string): Promise<T> {
  const response = await fetch(url);
  if (!response.ok) {
    throw new Error(`${response.status}: ${await response.text()}`);
  }
  return response.json() as Promise<T>;
}

async function waitFor(predicate: () => boolean, timeoutMs = 20_000): Promise<void> {
  const deadline = Date.now() + timeoutMs;
  while (Date.now() < deadline) {
    if (predicate()) {
      return;
    }
    await new Promise((resolve) => setTimeout(resolve, 100));
  }
  throw new Error("waitFor timeout");
}

function hasPaths(records: NetworkRecord[], paths: string[]): boolean {
  return paths.every((expected) => records.some((record) => {
    try {
      const url = new URL(record.url);
      return `${url.pathname}${url.search}` === expected;
    } catch {
      return false;
    }
  }));
}

function send(res: http.ServerResponse, status: number, contentType: string, body: string): void {
  res.writeHead(status, { "content-type": contentType });
  res.end(body);
}

function sendBuffer(res: http.ServerResponse, status: number, contentType: string, body: Buffer): void {
  res.writeHead(status, { "content-type": contentType });
  res.end(body);
}