import test from "node:test"; import assert from "node:assert/strict"; import http from "node:http"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { once } from "node:events"; import { Fetcher } from "../src/fetcher.js"; import { NetworkHistory } from "../src/network_history.js"; import { RequestHistory } from "../src/request_history.js"; import { Semaphore } from "../src/semaphore.js"; import { buildServer } from "../src/server.js"; import { SourceSessionManager } from "../src/source_session_manager.js"; import type { AppConfig, NetworkRecord } from "../src/types.js"; import { WebSocketHub } from "../src/websocket_hub.js"; const PNG = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=", "base64"); test("real Chromium source sessions and network observer", { timeout: 120_000 }, async (t) => { const target = await startTargetServer(); const profileRoot = await fs.mkdtemp(path.join(os.tmpdir(), "cloakfetch-test-profile-")); const config = createConfig(profileRoot); const history = new RequestHistory(config.historyLimit); const networkHistory = new NetworkHistory(config.networkHistoryLimit); let fetcher: Fetcher; const hub = new WebSocketHub(config.websocketHeartbeatMs, () => ({ server_time: new Date().toISOString(), stats: fetcher.stats, })); const sources = new SourceSessionManager(config, networkHistory, hub); fetcher = new Fetcher(config, sources, new Semaphore(config.concurrency)); const gateway = await buildServer(config, fetcher, history, networkHistory, sources, hub); await gateway.listen({ host: "127.0.0.1", port: 0 }); const gatewayBase = `http://127.0.0.1:${(gateway.server.address() as { port: number }).port}`; t.after(async () => { await gateway.close(); await sources.closeAll(); await target.close(); if (path.resolve(profileRoot).startsWith(os.tmpdir())) { await fs.rm(profileRoot, { recursive: true, force: true }); } }); const ws = new WebSocket(gatewayBase.replace("http://", "ws://") + "/v1/ws"); const wsMessages: unknown[] = []; ws.onmessage = (event) => { wsMessages.push(JSON.parse(event.data)); }; await waitFor(() => wsMessages.some((message) => (message as { type?: string }).type === "hello")); await postJson(`${gatewayBase}/v1/sources`, { id: "s1", warmup_url: `${target.base}/set-cookie?value=a`, viewport: { width: 1280, height: 720 }, }); await postJson(`${gatewayBase}/v1/sources`, { id: "s2", warmup_url: `${target.base}/set-cookie?value=b`, }); const echo1 = await postFetchText(gatewayBase, `${target.base}/echo-cookie`, "s1"); const echo2 = await postFetchText(gatewayBase, `${target.base}/echo-cookie`, "s2"); assert.match(echo1, /session=a/); assert.match(echo2, /session=b/); await postJson(`${gatewayBase}/v1/diagnostics/fetch`, { source_id: "s1", url: `${target.base}/`, timeout_ms: 30000, }); await waitFor(() => hasPaths(networkHistory.recent(2000), ["/", "/script.js", "/style.css", "/image.png", "/frame", "/api?script=1"])); assert.ok(target.hits.some((hit) => hit.pathname === "/api" && hit.searchParams.get("script") === "1")); await postJson(`${gatewayBase}/v1/sources/s1/warmup`, { url: `${target.base}/localstorage-check`, timeout_ms: 30000, }); await waitFor(() => target.hits.some((hit) => hit.pathname === "/api" && hit.searchParams.get("ls") === "a")); await postJson(`${gatewayBase}/v1/sources/s2/warmup`, { url: `${target.base}/localstorage-check`, timeout_ms: 30000, }); await waitFor(() => target.hits.some((hit) => hit.pathname === "/api" && hit.searchParams.get("ls") === "b")); await postJson(`${gatewayBase}/v1/diagnostics/fetch`, { source_id: "s1", url: `${target.base}/redirect`, timeout_ms: 30000, }); await waitFor(() => networkHistory.recent(2000).some((record) => record.redirect_to?.endsWith("/image.png"))); await postJson(`${gatewayBase}/v1/diagnostics/fetch`, { source_id: "s1", url: `${target.base}/fail-page`, timeout_ms: 30000, }); await waitFor(() => networkHistory.recent(2000).some((record) => record.failure_text && record.url.endsWith("/fail.png"))); await postFetchText(gatewayBase, `${target.base}/cache-page`, "s1"); await postFetchText(gatewayBase, `${target.base}/cache-page`, "s1"); await waitFor(() => networkHistory.recent(2000).some((record) => record.url.endsWith("/cache.png") && record.from_cache)); await postJson(`${gatewayBase}/v1/diagnostics/fetch`, { source_id: "s1", url: `${target.base}/many`, timeout_ms: 30000, }); assert.ok(networkHistory.summary().retained_total > 20); await waitFor(() => wsMessages.some((message) => (message as { type?: string }).type === "network.request")); await waitFor(() => wsMessages.some((message) => (message as { type?: string }).type === "network.finished")); const imageResponse = await fetch(`${gatewayBase}/v1/fetch`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ source_id: "s1", url: `${target.base}/image.png`, timeout_ms: 30000 }), }); const imageBody = Buffer.from(await imageResponse.arrayBuffer()); assert.equal(imageResponse.status, 200); assert.equal(imageBody[0], 0x89); const pages = await fetchJson(`${gatewayBase}/v1/sources/s1/pages`); const cookies = await fetchJson(`${gatewayBase}/v1/sources/s1/cookies`); assert.ok(pages.length > 0); assert.ok(cookies.length > 0); await fetch(`${gatewayBase}/v1/sources/s2`, { method: "DELETE" }); const removed = await fetch(`${gatewayBase}/v1/sources/s2`); assert.equal(removed.status, 404); ws.close(); }); async function startTargetServer() { const hits: URL[] = []; const server = http.createServer((req, res) => { const url = new URL(req.url ?? "/", "http://127.0.0.1"); hits.push(url); if (url.pathname === "/") { send(res, 200, "text/html", `root`); return; } if (url.pathname === "/script.js") { send(res, 200, "application/javascript", `window.scriptLoaded = true; fetch('/api?script=1').catch(() => {});`); return; } if (url.pathname === "/style.css") { send(res, 200, "text/css", "body { color: rgb(1, 2, 3); }"); return; } if (url.pathname === "/image.png") { sendBuffer(res, 200, "image/png", PNG); return; } if (url.pathname === "/frame") { send(res, 200, "text/html", "frame

frame

"); return; } if (url.pathname === "/api") { send(res, 200, "application/json", JSON.stringify({ ok: true })); return; } if (url.pathname === "/redirect") { res.writeHead(302, { location: "/image.png" }); res.end(); return; } if (url.pathname === "/set-cookie") { const value = url.searchParams.get("value") ?? "x"; res.writeHead(200, { "content-type": "text/html", "set-cookie": `session=${value}; Path=/`, }); res.end(``); return; } if (url.pathname === "/echo-cookie") { send(res, 200, "text/plain", req.headers.cookie ?? ""); return; } if (url.pathname === "/localstorage-check") { send(res, 200, "text/html", ""); return; } if (url.pathname === "/cache-page") { send(res, 200, "text/html", ""); return; } if (url.pathname === "/fail-page") { send(res, 200, "text/html", ""); return; } if (url.pathname === "/fail.png") { req.socket.destroy(); return; } if (url.pathname === "/cache.png") { if (req.headers["if-none-match"] === "\"cache-test\"") { res.writeHead(304, { etag: "\"cache-test\"", "cache-control": "no-cache" }); res.end(); return; } res.writeHead(200, { "content-type": "image/png", etag: "\"cache-test\"", "cache-control": "no-cache", "x-cache": "HIT", }); res.end(PNG); return; } if (url.pathname === "/many") { const images = Array.from({ length: 40 }, (_, index) => ``).join(""); send(res, 200, "text/html", `${images}`); return; } send(res, 404, "text/plain", "not found"); }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); const address = server.address() as { port: number }; return { base: `http://127.0.0.1:${address.port}`, hits, close: async () => { server.close(); await once(server, "close").catch(() => undefined); }, }; } function createConfig(profileDir: string): AppConfig { return { host: "127.0.0.1", port: 0, profileDir, headless: true, concurrency: 4, timeoutMs: 30_000, maxBodyBytes: 64 * 1024 * 1024, historyLimit: 1000, websocketHeartbeatMs: 1_000, networkHistoryLimit: 5000, networkBodyCaptureBytes: 262_144, }; } async function postFetchText(gatewayBase: string, url: string, sourceId: string): Promise { const response = await fetch(`${gatewayBase}/v1/fetch`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ source_id: sourceId, url, timeout_ms: 30000 }), }); return response.text(); } async function postJson(url: string, body: unknown): Promise { const response = await fetch(url, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body), }); if (!response.ok) { throw new Error(`${response.status}: ${await response.text()}`); } return response.json() as Promise; } async function fetchJson(url: string): Promise { const response = await fetch(url); if (!response.ok) { throw new Error(`${response.status}: ${await response.text()}`); } return response.json() as Promise; } async function waitFor(predicate: () => boolean, timeoutMs = 20_000): Promise { const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { if (predicate()) { return; } await new Promise((resolve) => setTimeout(resolve, 100)); } throw new Error("waitFor timeout"); } function hasPaths(records: NetworkRecord[], paths: string[]): boolean { return paths.every((expected) => records.some((record) => { try { const url = new URL(record.url); return `${url.pathname}${url.search}` === expected; } catch { return false; } })); } function send(res: http.ServerResponse, status: number, contentType: string, body: string): void { res.writeHead(status, { "content-type": contentType }); res.end(body); } function sendBuffer(res: http.ServerResponse, status: number, contentType: string, body: Buffer): void { res.writeHead(status, { "content-type": contentType }); res.end(body); }