tools/run-webgpu-benchmark.mjs

#!/usr/bin/env node
import { spawn } from "node:child_process";
import { existsSync } from "node:fs";
import { mkdir, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { dirname, resolve } from "node:path";
import { sourceFingerprint } from "./source-fingerprint.mjs";

const url = process.env.BROWSER_SPEAK_URL ?? "http://127.0.0.1:5174/";
const chrome =
  process.env.CHROME_BIN ??
  (existsSync("/opt/google/chrome/chrome")
    ? "/opt/google/chrome/chrome"
    : existsSync("/usr/bin/google-chrome")
      ? "/usr/bin/google-chrome"
      : "chromium");
const resultPath = resolve(process.env.BROWSER_SPEAK_WEBGPU_JSON ?? `${tmpdir()}/browser-speak-webgpu-results.json`);
const profileDir = resolve(process.env.BROWSER_SPEAK_WEBGPU_PROFILE_DIR ?? `${tmpdir()}/browser-speak-webgpu-profile`);
const llmCandidates = parseList(
  process.env.BROWSER_SPEAK_WEBGPU_LLMS ??
    "HuggingFaceTB/SmolLM2-135M-Instruct,onnx-community/granite-4.0-350m-ONNX-web,onnx-community/Qwen3-0.6B-ONNX",
);
const stackBase = {
  device: "webgpu",
  asr: process.env.BROWSER_SPEAK_ASR ?? "onnx-community/moonshine-base-ONNX",
  voice: process.env.BROWSER_SPEAK_VOICE ?? "F2",
  ttsSteps: Number(process.env.BROWSER_SPEAK_TTS_STEPS ?? 2),
  vadSilenceMs: Number(process.env.BROWSER_SPEAK_VAD_SILENCE_MS ?? 480),
  partialAsr: process.env.BROWSER_SPEAK_PARTIAL_ASR !== "false",
};
const loadTimeoutMs = Number(process.env.BROWSER_SPEAK_LOAD_TIMEOUT_MS ?? 600000);
const suiteTimeoutMs = Number(process.env.BROWSER_SPEAK_SUITE_TIMEOUT_MS ?? 900000);
const headless = process.env.BROWSER_SPEAK_HEADLESS !== "false";
const allowSoftwareWebgpu = process.env.BROWSER_SPEAK_ALLOW_SOFTWARE_WEBGPU === "true";
const protocolTimeoutMs = Number(process.env.BROWSER_SPEAK_CDP_TIMEOUT_MS ?? 60000);
const pollTimeoutMs = Number(process.env.BROWSER_SPEAK_CDP_POLL_TIMEOUT_MS ?? 5000);

async function main() {
  await ensureServer();
  await mkdir(dirname(resultPath), { recursive: true });
  await rm(profileDir, { recursive: true, force: true });

  const browser = launchBrowser(9343, profileDir);
  const candidates = llmCandidates.map((llm) => ({ llm, status: "pending" }));
  try {
    const client = await connectToPage(9343, browser);
    await waitForBenchApi(client);
    const webgpu = await runPageTask(client, "window.browserSpeakBench.webgpuInfo()", {
      label: "WebGPU probe",
      timeoutMs: 30000,
    });

    const skipReason = webgpuSkipReason(webgpu);
    if (skipReason) {
      for (const candidate of candidates) {
        candidate.status = "skipped";
        candidate.reason = skipReason;
      }
      await writePayload({
        skipped: true,
        reason: skipReason,
        webgpu,
        config: benchmarkConfig(),
        candidates,
        results: [],
        summary: null,
      });
      console.log(`Wrote WebGPU benchmark JSON: ${resultPath}`);
      console.log(`Skipped WebGPU benchmark: ${skipReason}.`);
      await client.closeBrowser();
      return;
    }

    await runPageTask(client, "window.browserSpeakBench.clearResults()", { label: "clear results" });
    console.log(`WebGPU adapter: ${formatAdapter(webgpu.adapter)}`);

    for (const candidate of candidates) {
      const stack = { ...stackBase, llm: candidate.llm };
      const label = shortModelName(candidate.llm);
      console.log(`Running WebGPU benchmark candidate: ${candidate.llm}`);
      const startedAt = Date.now();
      try {
        const before = await runPageTask(client, "window.browserSpeakBench.state()", {
          label: `${label} preflight`,
        });
        await runPageTask(client, `window.browserSpeakBench.loadStack(${JSON.stringify(stack)})`, {
          label: `${label} model load`,
          timeoutMs: loadTimeoutMs,
        });
        const snapshot = await runPageTask(client, "window.browserSpeakBench.runSuite()", {
          label: `${label} suite`,
          timeoutMs: suiteTimeoutMs,
        });
        const addedCount = Math.max(0, snapshot.results.length - before.results.length);
        const rows = snapshot.results.slice(0, addedCount);
        const errors = rows.filter((row) => row.error);
        candidate.status = rows.length >= 5 && errors.length === 0 ? "complete" : "partial";
        candidate.durationMs = Date.now() - startedAt;
        candidate.rowCount = rows.length;
        candidate.errorCount = errors.length;
        candidate.summary = snapshot.summary.current ?? snapshot.summary.all;
        candidate.stack = snapshot.stack;
        if (errors.length > 0) candidate.errors = errors.map((row) => `${row.kind}: ${row.error}`);
      } catch (error) {
        candidate.status = "failed";
        candidate.durationMs = Date.now() - startedAt;
        candidate.error = error.message;
        console.error(`${candidate.llm} failed: ${error.message}`);
      } finally {
        await runPageTask(client, "window.browserSpeakBench.stop()", {
          label: `${label} stop`,
          timeoutMs: 15000,
        }).catch(() => {});
        const state = await runPageTask(client, "window.browserSpeakBench.state()", {
          label: `${label} state`,
          timeoutMs: 15000,
        }).catch(() => null);
        if (state?.modelsLoaded || state?.modelsLoading) {
          await runPageTask(client, "window.browserSpeakBench.unload()", {
            label: `${label} unload`,
            timeoutMs: 60000,
          }).catch((error) => {
            candidate.unloadError = error.message;
          });
        }
        await writePayload(await exportPayload(client, webgpu, candidates));
      }
    }

    const payload = await exportPayload(client, webgpu, candidates);
    await writePayload(payload);
    console.log(`Wrote WebGPU benchmark JSON: ${resultPath}`);
    summarizeResults(payload);
    await client.closeBrowser();
  } finally {
    await stopBrowser(browser, profileDir);
  }
}

async function exportPayload(client, webgpu, candidates) {
  const exportResult = await runPageTask(client, "window.browserSpeakBench.exportResults()", {
    label: "export results",
    timeoutMs: 30000,
  }).catch(() => ({ summary: null, results: [] }));
  return {
    generatedAt: new Date().toISOString(),
    sourceFingerprint: await sourceFingerprint(),
    url,
    webgpu,
    config: benchmarkConfig(),
    candidates,
    summary: exportResult.summary,
    results: exportResult.results,
  };
}

function benchmarkConfig() {
  return {
    stackBase,
    loadTimeoutMs,
    suiteTimeoutMs,
    headless,
    allowSoftwareWebgpu,
    protocolTimeoutMs,
    pollTimeoutMs,
    chrome,
    extraChromeArgs: parseChromeArgs(),
  };
}

async function writePayload(payload) {
  await writeFile(
    resultPath,
    `${JSON.stringify(
      {
        generatedAt: new Date().toISOString(),
        sourceFingerprint: await sourceFingerprint(),
        url,
        ...payload,
      },
      null,
      2,
    )}\n`,
  );
}

async function ensureServer() {
  const response = await fetch(url).catch((error) => {
    throw new Error(`Could not reach ${url}: ${error.message}`);
  });
  if (!response.ok) throw new Error(`${url} returned HTTP ${response.status}`);
}

async function waitForBenchApi(client) {
  const deadline = Date.now() + 15000;
  while (Date.now() < deadline) {
    try {
      if (await client.evaluate("Boolean(window.browserSpeakBench?.webgpuInfo)")) return;
    } catch {
      // The target may still be navigating and can destroy the execution context.
    }
    await sleep(100);
  }
  throw new Error("window.browserSpeakBench.webgpuInfo was not installed.");
}

async function runPageTask(client, expression, { label = "page task", timeoutMs = 30000 } = {}) {
  const taskId = `task_${Date.now()}_${Math.random().toString(16).slice(2)}`;
  await client.evaluate(`(() => {
    const taskId = ${JSON.stringify(taskId)};
    window.__browserSpeakHarnessTasks ||= {};
    window.__browserSpeakHarnessTasks[taskId] = { done: false, label: ${JSON.stringify(label)} };
    Promise.resolve(${expression})
      .then((value) => {
        window.__browserSpeakHarnessTasks[taskId] = { done: true, value };
      })
      .catch((error) => {
        window.__browserSpeakHarnessTasks[taskId] = {
          done: true,
          error: error?.stack || error?.message || String(error),
        };
      });
    return true;
  })()`);
  const deadline = Date.now() + timeoutMs;
  let lastEvents = [];
  let lastPollError = "";
  while (Date.now() < deadline) {
    let task = null;
    try {
      task = await client.evaluate(
        `window.__browserSpeakHarnessTasks?.[${JSON.stringify(taskId)}] ?? null`,
        pollTimeoutMs,
      );
      lastPollError = "";
    } catch (error) {
      const message = error.message ?? String(error);
      if (message !== lastPollError) {
        console.log(`${label}: waiting for page response (${message})`);
        lastPollError = message;
      }
      await sleep(500);
      continue;
    }
    if (task?.done) {
      await client
        .evaluate(`delete window.__browserSpeakHarnessTasks?.[${JSON.stringify(taskId)}]`, pollTimeoutMs)
        .catch(() => {});
      if (task.error) throw new Error(`${label} failed: ${task.error}`);
      return task.value;
    }
    const snapshot = await client
      .evaluate(`(() => {
        const state = window.browserSpeakBench?.state?.();
        return state ? {
          modelsLoaded: state.modelsLoaded,
          modelsLoading: state.modelsLoading,
          activeBenchmark: state.activeBenchmark?.kind ?? null,
          suiteRunning: state.suiteRunning,
          events: state.events?.slice(0, 3) ?? [],
        } : null;
      })()`, pollTimeoutMs)
      .catch(() => null);
    const events = snapshot?.events ?? [];
    if (events.join("\\n") !== lastEvents.join("\\n")) {
      lastEvents = events;
      if (events[0]) console.log(`${label}: ${events[0]}`);
    }
    await sleep(500);
  }
  throw new Error(`${label} timed out after ${(timeoutMs / 1000).toFixed(0)} seconds.`);
}

function launchBrowser(port, profileDir) {
  const child = spawn(
    chrome,
    [
      ...(headless ? ["--headless=new"] : []),
      "--no-sandbox",
      "--disable-dev-shm-usage",
      "--disable-background-networking",
      "--disable-extensions",
      "--no-default-browser-check",
      "--no-first-run",
      "--autoplay-policy=no-user-gesture-required",
      "--enable-unsafe-webgpu",
      "--ignore-gpu-blocklist",
      `--remote-debugging-port=${port}`,
      `--user-data-dir=${profileDir}`,
      ...parseChromeArgs(),
    ],
    { stdio: ["ignore", "pipe", "pipe"] },
  );
  child.browserLog = "";
  const appendLog = (chunk) => {
    child.browserLog = `${child.browserLog}${chunk}`;
    if (child.browserLog.length > 8000) child.browserLog = child.browserLog.slice(-8000);
  };
  child.stdout.on("data", appendLog);
  child.stderr.on("data", appendLog);
  return child;
}

async function stopBrowser(child, profileDir) {
  if (child.exitCode == null) child.kill("SIGTERM");
  await new Promise((resolve) => {
    child.once("exit", resolve);
    setTimeout(resolve, 3000);
  });
  if (child.exitCode == null) child.kill("SIGKILL");
  for (let attempt = 0; attempt < 5; attempt += 1) {
    try {
      await rm(profileDir, { recursive: true, force: true });
      return;
    } catch (error) {
      if (attempt === 4) {
        console.warn(`Could not remove ${profileDir}: ${error.message}`);
        return;
      }
      await sleep(500);
    }
  }
}

async function connectToPage(port, child) {
  const deadline = Date.now() + 60000;
  let lastError = null;
  while (Date.now() < deadline) {
    if (child.exitCode != null) {
      throw new Error(`Chrome exited before DevTools became available.\n${child.browserLog}`);
    }
    try {
      const version = await fetch(`http://127.0.0.1:${port}/json/version`).then((response) => response.json());
      if (version.webSocketDebuggerUrl) {
        const page = await createPageTarget(port);
        return new CdpClient(page.webSocketDebuggerUrl);
      }
    } catch (error) {
      lastError = error;
    }
    await sleep(250);
  }
  throw new Error(
    `Could not connect to Chrome DevTools on port ${port}: ${lastError?.message ?? "unknown error"}\n${child.browserLog}`,
  );
}

async function createPageTarget(port) {
  for (const method of ["PUT", "GET"]) {
    const response = await fetch(`http://127.0.0.1:${port}/json/new?${encodeURIComponent(url)}`, {
      method,
    }).catch(() => null);
    if (response?.ok) {
      const target = await response.json();
      if (target.webSocketDebuggerUrl) return target;
    }
  }
  const targets = await fetch(`http://127.0.0.1:${port}/json`).then((response) => response.json());
  const page = targets.find((target) => target.type === "page" && target.url === url);
  if (page?.webSocketDebuggerUrl) return page;
  throw new Error("Could not create or find a page target.");
}

class CdpClient {
  constructor(webSocketUrl) {
    this.nextId = 1;
    this.pending = new Map();
    this.socket = new WebSocket(webSocketUrl);
    this.opened = new Promise((resolve, reject) => {
      this.socket.onopen = resolve;
      this.socket.onerror = reject;
      this.socket.onmessage = (event) => this.onMessage(event);
    });
  }

  onMessage(event) {
    const message = JSON.parse(event.data);
    if (!message.id || !this.pending.has(message.id)) return;
    const { resolve: onResolve, reject } = this.pending.get(message.id);
    this.pending.delete(message.id);
    if (message.error) reject(new Error(message.error.message));
    else onResolve(message.result);
  }

  async call(method, params = {}, timeoutMs = protocolTimeoutMs) {
    await this.opened;
    const id = this.nextId++;
    this.socket.send(JSON.stringify({ id, method, params }));
    return new Promise((resolvePromise, reject) => {
      const timer = setTimeout(() => {
        this.pending.delete(id);
        reject(new Error(`${method} timed out after ${(timeoutMs / 1000).toFixed(0)} seconds.`));
      }, timeoutMs);
      this.pending.set(id, {
        resolve: (value) => {
          clearTimeout(timer);
          resolvePromise(value);
        },
        reject: (error) => {
          clearTimeout(timer);
          reject(error);
        },
      });
    });
  }

  async evaluate(expression, timeoutMs = protocolTimeoutMs) {
    const result = await this.call("Runtime.evaluate", {
      expression,
      returnByValue: true,
    }, timeoutMs);
    if (result.exceptionDetails) throw new Error(formatException(result.exceptionDetails));
    return result.result.value;
  }

  closeBrowser() {
    try {
      this.socket.send(JSON.stringify({ id: this.nextId++, method: "Browser.close", params: {} }));
    } catch {
      // The surrounding process cleanup handles already-closed targets.
    }
    try {
      this.socket.close();
    } catch {
      // Ignore close races.
    }
  }
}

function formatException(exceptionDetails) {
  const exception = exceptionDetails.exception;
  return exception?.description ?? exception?.value ?? exceptionDetails.text ?? "Evaluation failed.";
}

function parseList(value) {
  return value
    .split(",")
    .map((item) => item.trim())
    .filter(Boolean);
}

function parseChromeArgs() {
  const raw = process.env.BROWSER_SPEAK_CHROME_ARGS ?? "";
  return (
    raw
      .match(/(?:[^\s"']+|"[^"]*"|'[^']*')+/g)
      ?.map((arg) => arg.replace(/^["']|["']$/g, "")) ?? []
  );
}

function shortModelName(model) {
  return model.split("/").at(-1) ?? model;
}

function formatAdapter(adapter) {
  if (!adapter) return "unknown";
  return [adapter.vendor, adapter.architecture, adapter.device, adapter.description].filter(Boolean).join(" / ") || "unknown";
}

function webgpuSkipReason(webgpu) {
  if (!webgpu?.available) return "WebGPU unavailable";
  if (allowSoftwareWebgpu) return "";
  if (webgpu.softwareAdapter || isSoftwareAdapter(webgpu.adapter)) {
    return `Software WebGPU adapter exposed (${formatAdapter(webgpu.adapter)})`;
  }
  return "";
}

function isSoftwareAdapter(adapter) {
  const text = [adapter?.vendor, adapter?.architecture, adapter?.device, adapter?.description]
    .filter(Boolean)
    .join(" ")
    .toLowerCase();
  return /\b(swiftshader|llvmpipe|software rasterizer|software adapter|warp)\b/.test(text);
}

function summarizeResults(payload) {
  for (const candidate of payload.candidates) {
    const suffix = candidate.error ? ` (${candidate.error})` : "";
    console.log(`${candidate.llm}: ${candidate.status}, ${candidate.rowCount ?? 0} rows${suffix}`);
  }
  const byStack = payload.summary?.byStack ?? {};
  const stackCount = Object.keys(byStack).length;
  console.log(`Exported ${payload.results.length} rows across ${stackCount} stack summaries.`);
}

function sleep(ms) {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

await main();