e2e/perf/reporter.ts

Linked repository file from this build.

import type {} from "./probe";
import { mkdir, writeFile } from "node:fs/promises";
import type { Reporter, TestCase, TestResult } from "@playwright/test/reporter";

type Sample = {
  component: string;
  mode: string;
  profile: { name: string };
  componentMs: number | null;
  wireBytes: number;
  apiJsonBytes: number;
  smoothness?: ReturnType<NonNullable<Window["__qcPerf"]>["summary"]>;
  [key: string]: unknown;
};
/** Keep diagnostics attached after measurement (including failed budget assertions). */
export function performanceSample(
  result: Pick<TestResult, "attachments" | "status" | "errors">,
): Sample | null {
  const body = result.attachments.find((attachment) => attachment.name === "performance")?.body;
  if (!body) return null;
  const jsonAttachment = (name: string) =>
    JSON.parse(
      result.attachments.find((attachment) => attachment.name === name)?.body?.toString() ?? "null",
    );
  return {
    ...JSON.parse(body.toString()),
    status: result.status,
    dataset: jsonAttachment("dataset"),
    dataDelivery: jsonAttachment("data-delivery"),
    navigationSetup: jsonAttachment("navigation-setup"),
    errors: result.errors.map((error) => error.message),
  };
}

export default class PerformanceReporter implements Reporter {
  private samples: Sample[] = [];
  private failures: string[] = [];
  onTestEnd(test: TestCase, result: TestResult) {
    const sample = performanceSample(result);
    if (sample) {
      // A continuous journey attaches one measurement for each navigation.
      for (const attachment of result.attachments.filter((a) => a.name === "performance")) {
        const measurement = performanceSample({
          ...result,
          attachments: [attachment, ...result.attachments.filter((a) => a.name !== "performance")],
        });
        if (measurement) this.samples.push(measurement);
      }
    }
    if (result.status !== "passed") this.failures.push(test.titlePath().join(" › "));
  }
  async onEnd() {
    const groups = Map.groupBy(
      this.samples.filter((s) => s.componentMs != null && s.component !== "instrumentation"),
      (s) => `${s.profile.name} / ${s.mode} / ${s.component}`,
    );
    const rows = [...groups].map(([name, samples]) => {
      const times = samples.map((s) => s.componentMs!).sort((a, b) => a - b);
      const percentile = (p: number) => Math.round(times[Math.ceil(times.length * p) - 1]);
      return {
        name,
        n: times.length,
        p50: percentile(0.5),
        p95: percentile(0.95),
        max: Math.round(times.at(-1)!),
        maxWireBytes: Math.max(...samples.map((s) => s.wireBytes)),
        maxApiJsonBytes: Math.max(...samples.map((s) => s.apiJsonBytes)),
      };
    });
    const directory = "test-results/performance-summary";
    await mkdir(directory, { recursive: true });
    await writeFile(
      `${directory}/results.json`,
      JSON.stringify(
        {
          generatedAt: new Date().toISOString(),
          rows,
          failures: this.failures,
          samples: this.samples,
        },
        null,
        2,
      ),
    );
    const escape = (s: string) =>
      s.replace(
        /[&<>"']/g,
        (c) => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&#39;" })[c]!,
      );
    const diagnostics = this.samples
      .filter((s) => s.component !== "instrumentation")
      .map((s) => {
        const d = s.smoothness;
        if (!d) return "";
        const ms = (n: number | null) => (n == null ? "—" : Math.round(n));
        return `<details><summary>${escape(`${s.profile.name} / ${s.mode} / ${s.component}`)}: fully visible ${ms(d.visualReadyMs)} ms; settled ${ms(d.settledMs)} ms; ${d.visibilityRegressions} visibility losses (${d.readinessRegressions} readiness losses); ${d.framesOver50Ms} frames &gt;50 ms</summary><p>Frame p95/max: ${ms(d.frameP95Ms)} / ${ms(d.maxFrameMs)} ms. Long-task blocking: ${ms(d.longTaskBlockingMs)} ms. Layout shift sum: ${d.layoutShiftSum.toFixed(4)}; input-associated: ${d.inputLayoutShiftSum.toFixed(4)}.</p><p>Frame-sampled states; IDs identify DOM remounts. Settled means the latest fully visible run through the observation end, not an unlimited stability guarantee.</p><pre>${escape(JSON.stringify({ timeline: d.timeline, animations: d.animations, frameGaps: d.frameGaps, longTasks: d.longTasks, longAnimationFrames: d.longAnimationFrames, shifts: d.shifts }, null, 2))}</pre></details>`;
      })
      .join("");
    await writeFile(
      `${directory}/index.html`,
      `<!doctype html><html lang="en"><meta charset="utf-8"><title>QC component performance</title><style>body{font:16px system-ui;margin:3rem;max-width:1200px;color:#182332}table{border-collapse:collapse;width:100%}th,td{padding:12px;text-align:right;border-bottom:1px solid #ddd}th:first-child,td:first-child{text-align:left}h1{font-size:28px}p{line-height:1.6}.bad{color:#a21}</style><h1>QC component performance</h1><p>Data-backed content visible after a rendering opportunity, in milliseconds. Cold = fresh browser context. Navigation = real UI action with session and source page loaded. Internet = simulated 80 ms latency, 10 Mbps down / 2 Mbps up; local = unthrottled. Remote runs use the actual connection without added throttling. Cache mode is recorded per sample in the raw measurements. Initial server data is included in HTML/RSC wire bytes, not API JSON bytes. API bytes are decoded; wire bytes include completed requests through 1 second after readiness. See JSON for unfinished requests and individual endpoints.</p><p>p95 is descriptive, not a reliable population estimate at small sample counts. No retries or discarded slow samples. Login, server cold starts and mobile CPU are not represented by local runs.</p><table><tr><th>Profile / action / component</th><th>n</th><th>p50 ms</th><th>p95 ms</th><th>max ms</th><th>max wire bytes</th><th>max API bytes</th></tr>${rows.map((r) => `<tr><td>${escape(r.name)}</td><td>${r.n}</td><td>${r.p50}</td><td>${r.p95}</td><td>${r.max}</td><td>${r.maxWireBytes}</td><td>${r.maxApiJsonBytes}</td></tr>`).join("")}</table><p class="bad">${this.failures.length} failed checks/tests. ${this.failures.map(escape).join("<br>")}</p>${diagnostics}<p><a href="results.json">Raw measurements and request breakdown</a></p></html>`,
    );
  }
}