🤖 ci: fail PRs that grow first-load JS over the recorded budget

.github/workflows/pr.yml

@@ -82,6 +82,9 @@ jobs:
               - 'bun.lock'
               - 'tsconfig*.json'
               - 'vite*.ts'
+              # index.html defines the first-load roots, so the first-load budget check
+              # (Smoke / Server) must run when it changes.
+              - 'index.html'
               - 'Makefile'
               - 'electron-builder.yml'
               # Pixel's default matrix and the Storybook config decide which snapshots exist,
@@ -470,6 +473,8 @@ jobs:
       - uses: ./.github/actions/setup-xum
       - run: ./scripts/generate-npm-shrinkwrap.sh
       - run: make build
+      # First-load JS budget (T3, #5971): reuses this build, so it adds no build.
+      - run: make check-first-load-js
       - run: make check-startup-imports-runtime
       - name: Pack npm package
         id: pack

Makefile

@@ -97,7 +97,7 @@ include fmt.mk
 .PHONY: vscode-ext vscode-ext-install
 .PHONY: docs-server check-docs-links
 .PHONY: storybook storybook-run storybook-build storybook-flake-check test-storybook storybook-budget
-.PHONY: benchmark-terminal bench bench-compare first-load-js bug-bash mcp-apps-e2e
+.PHONY: benchmark-terminal bench bench-compare first-load-js bug-bash mcp-apps-e2e check-first-load-js
 .PHONY: ensure-deps mux
 .PHONY: check-startup-imports check-startup-imports-runtime check-react-compiler check-test-routing check-test-seam-comments test-bench-scripts
 
@@ -771,6 +771,9 @@ bench-compare: node_modules/.installed src/version.ts ## Compare *.bench.ts resu
 first-load-js: ## Report first-load JS bytes of dist/ (run make build-renderer first)
 	@bun scripts/perf/firstLoadJs.ts dist
 
+check-first-load-js: ## Fail when first-load JS of dist/ is over scripts/perf/firstLoadBudget.json or loads a must-stay-lazy module (run make build-renderer first)
+	@bun scripts/perf/firstLoadJs.ts dist --budget scripts/perf/firstLoadBudget.json
+
 benchmark-terminal: ## Run Terminal-Bench 2.0 with Harbor (use TB_HARBOR_PACKAGE/TB_HARBOR_DAYTONA_PACKAGE/TB_DATASET/TB_CONCURRENCY/TB_TIMEOUT/TB_ENV/TB_MODEL/TB_ARGS to customize)
 	@# Pin Harbor with the Daytona extra so scheduled ingestion does not break on future CLI or adapter API drift.
 	@# Force the Daytona SDK to the cursor-pagination API while keeping Harbor stable.

scripts/perf/firstLoadBudget.json

@@ -0,0 +1,11 @@
+{
+  "comment": [
+    "First-load budget for `make check-first-load-js` (scripts/perf/firstLoadJs.ts --budget), run in CI after `make build`. The totals are index.html plus its statically reachable module JS.",
+    "The check fails when first-load brotli bytes grow more than 2% over brBytes, or raw bytes more than 100 KiB over rawBytes.",
+    "A smaller first load never fails, so a PR that shrinks it needs no update here.",
+    "If a PR grows the first load on purpose, the perf owner updates the recorded value in that same PR.",
+    "Recorded from a clean `make build` of main ea19a44101 with Bun 1.3.12."
+  ],
+  "rawBytes": 5134092,
+  "brBytes": 1219205
+}

scripts/perf/firstLoadJs.test.ts

@@ -32,13 +32,13 @@ describe("firstLoadJs", () => {
     // The script only stats sibling sizes, so the content need not be valid brotli.
     await fs.writeFile(path.join(dir, "main-AAAAAAAA.js.br"), "x".repeat(7));
     const size = async (file: string) => (await fs.stat(path.join(dir, file))).size;
-    const [main, shared, deep] = await Promise.all(
-      ["main-AAAAAAAA.js", "shared-BBBBBBBB.js", "deep-EEEEEEEE.js"].map(size)
+    const [main, shared, deep, html] = await Promise.all(
+      ["main-AAAAAAAA.js", "shared-BBBBBBBB.js", "deep-EEEEEEEE.js", "index.html"].map(size)
     );
     const { exitCode, stdout, stderr } = await runScript([dir, "--json"]);
     expect(exitCode, stderr).toBe(0);
     const report = JSON.parse(stdout) as { files: unknown[]; totals: unknown };
-    // Sorted by raw size. Only shared's `export * from` reaches the unpreloaded deep chunk. The
+    // Sorted by raw size; index.html itself counts. Only shared's `export * from` reaches the unpreloaded deep chunk. The
     // `import()` and __vite__mapDeps string keep the lazy chunk out; the stylesheet is not JS.
     const raw = (file: string, bytes: number) => ({
       file,
@@ -47,16 +47,19 @@ describe("firstLoadJs", () => {
       gzipBytes: bytes,
       precompressed: { br: false, gzip: false },
     });
-    expect(report.files).toEqual([
-      { ...raw("main-AAAAAAAA.js", main), brBytes: 7, precompressed: { br: true, gzip: false } },
-      raw("shared-BBBBBBBB.js", shared),
-      raw("deep-EEEEEEEE.js", deep),
-    ]);
+    expect(report.files).toEqual(
+      [
+        { ...raw("main-AAAAAAAA.js", main), brBytes: 7, precompressed: { br: true, gzip: false } },
+        raw("shared-BBBBBBBB.js", shared),
+        raw("deep-EEEEEEEE.js", deep),
+        raw("index.html", html),
+      ].sort((a, b) => b.rawBytes - a.rawBytes || a.file.localeCompare(b.file))
+    );
     expect(report.totals).toEqual({
-      files: 3,
-      rawBytes: main + shared + deep,
-      brBytes: 7 + shared + deep,
-      gzipBytes: main + shared + deep,
+      files: 4,
+      rawBytes: main + shared + deep + html,
+      brBytes: 7 + shared + deep + html,
+      gzipBytes: main + shared + deep + html,
     });
   });
 
@@ -74,4 +77,117 @@ describe("firstLoadJs", () => {
     expect(noMap.exitCode).toBe(2);
     expect(noMap.stderr).toContain("deep-EEEEEEEE.js has no source map");
   });
+
+  test("a classic external script exits 2, because the graph does not measure it", async () => {
+    const html = path.join(dir, "index.html");
+    await fs.writeFile(path.join(dir, "classic.js"), "window.__classic = true;");
+    const original = await fs.readFile(html, "utf-8");
+    const classicTag = '<script src="./classic.js"></script>';
+    await fs.writeFile(html, original.replace("</head>", `${classicTag}</head>`));
+    const classic = await runScript([dir]);
+    expect(classic.exitCode).toBe(2);
+    expect(classic.stderr).toContain("./classic.js");
+    await fs.writeFile(html, original);
+  });
+
+  describe("--budget", () => {
+    const BUDGET_FILE = path.join(import.meta.dir, "firstLoadBudget.json");
+
+    async function totals() {
+      const { exitCode, stdout, stderr } = await runScript([dir, "--json"]);
+      expect(exitCode, stderr).toBe(0);
+      return (JSON.parse(stdout) as { totals: { rawBytes: number; brBytes: number } }).totals;
+    }
+
+    async function writeBudget(budget: unknown) {
+      const file = path.join(dir, "budget.json");
+      await fs.writeFile(file, typeof budget === "string" ? budget : JSON.stringify(budget));
+      return file;
+    }
+
+    test("passes at or under the recorded values and fails on brotli growth over 2%", async () => {
+      const { rawBytes, brBytes } = await totals();
+      const exact = await runScript([dir, "--budget", await writeBudget({ rawBytes, brBytes })]);
+      expect(exact.exitCode, exact.stderr).toBe(0);
+      // A smaller first load never fails, so shrinking needs no budget update.
+      const shrunk = { rawBytes: rawBytes * 2, brBytes: brBytes * 2 };
+      expect((await runScript([dir, "--budget", await writeBudget(shrunk)])).exitCode).toBe(0);
+      // The committed budget file must stay valid; this fixture is far under it.
+      const committed = await runScript([dir, "--budget", BUDGET_FILE]);
+      expect(committed.exitCode, committed.stderr).toBe(0);
+
+      const budgetFile = await writeBudget({ rawBytes, brBytes: Math.floor(brBytes / 1.03) });
+      const grown = await runScript([dir, "--budget", budgetFile]);
+      expect(grown.exitCode).toBe(1);
+      expect(grown.stderr).toContain(budgetFile);
+      // The message names the new value, the budget key and the budget file.
+      expect(grown.stderr).toContain(String(brBytes));
+      expect(grown.stderr).toContain("brBytes");
+      expect(grown.stderr).not.toContain("rawBytes");
+    });
+
+    test("fails when raw grows more than 100 KiB, even with brotli unchanged", async () => {
+      // main has a .br sibling, so padding it grows raw bytes only.
+      await fs.writeFile(path.join(dir, "main-AAAAAAAA.js.br"), "x".repeat(7));
+      const before = await totals();
+      const padded = 100 * 1024 + 1;
+      await fs.appendFile(path.join(dir, "main-AAAAAAAA.js"), `\n/*${"x".repeat(padded - 5)}*/`);
+      const after = await totals();
+      expect(after.rawBytes - before.rawBytes).toBe(padded);
+
+      const recorded = { rawBytes: before.rawBytes, brBytes: after.brBytes };
+      const grown = await runScript([dir, "--budget", await writeBudget(recorded)]);
+      expect(grown.exitCode).toBe(1);
+      expect(grown.stderr).toContain(String(after.rawBytes));
+      expect(grown.stderr).toContain("rawBytes");
+      expect(grown.stderr).not.toContain("brBytes");
+      // Exactly 100 KiB of growth is still within budget.
+      const edge = { rawBytes: after.rawBytes - 100 * 1024, brBytes: after.brBytes };
+      expect((await runScript([dir, "--budget", await writeBudget(edge)])).exitCode).toBe(0);
+    });
+
+    test("fails when the inline boot script in index.html grows more than 100 KiB", async () => {
+      // index.html counts as one entry, so inline JS (and other markup) growth is measured.
+      const recorded = await totals();
+      const html = path.join(dir, "index.html");
+      const grownScript = `window.__boot = "${"x".repeat(101 * 1024)}";`;
+      await fs.writeFile(
+        html,
+        (await fs.readFile(html, "utf-8")).replace("window.__boot = true;", grownScript)
+      );
+      const grown = await runScript([dir, "--budget", await writeBudget(recorded)]);
+      expect(grown.exitCode).toBe(1);
+      expect(grown.stderr).toContain("rawBytes");
+    });
+
+    test("a forbidden module fails even when the bytes are under budget", async () => {
+      const budgetFile = await writeBudget(await totals());
+      const result = await runScript([
+        dir,
+        "--budget",
+        budgetFile,
+        "--forbid",
+        "node_modules/shared-dep/",
+      ]);
+      expect(result.exitCode).toBe(1);
+      expect(result.stderr).toContain("forbidden on first load");
+    });
+
+    test("an unusable budget file exits 2", async () => {
+      const { rawBytes, brBytes } = await totals();
+      for (const budget of [
+        "{ not json",
+        "null",
+        { rawBytes },
+        { rawBytes, brBytes: -1 },
+        { rawBytes: "1000", brBytes },
+        { rawBytes: 1.5, brBytes },
+      ]) {
+        const result = await runScript([dir, "--budget", await writeBudget(budget)]);
+        expect(result.exitCode, JSON.stringify(budget)).toBe(2);
+      }
+      expect((await runScript([dir, "--budget", path.join(dir, "missing.json")])).exitCode).toBe(2);
+      expect((await runScript([dir, "--budget"])).exitCode).toBe(2);
+    });
+  });
 });

scripts/perf/firstLoadJs.ts

@@ -8,7 +8,14 @@
  * `xum server` sends those bytes. It skips files under 1 KiB, which are then sent raw.
  *
  * Usage: bun scripts/perf/firstLoadJs.ts [distDir] [--json] [--forbid <substring>]...
- * Exit codes: 0 ok, 1 a forbidden source is on the first load, 2 unusable input.
+ *   [--budget <file>]
+ * The totals also count index.html itself (T3 PR8), so growth of its inline boot script or
+ * markup is measured. Before PR8 the totals were module JS only (9.1 KiB less on main).
+ * An external classic `<script src>` exits 2: the graph does not measure it (index.html has none).
+ * `--budget` (T3 PR8, `make check-first-load-js` in CI) also fails when the first load grows
+ * over the recorded values in that file: brotli by more than 2%, raw by more than 100 KiB.
+ * Exit codes: 0 ok, 1 a forbidden source is on the first load or a budget is exceeded,
+ * 2 unusable input.
  */
 import * as fs from "fs";
 import * as path from "path";
@@ -22,9 +29,28 @@ export const FIRST_LOAD_FORBIDDEN_SOURCES: readonly string[] = [
   "features/Settings/Sections/ProvidersSection",
   "node_modules/@shikijs/langs/",
   "node_modules/recharts/",
+  // Right-sidebar panels made lazy by PR7. Timeline and Artifacts stay eager: TimelineDialog
+  // and ArtifactsDialog import them statically.
   "features/desktop/DesktopPanel",
+  "components/InstructionsTab/InstructionsTab",
+  "components/OutputTab/OutputTab",
+  "features/RightSidebar/BrowserTab/BrowserTab",
+  "features/RightSidebar/DevToolsTab/DevToolsTab",
+  "features/RightSidebar/GoalTab",
+  "features/RightSidebar/Memory/MemoryTab",
+  "features/RightSidebar/Workflows/WorkflowsTab",
 ];
 
+// Growth allowed over the recorded budget before `--budget` fails. A smaller first load never
+// fails, so a PR that shrinks it needs no budget update.
+const BUDGET_BR_GROWTH = 0.02;
+const BUDGET_RAW_GROWTH_BYTES = 100 * 1024;
+
+interface FirstLoadBudget {
+  rawBytes: number;
+  brBytes: number;
+}
+
 function fail(message: string): never {
   console.error(`firstLoadJs: ${message}`);
   process.exit(2);
@@ -57,6 +83,9 @@ function htmlRoots(distDir: string): string[] {
       ? attrs.get("type") === "module"
       : attrs.get("rel")?.toLowerCase().split(/\s+/).includes("modulepreload") === true;
     const href = attrs.get(isScript ? "src" : "href");
+    if (isScript && !isRoot && href != null) {
+      fail(`index.html loads classic script ${href}, which first-load totals cannot measure`);
+    }
     if (!isRoot || href == null) continue; // inline boot scripts, stylesheets, icons
     // vite `base: "./"` writes `./main-<hash>.js`; a leading `/` means the dist root.
     const target = href.startsWith("/") ? path.join(distDir, href) : path.resolve(distDir, href);
@@ -101,6 +130,50 @@ function servedSize(file: string, extension: string, rawBytes: number): [number,
   return sibling?.isFile() ? [sibling.size, true] : [rawBytes, false];
 }
 
+function readBudget(file: string): FirstLoadBudget {
+  let budget: unknown;
+  try {
+    budget = JSON.parse(fs.readFileSync(file, "utf-8"));
+  } catch (error) {
+    fail(`cannot read budget ${file}: ${error instanceof Error ? error.message : String(error)}`);
+  }
+  const isBytes = (value: unknown) => Number.isSafeInteger(value) && (value as number) > 0;
+  if (
+    typeof budget !== "object" ||
+    budget === null ||
+    !isBytes((budget as Partial<FirstLoadBudget>).rawBytes) ||
+    !isBytes((budget as Partial<FirstLoadBudget>).brBytes)
+  ) {
+    fail(`${file} must be a JSON object with positive integer rawBytes and brBytes`);
+  }
+  const { rawBytes, brBytes } = budget as FirstLoadBudget;
+  return { rawBytes, brBytes };
+}
+
+/** Compares totals with the budget file; over-budget messages say how to update the budget. */
+function checkBudget(
+  totals: FirstLoadBudget,
+  file: string
+): Array<{ over: boolean; text: string }> {
+  const budget = readBudget(file);
+  const checks = [
+    ["brBytes", "brotli", Math.floor(budget.brBytes * (1 + BUDGET_BR_GROWTH)), "+2%"],
+    ["rawBytes", "raw", budget.rawBytes + BUDGET_RAW_GROWTH_BYTES, "+100 KiB"],
+  ] as const;
+  return checks.map(([key, name, limit, growth]) => {
+    const over = totals[key] > limit;
+    const limitText = `budget of ${limit} bytes (${key} ${budget[key]} in ${file}, ${growth})`;
+    return over
+      ? {
+          over,
+          text:
+            `first-load ${name} is ${totals[key]} bytes, over the ${limitText}. If this growth ` +
+            `is intended, the perf owner updates ${key} in ${file} in the PR that causes it.`,
+        }
+      : { over, text: `first-load ${name} ${totals[key]} bytes is within the ${limitText}` };
+  });
+}
+
 const KIB = new Intl.NumberFormat("en-US", { minimumFractionDigits: 1, maximumFractionDigits: 1 });
 
 /** A right-aligned table cell; `*` marks a file served raw for lack of a precompressed sibling. */
@@ -111,19 +184,22 @@ function kib(bytes: number, precompressed = true): string {
 function main(argv: string[]): void {
   let distArg: string | undefined;
   let json = false;
+  let budgetFile: string | undefined;
   const patterns = [...FIRST_LOAD_FORBIDDEN_SOURCES];
   for (let i = 0; i < argv.length; i++) {
     const arg = argv[i];
     if (arg === "--json") json = true;
     else if (arg === "--forbid" && /^[^-]/.test(argv[i + 1] ?? "")) patterns.push(argv[++i]);
+    else if (arg === "--budget" && /^[^-]/.test(argv[i + 1] ?? "")) budgetFile = argv[++i];
     else if (arg.startsWith("-") || distArg != null) fail(`bad or incomplete argument ${arg}`);
     else distArg = arg;
   }
   const distDir = path.resolve(distArg ?? "dist");
   const relative = (file: string) => path.relative(distDir, file).replaceAll("\\", "/");
 
   const code = firstLoadFiles(distDir);
-  const files = [...code.keys()].map((file) => {
+  // index.html is measured but not scanned for forbidden sources: it has no source map.
+  const files = [path.join(distDir, "index.html"), ...code.keys()].map((file) => {
     const rawBytes = fs.statSync(file).size;
     const [brBytes, br] = servedSize(file, ".br", rawBytes);
     const [gzipBytes, gzip] = servedSize(file, ".gz", rawBytes);
@@ -162,10 +238,15 @@ function main(argv: string[]): void {
     }
   }
 
+  const budgetChecks = budgetFile == null ? [] : checkBudget(totals, budgetFile);
   for (const v of violations) {
     console.error(`forbidden on first load: ${v.file} has ${v.source} (matches "${v.pattern}")`);
   }
-  if (violations.length > 0) process.exit(1);
+  for (const check of budgetChecks) {
+    if (check.over) console.error(check.text);
+    else if (!json) console.log(check.text);
+  }
+  if (violations.length > 0 || budgetChecks.some((c) => c.over)) process.exit(1);
 }
 
 if (import.meta.main) {