From 57a8990979002d7af25b6e07d4241ab7b2368913 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Sat, 10 Oct 2026 19:59:56 +0000 Subject: [PATCH 1/5] =?UTF-8?q?=F0=9F=A4=96=20tests:=20cover=20the=20first?= =?UTF-8?q?-load=20JS=20budget=20mode?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs #5971. Adds --budget tests to scripts/perf/firstLoadJs.test.ts: growth at or under the recorded values passes, brotli growth over 2% fails, raw growth over 100 KiB fails, a forbidden module fails even under budget, and an unusable budget file exits 2. They fail on the script before the budget mode exists. --- scripts/perf/firstLoadJs.test.ts | 87 ++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/scripts/perf/firstLoadJs.test.ts b/scripts/perf/firstLoadJs.test.ts index ec7d3173dc8..2073aeda8b2 100644 --- a/scripts/perf/firstLoadJs.test.ts +++ b/scripts/perf/firstLoadJs.test.ts @@ -74,4 +74,91 @@ describe("firstLoadJs", () => { expect(noMap.exitCode).toBe(2); expect(noMap.stderr).toContain("deep-EEEEEEEE.js has no source map"); }); + + describe("--budget", () => { + const BUDGET_FILE = path.join(import.meta.dir, "firstLoadBudget.json"); + + async function totals() { + const { exitCode, stdout, stderr } = await runScript([dir, "--json"]); + expect(exitCode, stderr).toBe(0); + return (JSON.parse(stdout) as { totals: { rawBytes: number; brBytes: number } }).totals; + } + + async function writeBudget(budget: unknown) { + const file = path.join(dir, "budget.json"); + await fs.writeFile(file, typeof budget === "string" ? budget : JSON.stringify(budget)); + return file; + } + + test("passes at or under the recorded values and fails on brotli growth over 2%", async () => { + const { rawBytes, brBytes } = await totals(); + const exact = await runScript([dir, "--budget", await writeBudget({ rawBytes, brBytes })]); + expect(exact.exitCode, exact.stderr).toBe(0); + // A smaller first load never fails, so shrinking needs no budget update. + const shrunk = { rawBytes: rawBytes * 2, brBytes: brBytes * 2 }; + expect((await runScript([dir, "--budget", await writeBudget(shrunk)])).exitCode).toBe(0); + // The committed budget file must stay valid; this fixture is far under it. + const committed = await runScript([dir, "--budget", BUDGET_FILE]); + expect(committed.exitCode, committed.stderr).toBe(0); + + const budgetFile = await writeBudget({ rawBytes, brBytes: Math.floor(brBytes / 1.03) }); + const grown = await runScript([dir, "--budget", budgetFile]); + expect(grown.exitCode).toBe(1); + expect(grown.stderr).toContain(budgetFile); + // The message names the new value, the budget key and the budget file. + expect(grown.stderr).toContain(String(brBytes)); + expect(grown.stderr).toContain("brBytes"); + expect(grown.stderr).not.toContain("rawBytes"); + }); + + test("fails when raw grows more than 100 KiB, even with brotli unchanged", async () => { + // main has a .br sibling, so padding it grows raw bytes only. + await fs.writeFile(path.join(dir, "main-AAAAAAAA.js.br"), "x".repeat(7)); + const before = await totals(); + const padded = 100 * 1024 + 1; + await fs.appendFile(path.join(dir, "main-AAAAAAAA.js"), `\n/*${"x".repeat(padded - 5)}*/`); + const after = await totals(); + expect(after.rawBytes - before.rawBytes).toBe(padded); + + const recorded = { rawBytes: before.rawBytes, brBytes: after.brBytes }; + const grown = await runScript([dir, "--budget", await writeBudget(recorded)]); + expect(grown.exitCode).toBe(1); + expect(grown.stderr).toContain(String(after.rawBytes)); + expect(grown.stderr).toContain("rawBytes"); + expect(grown.stderr).not.toContain("brBytes"); + // Exactly 100 KiB of growth is still within budget. + const edge = { rawBytes: after.rawBytes - 100 * 1024, brBytes: after.brBytes }; + expect((await runScript([dir, "--budget", await writeBudget(edge)])).exitCode).toBe(0); + }); + + test("a forbidden module fails even when the bytes are under budget", async () => { + const budgetFile = await writeBudget(await totals()); + const result = await runScript([ + dir, + "--budget", + budgetFile, + "--forbid", + "node_modules/shared-dep/", + ]); + expect(result.exitCode).toBe(1); + expect(result.stderr).toContain("forbidden on first load"); + }); + + test("an unusable budget file exits 2", async () => { + const { rawBytes, brBytes } = await totals(); + for (const budget of [ + "{ not json", + "null", + { rawBytes }, + { rawBytes, brBytes: -1 }, + { rawBytes: "1000", brBytes }, + { rawBytes: 1.5, brBytes }, + ]) { + const result = await runScript([dir, "--budget", await writeBudget(budget)]); + expect(result.exitCode, JSON.stringify(budget)).toBe(2); + } + expect((await runScript([dir, "--budget", path.join(dir, "missing.json")])).exitCode).toBe(2); + expect((await runScript([dir, "--budget"])).exitCode).toBe(2); + }); + }); }); From c265159054f6f1790436d18baab228af777b6d50 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Sat, 10 Oct 2026 19:59:57 +0000 Subject: [PATCH 2/5] =?UTF-8?q?=F0=9F=A4=96=20ci:=20fail=20PRs=20that=20gr?= =?UTF-8?q?ow=20first-load=20JS=20over=20the=20recorded=20budget?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs #5971. firstLoadJs.ts --budget fails when first-load brotli grows more than 2% or raw more than 100 KiB over scripts/perf/firstLoadBudget.json (recorded from a clean build of main ea19a44101). The must-stay-lazy list gains the seven lazy right-sidebar panels. make check-first-load-js runs it in Smoke / Server right after that job's make build, so CI adds no build. --- .github/workflows/pr.yml | 2 + Makefile | 5 +- scripts/perf/firstLoadBudget.json | 11 +++++ scripts/perf/firstLoadJs.ts | 78 ++++++++++++++++++++++++++++++- 4 files changed, 93 insertions(+), 3 deletions(-) create mode 100644 scripts/perf/firstLoadBudget.json diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 55192f60ac0..0444eb20f98 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -470,6 +470,8 @@ jobs: - uses: ./.github/actions/setup-xum - run: ./scripts/generate-npm-shrinkwrap.sh - run: make build + # First-load JS budget (T3, #5971): reuses this build, so it adds no build. + - run: make check-first-load-js - run: make check-startup-imports-runtime - name: Pack npm package id: pack diff --git a/Makefile b/Makefile index a7e02c2ce11..a3081f877e4 100644 --- a/Makefile +++ b/Makefile @@ -97,7 +97,7 @@ include fmt.mk .PHONY: vscode-ext vscode-ext-install .PHONY: docs-server check-docs-links .PHONY: storybook storybook-run storybook-build storybook-flake-check test-storybook storybook-budget -.PHONY: benchmark-terminal bench bench-compare first-load-js bug-bash mcp-apps-e2e +.PHONY: benchmark-terminal bench bench-compare first-load-js bug-bash mcp-apps-e2e check-first-load-js .PHONY: ensure-deps mux .PHONY: check-startup-imports check-startup-imports-runtime check-react-compiler check-test-routing check-test-seam-comments test-bench-scripts @@ -771,6 +771,9 @@ bench-compare: node_modules/.installed src/version.ts ## Compare *.bench.ts resu first-load-js: ## Report first-load JS bytes of dist/ (run make build-renderer first) @bun scripts/perf/firstLoadJs.ts dist +check-first-load-js: ## Fail when first-load JS of dist/ is over scripts/perf/firstLoadBudget.json or loads a must-stay-lazy module (run make build-renderer first) + @bun scripts/perf/firstLoadJs.ts dist --budget scripts/perf/firstLoadBudget.json + benchmark-terminal: ## Run Terminal-Bench 2.0 with Harbor (use TB_HARBOR_PACKAGE/TB_HARBOR_DAYTONA_PACKAGE/TB_DATASET/TB_CONCURRENCY/TB_TIMEOUT/TB_ENV/TB_MODEL/TB_ARGS to customize) @# Pin Harbor with the Daytona extra so scheduled ingestion does not break on future CLI or adapter API drift. @# Force the Daytona SDK to the cursor-pagination API while keeping Harbor stable. diff --git a/scripts/perf/firstLoadBudget.json b/scripts/perf/firstLoadBudget.json new file mode 100644 index 00000000000..d91b3a6eac3 --- /dev/null +++ b/scripts/perf/firstLoadBudget.json @@ -0,0 +1,11 @@ +{ + "comment": [ + "First-load JS budget for `make check-first-load-js` (scripts/perf/firstLoadJs.ts --budget), run in CI after `make build`.", + "The check fails when first-load brotli bytes grow more than 2% over brBytes, or raw bytes more than 100 KiB over rawBytes.", + "A smaller first load never fails, so a PR that shrinks it needs no update here.", + "If a PR grows the first load on purpose, the perf owner updates the recorded value in that same PR.", + "Recorded from a clean `make build` of main ea19a44101 with Bun 1.3.12." + ], + "rawBytes": 5124751, + "brBytes": 1209864 +} diff --git a/scripts/perf/firstLoadJs.ts b/scripts/perf/firstLoadJs.ts index fb78394cf08..ca113cb6a41 100644 --- a/scripts/perf/firstLoadJs.ts +++ b/scripts/perf/firstLoadJs.ts @@ -8,7 +8,11 @@ * `xum server` sends those bytes. It skips files under 1 KiB, which are then sent raw. * * Usage: bun scripts/perf/firstLoadJs.ts [distDir] [--json] [--forbid ]... - * Exit codes: 0 ok, 1 a forbidden source is on the first load, 2 unusable input. + * [--budget ] + * `--budget` (T3 PR8, `make check-first-load-js` in CI) also fails when the first load grows + * over the recorded values in that file: brotli by more than 2%, raw by more than 100 KiB. + * Exit codes: 0 ok, 1 a forbidden source is on the first load or a budget is exceeded, + * 2 unusable input. */ import * as fs from "fs"; import * as path from "path"; @@ -22,9 +26,28 @@ export const FIRST_LOAD_FORBIDDEN_SOURCES: readonly string[] = [ "features/Settings/Sections/ProvidersSection", "node_modules/@shikijs/langs/", "node_modules/recharts/", + // Right-sidebar panels made lazy by PR7. Timeline and Artifacts stay eager: TimelineDialog + // and ArtifactsDialog import them statically. "features/desktop/DesktopPanel", + "components/InstructionsTab/InstructionsTab", + "components/OutputTab/OutputTab", + "features/RightSidebar/BrowserTab/BrowserTab", + "features/RightSidebar/DevToolsTab/DevToolsTab", + "features/RightSidebar/GoalTab", + "features/RightSidebar/Memory/MemoryTab", + "features/RightSidebar/Workflows/WorkflowsTab", ]; +// Growth allowed over the recorded budget before `--budget` fails. A smaller first load never +// fails, so a PR that shrinks it needs no budget update. +const BUDGET_BR_GROWTH = 0.02; +const BUDGET_RAW_GROWTH_BYTES = 100 * 1024; + +interface FirstLoadBudget { + rawBytes: number; + brBytes: number; +} + function fail(message: string): never { console.error(`firstLoadJs: ${message}`); process.exit(2); @@ -101,6 +124,50 @@ function servedSize(file: string, extension: string, rawBytes: number): [number, return sibling?.isFile() ? [sibling.size, true] : [rawBytes, false]; } +function readBudget(file: string): FirstLoadBudget { + let budget: unknown; + try { + budget = JSON.parse(fs.readFileSync(file, "utf-8")); + } catch (error) { + fail(`cannot read budget ${file}: ${error instanceof Error ? error.message : String(error)}`); + } + const isBytes = (value: unknown) => Number.isSafeInteger(value) && (value as number) > 0; + if ( + typeof budget !== "object" || + budget === null || + !isBytes((budget as Partial).rawBytes) || + !isBytes((budget as Partial).brBytes) + ) { + fail(`${file} must be a JSON object with positive integer rawBytes and brBytes`); + } + const { rawBytes, brBytes } = budget as FirstLoadBudget; + return { rawBytes, brBytes }; +} + +/** Compares totals with the budget file; over-budget messages say how to update the budget. */ +function checkBudget( + totals: FirstLoadBudget, + file: string +): Array<{ over: boolean; text: string }> { + const budget = readBudget(file); + const checks = [ + ["brBytes", "brotli", Math.floor(budget.brBytes * (1 + BUDGET_BR_GROWTH)), "+2%"], + ["rawBytes", "raw", budget.rawBytes + BUDGET_RAW_GROWTH_BYTES, "+100 KiB"], + ] as const; + return checks.map(([key, name, limit, growth]) => { + const over = totals[key] > limit; + const limitText = `budget of ${limit} bytes (${key} ${budget[key]} in ${file}, ${growth})`; + return over + ? { + over, + text: + `first-load ${name} is ${totals[key]} bytes, over the ${limitText}. If this growth ` + + `is intended, the perf owner updates ${key} in ${file} in the PR that causes it.`, + } + : { over, text: `first-load ${name} ${totals[key]} bytes is within the ${limitText}` }; + }); +} + const KIB = new Intl.NumberFormat("en-US", { minimumFractionDigits: 1, maximumFractionDigits: 1 }); /** A right-aligned table cell; `*` marks a file served raw for lack of a precompressed sibling. */ @@ -111,11 +178,13 @@ function kib(bytes: number, precompressed = true): string { function main(argv: string[]): void { let distArg: string | undefined; let json = false; + let budgetFile: string | undefined; const patterns = [...FIRST_LOAD_FORBIDDEN_SOURCES]; for (let i = 0; i < argv.length; i++) { const arg = argv[i]; if (arg === "--json") json = true; else if (arg === "--forbid" && /^[^-]/.test(argv[i + 1] ?? "")) patterns.push(argv[++i]); + else if (arg === "--budget" && /^[^-]/.test(argv[i + 1] ?? "")) budgetFile = argv[++i]; else if (arg.startsWith("-") || distArg != null) fail(`bad or incomplete argument ${arg}`); else distArg = arg; } @@ -162,10 +231,15 @@ function main(argv: string[]): void { } } + const budgetChecks = budgetFile == null ? [] : checkBudget(totals, budgetFile); for (const v of violations) { console.error(`forbidden on first load: ${v.file} has ${v.source} (matches "${v.pattern}")`); } - if (violations.length > 0) process.exit(1); + for (const check of budgetChecks) { + if (check.over) console.error(check.text); + else if (!json) console.log(check.text); + } + if (violations.length > 0 || budgetChecks.some((c) => c.over)) process.exit(1); } if (import.meta.main) { From 4974e5e8eef3f163583da6282ce25ef0b50e02e7 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Sat, 10 Oct 2026 20:09:47 +0000 Subject: [PATCH 3/5] ci: run the test jobs when index.html changes index.html defines the first-load roots, so an index.html-only PR must run Smoke / Server and its first-load budget check. --- .github/workflows/pr.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 0444eb20f98..03f9d8a2298 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -82,6 +82,9 @@ jobs: - 'bun.lock' - 'tsconfig*.json' - 'vite*.ts' + # index.html defines the first-load roots, so the first-load budget check + # (Smoke / Server) must run when it changes. + - 'index.html' - 'Makefile' - 'electron-builder.yml' # Pixel's default matrix and the Storybook config decide which snapshots exist, From 3fe3d4b06e97a56867f183eb005b7d65133ce4de Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Sat, 10 Oct 2026 20:19:12 +0000 Subject: [PATCH 4/5] ci: count index.html in the first-load budget totals Codex round 2: the totals skipped index.html, so its inline boot script could grow without failing the budget. index.html now counts as one measured entry (not scanned for forbidden sources). Budget re-recorded from the same clean build of main ea19a44101: +9,341 bytes. --- scripts/perf/firstLoadBudget.json | 6 ++--- scripts/perf/firstLoadJs.test.ts | 41 ++++++++++++++++++++++--------- scripts/perf/firstLoadJs.ts | 6 ++++- 3 files changed, 37 insertions(+), 16 deletions(-) diff --git a/scripts/perf/firstLoadBudget.json b/scripts/perf/firstLoadBudget.json index d91b3a6eac3..bfd71aa4eb3 100644 --- a/scripts/perf/firstLoadBudget.json +++ b/scripts/perf/firstLoadBudget.json @@ -1,11 +1,11 @@ { "comment": [ - "First-load JS budget for `make check-first-load-js` (scripts/perf/firstLoadJs.ts --budget), run in CI after `make build`.", + "First-load budget for `make check-first-load-js` (scripts/perf/firstLoadJs.ts --budget), run in CI after `make build`. The totals are index.html plus its statically reachable module JS.", "The check fails when first-load brotli bytes grow more than 2% over brBytes, or raw bytes more than 100 KiB over rawBytes.", "A smaller first load never fails, so a PR that shrinks it needs no update here.", "If a PR grows the first load on purpose, the perf owner updates the recorded value in that same PR.", "Recorded from a clean `make build` of main ea19a44101 with Bun 1.3.12." ], - "rawBytes": 5124751, - "brBytes": 1209864 + "rawBytes": 5134092, + "brBytes": 1219205 } diff --git a/scripts/perf/firstLoadJs.test.ts b/scripts/perf/firstLoadJs.test.ts index 2073aeda8b2..67b3454ebe4 100644 --- a/scripts/perf/firstLoadJs.test.ts +++ b/scripts/perf/firstLoadJs.test.ts @@ -32,13 +32,13 @@ describe("firstLoadJs", () => { // The script only stats sibling sizes, so the content need not be valid brotli. await fs.writeFile(path.join(dir, "main-AAAAAAAA.js.br"), "x".repeat(7)); const size = async (file: string) => (await fs.stat(path.join(dir, file))).size; - const [main, shared, deep] = await Promise.all( - ["main-AAAAAAAA.js", "shared-BBBBBBBB.js", "deep-EEEEEEEE.js"].map(size) + const [main, shared, deep, html] = await Promise.all( + ["main-AAAAAAAA.js", "shared-BBBBBBBB.js", "deep-EEEEEEEE.js", "index.html"].map(size) ); const { exitCode, stdout, stderr } = await runScript([dir, "--json"]); expect(exitCode, stderr).toBe(0); const report = JSON.parse(stdout) as { files: unknown[]; totals: unknown }; - // Sorted by raw size. Only shared's `export * from` reaches the unpreloaded deep chunk. The + // Sorted by raw size; index.html itself counts. Only shared's `export * from` reaches the unpreloaded deep chunk. The // `import()` and __vite__mapDeps string keep the lazy chunk out; the stylesheet is not JS. const raw = (file: string, bytes: number) => ({ file, @@ -47,16 +47,19 @@ describe("firstLoadJs", () => { gzipBytes: bytes, precompressed: { br: false, gzip: false }, }); - expect(report.files).toEqual([ - { ...raw("main-AAAAAAAA.js", main), brBytes: 7, precompressed: { br: true, gzip: false } }, - raw("shared-BBBBBBBB.js", shared), - raw("deep-EEEEEEEE.js", deep), - ]); + expect(report.files).toEqual( + [ + { ...raw("main-AAAAAAAA.js", main), brBytes: 7, precompressed: { br: true, gzip: false } }, + raw("shared-BBBBBBBB.js", shared), + raw("deep-EEEEEEEE.js", deep), + raw("index.html", html), + ].sort((a, b) => b.rawBytes - a.rawBytes || a.file.localeCompare(b.file)) + ); expect(report.totals).toEqual({ - files: 3, - rawBytes: main + shared + deep, - brBytes: 7 + shared + deep, - gzipBytes: main + shared + deep, + files: 4, + rawBytes: main + shared + deep + html, + brBytes: 7 + shared + deep + html, + gzipBytes: main + shared + deep + html, }); }); @@ -131,6 +134,20 @@ describe("firstLoadJs", () => { expect((await runScript([dir, "--budget", await writeBudget(edge)])).exitCode).toBe(0); }); + test("fails when the inline boot script in index.html grows more than 100 KiB", async () => { + // index.html counts as one entry, so inline JS (and other markup) growth is measured. + const recorded = await totals(); + const html = path.join(dir, "index.html"); + const grownScript = `window.__boot = "${"x".repeat(101 * 1024)}";`; + await fs.writeFile( + html, + (await fs.readFile(html, "utf-8")).replace("window.__boot = true;", grownScript) + ); + const grown = await runScript([dir, "--budget", await writeBudget(recorded)]); + expect(grown.exitCode).toBe(1); + expect(grown.stderr).toContain("rawBytes"); + }); + test("a forbidden module fails even when the bytes are under budget", async () => { const budgetFile = await writeBudget(await totals()); const result = await runScript([ diff --git a/scripts/perf/firstLoadJs.ts b/scripts/perf/firstLoadJs.ts index ca113cb6a41..41d6f085276 100644 --- a/scripts/perf/firstLoadJs.ts +++ b/scripts/perf/firstLoadJs.ts @@ -9,6 +9,9 @@ * * Usage: bun scripts/perf/firstLoadJs.ts [distDir] [--json] [--forbid ]... * [--budget ] + * The totals also count index.html itself (T3 PR8), so growth of its inline boot script or + * markup is measured. Before PR8 the totals were module JS only (9.1 KiB less on main). + * External classic `'; + await fs.writeFile(html, original.replace("", `${classicTag}`)); + const classic = await runScript([dir]); + expect(classic.exitCode).toBe(2); + expect(classic.stderr).toContain("./classic.js"); + await fs.writeFile(html, original); + }); + describe("--budget", () => { const BUDGET_FILE = path.join(import.meta.dir, "firstLoadBudget.json"); diff --git a/scripts/perf/firstLoadJs.ts b/scripts/perf/firstLoadJs.ts index 41d6f085276..0d00818a395 100644 --- a/scripts/perf/firstLoadJs.ts +++ b/scripts/perf/firstLoadJs.ts @@ -11,7 +11,7 @@ * [--budget ] * The totals also count index.html itself (T3 PR8), so growth of its inline boot script or * markup is measured. Before PR8 the totals were module JS only (9.1 KiB less on main). - * External classic `