diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 55192f60ac0..03f9d8a2298 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -82,6 +82,9 @@ jobs: - 'bun.lock' - 'tsconfig*.json' - 'vite*.ts' + # index.html defines the first-load roots, so the first-load budget check + # (Smoke / Server) must run when it changes. + - 'index.html' - 'Makefile' - 'electron-builder.yml' # Pixel's default matrix and the Storybook config decide which snapshots exist, @@ -470,6 +473,8 @@ jobs: - uses: ./.github/actions/setup-xum - run: ./scripts/generate-npm-shrinkwrap.sh - run: make build + # First-load JS budget (T3, #5971): reuses this build, so it adds no build. + - run: make check-first-load-js - run: make check-startup-imports-runtime - name: Pack npm package id: pack diff --git a/Makefile b/Makefile index a7e02c2ce11..a3081f877e4 100644 --- a/Makefile +++ b/Makefile @@ -97,7 +97,7 @@ include fmt.mk .PHONY: vscode-ext vscode-ext-install .PHONY: docs-server check-docs-links .PHONY: storybook storybook-run storybook-build storybook-flake-check test-storybook storybook-budget -.PHONY: benchmark-terminal bench bench-compare first-load-js bug-bash mcp-apps-e2e +.PHONY: benchmark-terminal bench bench-compare first-load-js bug-bash mcp-apps-e2e check-first-load-js .PHONY: ensure-deps mux .PHONY: check-startup-imports check-startup-imports-runtime check-react-compiler check-test-routing check-test-seam-comments test-bench-scripts @@ -771,6 +771,9 @@ bench-compare: node_modules/.installed src/version.ts ## Compare *.bench.ts resu first-load-js: ## Report first-load JS bytes of dist/ (run make build-renderer first) @bun scripts/perf/firstLoadJs.ts dist +check-first-load-js: ## Fail when first-load JS of dist/ is over scripts/perf/firstLoadBudget.json or loads a must-stay-lazy module (run make build-renderer first) + @bun scripts/perf/firstLoadJs.ts dist --budget scripts/perf/firstLoadBudget.json + benchmark-terminal: ## Run Terminal-Bench 2.0 with Harbor (use TB_HARBOR_PACKAGE/TB_HARBOR_DAYTONA_PACKAGE/TB_DATASET/TB_CONCURRENCY/TB_TIMEOUT/TB_ENV/TB_MODEL/TB_ARGS to customize) @# Pin Harbor with the Daytona extra so scheduled ingestion does not break on future CLI or adapter API drift. @# Force the Daytona SDK to the cursor-pagination API while keeping Harbor stable. diff --git a/scripts/perf/firstLoadBudget.json b/scripts/perf/firstLoadBudget.json new file mode 100644 index 00000000000..bfd71aa4eb3 --- /dev/null +++ b/scripts/perf/firstLoadBudget.json @@ -0,0 +1,11 @@ +{ + "comment": [ + "First-load budget for `make check-first-load-js` (scripts/perf/firstLoadJs.ts --budget), run in CI after `make build`. The totals are index.html plus its statically reachable module JS.", + "The check fails when first-load brotli bytes grow more than 2% over brBytes, or raw bytes more than 100 KiB over rawBytes.", + "A smaller first load never fails, so a PR that shrinks it needs no update here.", + "If a PR grows the first load on purpose, the perf owner updates the recorded value in that same PR.", + "Recorded from a clean `make build` of main ea19a44101 with Bun 1.3.12." + ], + "rawBytes": 5134092, + "brBytes": 1219205 +} diff --git a/scripts/perf/firstLoadJs.test.ts b/scripts/perf/firstLoadJs.test.ts index ec7d3173dc8..94c505365fe 100644 --- a/scripts/perf/firstLoadJs.test.ts +++ b/scripts/perf/firstLoadJs.test.ts @@ -32,13 +32,13 @@ describe("firstLoadJs", () => { // The script only stats sibling sizes, so the content need not be valid brotli. await fs.writeFile(path.join(dir, "main-AAAAAAAA.js.br"), "x".repeat(7)); const size = async (file: string) => (await fs.stat(path.join(dir, file))).size; - const [main, shared, deep] = await Promise.all( - ["main-AAAAAAAA.js", "shared-BBBBBBBB.js", "deep-EEEEEEEE.js"].map(size) + const [main, shared, deep, html] = await Promise.all( + ["main-AAAAAAAA.js", "shared-BBBBBBBB.js", "deep-EEEEEEEE.js", "index.html"].map(size) ); const { exitCode, stdout, stderr } = await runScript([dir, "--json"]); expect(exitCode, stderr).toBe(0); const report = JSON.parse(stdout) as { files: unknown[]; totals: unknown }; - // Sorted by raw size. Only shared's `export * from` reaches the unpreloaded deep chunk. The + // Sorted by raw size; index.html itself counts. Only shared's `export * from` reaches the unpreloaded deep chunk. The // `import()` and __vite__mapDeps string keep the lazy chunk out; the stylesheet is not JS. const raw = (file: string, bytes: number) => ({ file, @@ -47,16 +47,19 @@ describe("firstLoadJs", () => { gzipBytes: bytes, precompressed: { br: false, gzip: false }, }); - expect(report.files).toEqual([ - { ...raw("main-AAAAAAAA.js", main), brBytes: 7, precompressed: { br: true, gzip: false } }, - raw("shared-BBBBBBBB.js", shared), - raw("deep-EEEEEEEE.js", deep), - ]); + expect(report.files).toEqual( + [ + { ...raw("main-AAAAAAAA.js", main), brBytes: 7, precompressed: { br: true, gzip: false } }, + raw("shared-BBBBBBBB.js", shared), + raw("deep-EEEEEEEE.js", deep), + raw("index.html", html), + ].sort((a, b) => b.rawBytes - a.rawBytes || a.file.localeCompare(b.file)) + ); expect(report.totals).toEqual({ - files: 3, - rawBytes: main + shared + deep, - brBytes: 7 + shared + deep, - gzipBytes: main + shared + deep, + files: 4, + rawBytes: main + shared + deep + html, + brBytes: 7 + shared + deep + html, + gzipBytes: main + shared + deep + html, }); }); @@ -74,4 +77,117 @@ describe("firstLoadJs", () => { expect(noMap.exitCode).toBe(2); expect(noMap.stderr).toContain("deep-EEEEEEEE.js has no source map"); }); + + test("a classic external script exits 2, because the graph does not measure it", async () => { + const html = path.join(dir, "index.html"); + await fs.writeFile(path.join(dir, "classic.js"), "window.__classic = true;"); + const original = await fs.readFile(html, "utf-8"); + const classicTag = ''; + await fs.writeFile(html, original.replace("", `${classicTag}`)); + const classic = await runScript([dir]); + expect(classic.exitCode).toBe(2); + expect(classic.stderr).toContain("./classic.js"); + await fs.writeFile(html, original); + }); + + describe("--budget", () => { + const BUDGET_FILE = path.join(import.meta.dir, "firstLoadBudget.json"); + + async function totals() { + const { exitCode, stdout, stderr } = await runScript([dir, "--json"]); + expect(exitCode, stderr).toBe(0); + return (JSON.parse(stdout) as { totals: { rawBytes: number; brBytes: number } }).totals; + } + + async function writeBudget(budget: unknown) { + const file = path.join(dir, "budget.json"); + await fs.writeFile(file, typeof budget === "string" ? budget : JSON.stringify(budget)); + return file; + } + + test("passes at or under the recorded values and fails on brotli growth over 2%", async () => { + const { rawBytes, brBytes } = await totals(); + const exact = await runScript([dir, "--budget", await writeBudget({ rawBytes, brBytes })]); + expect(exact.exitCode, exact.stderr).toBe(0); + // A smaller first load never fails, so shrinking needs no budget update. + const shrunk = { rawBytes: rawBytes * 2, brBytes: brBytes * 2 }; + expect((await runScript([dir, "--budget", await writeBudget(shrunk)])).exitCode).toBe(0); + // The committed budget file must stay valid; this fixture is far under it. + const committed = await runScript([dir, "--budget", BUDGET_FILE]); + expect(committed.exitCode, committed.stderr).toBe(0); + + const budgetFile = await writeBudget({ rawBytes, brBytes: Math.floor(brBytes / 1.03) }); + const grown = await runScript([dir, "--budget", budgetFile]); + expect(grown.exitCode).toBe(1); + expect(grown.stderr).toContain(budgetFile); + // The message names the new value, the budget key and the budget file. + expect(grown.stderr).toContain(String(brBytes)); + expect(grown.stderr).toContain("brBytes"); + expect(grown.stderr).not.toContain("rawBytes"); + }); + + test("fails when raw grows more than 100 KiB, even with brotli unchanged", async () => { + // main has a .br sibling, so padding it grows raw bytes only. + await fs.writeFile(path.join(dir, "main-AAAAAAAA.js.br"), "x".repeat(7)); + const before = await totals(); + const padded = 100 * 1024 + 1; + await fs.appendFile(path.join(dir, "main-AAAAAAAA.js"), `\n/*${"x".repeat(padded - 5)}*/`); + const after = await totals(); + expect(after.rawBytes - before.rawBytes).toBe(padded); + + const recorded = { rawBytes: before.rawBytes, brBytes: after.brBytes }; + const grown = await runScript([dir, "--budget", await writeBudget(recorded)]); + expect(grown.exitCode).toBe(1); + expect(grown.stderr).toContain(String(after.rawBytes)); + expect(grown.stderr).toContain("rawBytes"); + expect(grown.stderr).not.toContain("brBytes"); + // Exactly 100 KiB of growth is still within budget. + const edge = { rawBytes: after.rawBytes - 100 * 1024, brBytes: after.brBytes }; + expect((await runScript([dir, "--budget", await writeBudget(edge)])).exitCode).toBe(0); + }); + + test("fails when the inline boot script in index.html grows more than 100 KiB", async () => { + // index.html counts as one entry, so inline JS (and other markup) growth is measured. + const recorded = await totals(); + const html = path.join(dir, "index.html"); + const grownScript = `window.__boot = "${"x".repeat(101 * 1024)}";`; + await fs.writeFile( + html, + (await fs.readFile(html, "utf-8")).replace("window.__boot = true;", grownScript) + ); + const grown = await runScript([dir, "--budget", await writeBudget(recorded)]); + expect(grown.exitCode).toBe(1); + expect(grown.stderr).toContain("rawBytes"); + }); + + test("a forbidden module fails even when the bytes are under budget", async () => { + const budgetFile = await writeBudget(await totals()); + const result = await runScript([ + dir, + "--budget", + budgetFile, + "--forbid", + "node_modules/shared-dep/", + ]); + expect(result.exitCode).toBe(1); + expect(result.stderr).toContain("forbidden on first load"); + }); + + test("an unusable budget file exits 2", async () => { + const { rawBytes, brBytes } = await totals(); + for (const budget of [ + "{ not json", + "null", + { rawBytes }, + { rawBytes, brBytes: -1 }, + { rawBytes: "1000", brBytes }, + { rawBytes: 1.5, brBytes }, + ]) { + const result = await runScript([dir, "--budget", await writeBudget(budget)]); + expect(result.exitCode, JSON.stringify(budget)).toBe(2); + } + expect((await runScript([dir, "--budget", path.join(dir, "missing.json")])).exitCode).toBe(2); + expect((await runScript([dir, "--budget"])).exitCode).toBe(2); + }); + }); }); diff --git a/scripts/perf/firstLoadJs.ts b/scripts/perf/firstLoadJs.ts index fb78394cf08..0d00818a395 100644 --- a/scripts/perf/firstLoadJs.ts +++ b/scripts/perf/firstLoadJs.ts @@ -8,7 +8,14 @@ * `xum server` sends those bytes. It skips files under 1 KiB, which are then sent raw. * * Usage: bun scripts/perf/firstLoadJs.ts [distDir] [--json] [--forbid ]... - * Exit codes: 0 ok, 1 a forbidden source is on the first load, 2 unusable input. + * [--budget ] + * The totals also count index.html itself (T3 PR8), so growth of its inline boot script or + * markup is measured. Before PR8 the totals were module JS only (9.1 KiB less on main). + * An external classic `