diff --git a/.github/workflows/homebrew-main-shell-ci.yml b/.github/workflows/homebrew-main-shell-ci.yml index e2d8197c21..44f8539869 100644 --- a/.github/workflows/homebrew-main-shell-ci.yml +++ b/.github/workflows/homebrew-main-shell-ci.yml @@ -16,6 +16,7 @@ on: - "flake.lock" - "flake.nix" - "homebrew/main-shell*" + - "homebrew/test/homebrew_guest_lifecycle*.ts" - "host/package-lock.json" - "host/package.json" - "host/src/**" @@ -40,6 +41,7 @@ on: - "scripts/fetch-binaries.sh" - "scripts/homebrew-brewfile-selection.rb" - "scripts/homebrew-language-runtime-contract.ts" + - "scripts/homebrew-closed-lazy-assets*.ts" - "scripts/homebrew-main-shell-node-smoke.ts" - "scripts/homebrew-main-shell-image-contract*.ts" - "scripts/homebrew-vfs-acceptance-smoke.ts" @@ -77,6 +79,7 @@ on: - "flake.lock" - "flake.nix" - "homebrew/main-shell*" + - "homebrew/test/homebrew_guest_lifecycle*.ts" - "host/package-lock.json" - "host/package.json" - "host/src/**" @@ -101,6 +104,7 @@ on: - "scripts/fetch-binaries.sh" - "scripts/homebrew-brewfile-selection.rb" - "scripts/homebrew-language-runtime-contract.ts" + - "scripts/homebrew-closed-lazy-assets*.ts" - "scripts/homebrew-main-shell-node-smoke.ts" - "scripts/homebrew-main-shell-image-contract*.ts" - "scripts/homebrew-vfs-acceptance-smoke.ts" @@ -217,6 +221,25 @@ jobs: npx playwright install chromium --with-deps ) + - name: Validate the stock guest lifecycle contracts + run: | + set -euo pipefail + npx tsx --test \ + apps/browser-demos/playwright-server-policy.test.ts \ + homebrew/test/homebrew_guest_lifecycle_browser_fixture.test.ts \ + homebrew/test/homebrew_guest_lifecycle_contract.test.ts \ + homebrew/test/homebrew_guest_lifecycle_runner.test.ts \ + homebrew/test/homebrew_guest_lifecycle_runtime_contract.test.ts \ + homebrew/test/homebrew_guest_lifecycle_runtime_inputs.test.ts \ + scripts/homebrew-main-shell-image-contract.test.ts \ + scripts/homebrew-closed-lazy-assets.test.ts + npx esbuild homebrew/test/homebrew_guest_lifecycle_node.ts \ + --bundle \ + --platform=node \ + --format=esm \ + --packages=external \ + --outfile="$RUNNER_TEMP/homebrew_guest_lifecycle_node.mjs" + - name: Select one verified package generation env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -305,9 +328,9 @@ jobs: # Vite resolves static @binaries imports across the wider demo app. # Derive those registry roots from the imports and package manifests # so adding an import cannot silently leave this browser proof stale. - # They are browser bundling inputs, not shell VFS inputs: archive-stage - # builds shell's dependency-free package from public bottles, and the - # exact archive replaces shell before either runtime proof starts. + # They are browser bundling inputs. The shell VFS consumes the exact + # standalone homebrew-bootstrap package as a dependency, while the + # exact candidate image still replaces shell before runtime proof. # Use the normal resolver contract for these supporting packages. A # matching public archive is reused; a PR that changes one of its # declared build inputs source-builds the exact current recipe. Using @@ -335,7 +358,7 @@ jobs: local-binaries/kernel.wasm ' - - name: Build the exact lazy shell from public bottles + - name: Build the exact lazy shell and its source-materialized derivative id: candidate run: | set -euo pipefail @@ -344,8 +367,10 @@ jobs: [[ "$tap_sha" =~ ^[0-9a-f]{40}$ ]] tap_root="$RUNNER_TEMP/homebrew-tap-core" candidate_root="$RUNNER_TEMP/homebrew-main-shell-candidate" + eager_root="$RUNNER_TEMP/homebrew-main-shell-eager-candidate" test ! -e "$tap_root" test ! -e "$candidate_root" + test ! -e "$eager_root" git init "$tap_root" git -C "$tap_root" remote add origin \ https://github.com/Kandelo-dev/homebrew-tap-core.git @@ -353,6 +378,12 @@ jobs: git -C "$tap_root" checkout --detach FETCH_HEAD test "$(git -C "$tap_root" rev-parse HEAD)" = "$tap_sha" test -z "$(git -C "$tap_root" status --porcelain=v1 --untracked-files=all)" + bootstrap=$(bash scripts/resolve-binary.sh \ + programs/homebrew-bootstrap/homebrew-bootstrap.zip) + bootstrap_env=$(bash scripts/resolve-binary.sh \ + programs/homebrew-bootstrap/homebrew-brew.env) + test -f "$bootstrap" + test -f "$bootstrap_env" env -u GH_TOKEN -u GITHUB_TOKEN \ -u HOMEBREW_GITHUB_API_TOKEN \ -u HOMEBREW_GITHUB_PACKAGES_TOKEN \ @@ -362,6 +393,9 @@ jobs: --lazy-shell \ --tap-root "$tap_root" \ --expected-tap-sha "$tap_sha" \ + --package-tree-spec homebrew/main-shell-brew-package-tree.json \ + --package-tree-archive "$bootstrap" \ + --homebrew-bootstrap-env "$bootstrap_env" \ --work-dir "$candidate_root" test -f "$candidate_root/main-shell.vfs.zst" test -f "$candidate_root/main-shell-report.json" @@ -369,9 +403,50 @@ jobs: "$candidate_root/main-shell-report.json") test "$(find "$candidate_root/bottle-mirror" -maxdepth 1 -type f | wc -l)" \ -eq "$expected_mirror_files" + env -u GH_TOKEN -u GITHUB_TOKEN \ + -u HOMEBREW_GITHUB_API_TOKEN \ + -u HOMEBREW_GITHUB_PACKAGES_TOKEN \ + -u HOMEBREW_DOCKER_REGISTRY_TOKEN \ + bash scripts/dev-shell.sh bash \ + scripts/build-homebrew-main-shell-closure.sh \ + --lazy-shell \ + --materialize-package-tree \ + --tap-root "$tap_root" \ + --expected-tap-sha "$tap_sha" \ + --package-tree-spec homebrew/main-shell-brew-package-tree.json \ + --package-tree-archive "$bootstrap" \ + --homebrew-bootstrap-env "$bootstrap_env" \ + --bottle-cache "$candidate_root/bottle-cache" \ + --work-dir "$eager_root" + test -f "$eager_root/main-shell.vfs.zst" + test -f "$eager_root/main-shell-report.json" + test "$(jq -er '.package_deferred_trees[0].state' \ + "$candidate_root/main-shell-report.json")" = deferred + test "$(jq -er '.package_deferred_trees[0].state' \ + "$eager_root/main-shell-report.json")" = materialized + jq -S '.package_deferred_trees[0] | del(.state)' \ + "$candidate_root/main-shell-report.json" \ + > "$RUNNER_TEMP/homebrew-main-shell-lazy-package-tree.json" + jq -S '.package_deferred_trees[0] | del(.state)' \ + "$eager_root/main-shell-report.json" \ + > "$RUNNER_TEMP/homebrew-main-shell-eager-package-tree.json" + cmp "$RUNNER_TEMP/homebrew-main-shell-lazy-package-tree.json" \ + "$RUNNER_TEMP/homebrew-main-shell-eager-package-tree.json" + jq -S '.homebrew_bootstrap' \ + "$candidate_root/main-shell-report.json" \ + > "$RUNNER_TEMP/homebrew-main-shell-lazy-bootstrap.json" + jq -S '.homebrew_bootstrap' \ + "$eager_root/main-shell-report.json" \ + > "$RUNNER_TEMP/homebrew-main-shell-eager-bootstrap.json" + cmp "$RUNNER_TEMP/homebrew-main-shell-lazy-bootstrap.json" \ + "$RUNNER_TEMP/homebrew-main-shell-eager-bootstrap.json" { echo "image=$candidate_root/main-shell.vfs.zst" echo "report=$candidate_root/main-shell-report.json" + echo "bootstrap=$bootstrap" + echo "bootstrap_env=$bootstrap_env" + echo "eager_image=$eager_root/main-shell.vfs.zst" + echo "eager_report=$eager_root/main-shell-report.json" echo "tap_sha=$tap_sha" } >> "$GITHUB_OUTPUT" @@ -379,11 +454,15 @@ jobs: id: image env: CANDIDATE_PATH: ${{ steps.candidate.outputs.image }} + CANDIDATE_BOOTSTRAP_PATH: ${{ steps.candidate.outputs.bootstrap }} + CANDIDATE_BOOTSTRAP_ENV_PATH: ${{ steps.candidate.outputs.bootstrap_env }} run: | set -euo pipefail browser_copy=apps/browser-demos/public/shell.vfs.zst install_session="homebrew-main-shell-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_JOB}" test -f "$CANDIDATE_PATH" + test -f "$CANDIDATE_BOOTSTRAP_PATH" + test -f "$CANDIDATE_BOOTSTRAP_ENV_PATH" mkdir -p "$(dirname "$browser_copy")" bash scripts/dev-shell.sh bash -c ' set -euo pipefail @@ -399,9 +478,21 @@ jobs: cmp "$CANDIDATE_PATH" "$resolved" cp "$CANDIDATE_PATH" "$browser_copy" cmp "$CANDIDATE_PATH" "$browser_copy" + resolved_bootstrap=$(bash scripts/resolve-binary.sh \ + programs/homebrew-bootstrap/homebrew-bootstrap.zip) + resolved_bootstrap_env=$(bash scripts/resolve-binary.sh \ + programs/homebrew-bootstrap/homebrew-brew.env) + cmp "$CANDIDATE_BOOTSTRAP_PATH" "$resolved_bootstrap" + cmp "$CANDIDATE_BOOTSTRAP_ENV_PATH" "$resolved_bootstrap_env" image_sha=$(sha256sum "$CANDIDATE_PATH" | awk '{print $1}') + bootstrap_sha=$(sha256sum "$CANDIDATE_BOOTSTRAP_PATH" | awk '{print $1}') + bootstrap_bytes=$(wc -c <"$CANDIDATE_BOOTSTRAP_PATH" | tr -d '[:space:]') [[ "$image_sha" =~ ^[0-9a-f]{64}$ ]] + [[ "$bootstrap_sha" =~ ^[0-9a-f]{64}$ ]] + [[ "$bootstrap_bytes" =~ ^[1-9][0-9]*$ ]] echo "sha256=$image_sha" >> "$GITHUB_OUTPUT" + echo "bootstrap_sha256=$bootstrap_sha" >> "$GITHUB_OUTPUT" + echo "bootstrap_bytes=$bootstrap_bytes" >> "$GITHUB_OUTPUT" - name: Recover the exact bottle mirror from anonymous source packages id: mirror @@ -462,33 +553,46 @@ jobs: - name: Boot the exact installed bytes in Node run: | - node_smoke_args=( - --image "${{ steps.candidate.outputs.image }}" - --migration-lock homebrew/main-shell-migration-lock.json - --demo-config homebrew/main-shell-demo.json - --transport-mode "$TRANSPORT_MODE" - ) - case "$TRANSPORT_MODE" in - closed) - node_smoke_args+=( - --bottle-mirror-plan "${{ steps.mirror.outputs.plan }}" - ) - ;; - public) - ;; - *) - echo "unsupported transport mode: $TRANSPORT_MODE" >&2 - exit 1 - ;; - esac - bash scripts/dev-shell.sh npx tsx \ - scripts/homebrew-main-shell-node-smoke.ts \ - "${node_smoke_args[@]}" + set -euo pipefail + run_node_smoke() { + local image="$1" + local state="$2" + local node_smoke_args=( + --image "$image" + --migration-lock homebrew/main-shell-migration-lock.json + --homebrew-bootstrap-spec homebrew/main-shell-brew-package-tree.json + --homebrew-bootstrap-archive "${{ steps.candidate.outputs.bootstrap }}" + --homebrew-bootstrap-env "${{ steps.candidate.outputs.bootstrap_env }}" + --homebrew-bootstrap-state "$state" + --demo-config homebrew/main-shell-demo.json + --transport-mode "$TRANSPORT_MODE" + ) + case "$TRANSPORT_MODE" in + closed) + node_smoke_args+=( + --bottle-mirror-plan "${{ steps.mirror.outputs.plan }}" + ) + ;; + public) + ;; + *) + echo "unsupported transport mode: $TRANSPORT_MODE" >&2 + exit 1 + ;; + esac + bash scripts/dev-shell.sh npx tsx \ + scripts/homebrew-main-shell-node-smoke.ts \ + "${node_smoke_args[@]}" + } + run_node_smoke "${{ steps.candidate.outputs.image }}" deferred + run_node_smoke "${{ steps.candidate.outputs.eager_image }}" materialized - name: Boot the current main-shell path in Chromium env: KANDELO_HOMEBREW_MAIN_SHELL_STRICT: "1" KANDELO_HOMEBREW_MAIN_SHELL_SHA256: ${{ steps.image.outputs.sha256 }} + KANDELO_HOMEBREW_MAIN_SHELL_BOOTSTRAP_SHA256: ${{ steps.image.outputs.bootstrap_sha256 }} + KANDELO_HOMEBREW_MAIN_SHELL_BOOTSTRAP_BYTES: ${{ steps.image.outputs.bootstrap_bytes }} KANDELO_HOMEBREW_MAIN_SHELL_TRANSPORT_MODE: ${{ env.TRANSPORT_MODE }} KANDELO_HOMEBREW_MAIN_SHELL_MIRROR_PLAN_URL: /homebrew-main-shell-bottles/kandelo-homebrew-bottle-mirror-plan.json run: | @@ -498,6 +602,8 @@ jobs: playwright_env=( "KANDELO_HOMEBREW_MAIN_SHELL_STRICT=$KANDELO_HOMEBREW_MAIN_SHELL_STRICT" "KANDELO_HOMEBREW_MAIN_SHELL_SHA256=$KANDELO_HOMEBREW_MAIN_SHELL_SHA256" + "KANDELO_HOMEBREW_MAIN_SHELL_BOOTSTRAP_SHA256=$KANDELO_HOMEBREW_MAIN_SHELL_BOOTSTRAP_SHA256" + "KANDELO_HOMEBREW_MAIN_SHELL_BOOTSTRAP_BYTES=$KANDELO_HOMEBREW_MAIN_SHELL_BOOTSTRAP_BYTES" "KANDELO_HOMEBREW_MAIN_SHELL_TRANSPORT_MODE=$KANDELO_HOMEBREW_MAIN_SHELL_TRANSPORT_MODE" "KANDELO_HOMEBREW_MAIN_SHELL_MIRROR_PLAN_URL=$KANDELO_HOMEBREW_MAIN_SHELL_MIRROR_PLAN_URL" ) @@ -513,6 +619,14 @@ jobs: fi ( cd apps/browser-demos + # This offline browser contract must reject before any external + # fixture fetch. The separately gated live test runs only when an + # exact immutable fixture is explicitly supplied. + bash ../../scripts/dev-shell.sh env \ + "${playwright_env[@]}" \ + npx playwright test test/homebrew-guest-lifecycle.spec.ts \ + --project=chromium \ + --grep "rejects a guest lifecycle fixture" bash ../../scripts/dev-shell.sh env \ "PLAYWRIGHT_JSON_OUTPUT_FILE=$shell_report" \ "${playwright_env[@]}" \ @@ -528,12 +642,17 @@ jobs: npx playwright test test/kandelo-modeset.spec.ts \ --project=chromium --reporter=json ) - for report in "$shell_report" "$modeset_report"; do - jq -e ' - .stats.expected == 1 and .stats.unexpected == 0 and - .stats.flaky == 0 and .stats.skipped == 0 - ' "$report" >/dev/null - done + # WHY: the shell proof uses two independent browser machines so + # brew's Ruby first use cannot pre-materialize the language-isolation + # fixture. MODESET remains one separate heavyweight browser proof. + jq -e ' + .stats.expected == 2 and .stats.unexpected == 0 and + .stats.flaky == 0 and .stats.skipped == 0 + ' "$shell_report" >/dev/null + jq -e ' + .stats.expected == 1 and .stats.unexpected == 0 and + .stats.flaky == 0 and .stats.skipped == 0 + ' "$modeset_report" >/dev/null - name: Upload exact closure evidence if: always() @@ -543,6 +662,10 @@ jobs: path: | ${{ steps.candidate.outputs.image }} ${{ steps.candidate.outputs.report }} + ${{ steps.candidate.outputs.bootstrap }} + ${{ steps.candidate.outputs.bootstrap_env }} + ${{ steps.candidate.outputs.eager_image }} + ${{ steps.candidate.outputs.eager_report }} ${{ runner.temp }}/homebrew-main-shell-bottles ${{ runner.temp }}/homebrew-main-shell-bottle-recovery.json ${{ runner.temp }}/homebrew-main-shell-bottle-publish.json diff --git a/apps/browser-demos/lib/init/lazy-archives.ts b/apps/browser-demos/lib/init/lazy-archives.ts index dc369e783a..be267229c8 100644 --- a/apps/browser-demos/lib/init/lazy-archives.ts +++ b/apps/browser-demos/lib/init/lazy-archives.ts @@ -1,9 +1,14 @@ import vimZipUrl from "@binaries/programs/vim.zip?url"; import nethackZipUrl from "@binaries/programs/nethack.zip?url"; +import homebrewBootstrapZipUrl from "@binaries/programs/homebrew-bootstrap/homebrew-bootstrap.zip?url"; const SHELL_LAZY_ARCHIVES: Record = { "vim.zip": vimZipUrl, "nethack.zip": nethackZipUrl, + // The shell descriptor intentionally keeps a package-relative URL. Vite + // owns the browser deployment path, so resolve it through the package + // projection instead of baking a development-server URL into the VFS. + "homebrew-bootstrap.zip": homebrewBootstrapZipUrl, }; export function resolveShellLazyArchiveUrl(url: string): string { diff --git a/apps/browser-demos/pages/homebrew-vfs-test/main.ts b/apps/browser-demos/pages/homebrew-vfs-test/main.ts index a3946131dc..fc89cd1969 100644 --- a/apps/browser-demos/pages/homebrew-vfs-test/main.ts +++ b/apps/browser-demos/pages/homebrew-vfs-test/main.ts @@ -12,9 +12,18 @@ import { import type { BootDescriptor, } from "../../../../web-libs/kandelo-session/src/kernel-host"; +import { + runHomebrewGuestLifecycleInBrowser, + type HomebrewGuestLifecycleBrowserFixture, + type HomebrewGuestLifecycleBrowserResult, +} from "../../../../homebrew/test/homebrew_guest_lifecycle_browser"; import kernelWasmUrl from "@kernel-wasm?url"; const MAX_OUTPUT_BYTES = 1024 * 1024; +const corsProxyUrl = new URL( + `${import.meta.env.BASE_URL}__kandelo_cors_proxy?url=`, + window.location.href, +).href; interface HomebrewVfsAcceptanceRequest { vfsUrl: string; @@ -77,6 +86,42 @@ interface PackageLayerExecResult { stderr: string; } +interface RootfsExportAcceptanceRequest { + vfsUrl: string; + writePath: string; + writeText: string; + liveProcessUrl: string; + teardownProcessUrl: string; + lazyReadPath: string; + lazyReadUrl: string; + lazyReadText: string; + lateWritePath: string; + lateWriteText: string; +} + +interface RootfsExportAcceptanceResult { + persistedText: string; + firstExportSha256: string; + secondExportSha256: string; + firstExportBytes: number; + secondExportBytes: number; + liveProcessExitCode: number; + liveProcessExportError: string; + teardownProcessExitCode: number; + teardownExportError: string; + overlappingExportError: string; + overlappingWriteError: string; + lazyReadText: string; + lateWritePresentInExport: boolean; + writeAfterExportText: string; + diagnostics: Array<{ source: string; message: string }>; + lazyEntries: Array<{ + path: string; + url: string; + size: number; + }>; +} + declare global { interface Window { __homebrewVfsTestReady: boolean; @@ -95,6 +140,13 @@ declare global { ) => Promise; __destroyPackageLayerAcceptance: () => Promise; __packageLayerDiscardedBufferCount: () => number; + __runRootfsExportAcceptance: ( + request: RootfsExportAcceptanceRequest, + ) => Promise; + __releaseRootfsExportLazyResponse: () => Promise; + __runHomebrewGuestLifecycleAcceptance: ( + fixture: HomebrewGuestLifecycleBrowserFixture, + ) => Promise; } } @@ -135,8 +187,15 @@ function appendOutput(current: string, bytes: Uint8Array, label: string): string return next; } -async function sha256(bytes: ArrayBuffer): Promise { - const digest = new Uint8Array(await crypto.subtle.digest("SHA-256", bytes)); +async function sha256(bytes: ArrayBuffer | Uint8Array): Promise { + const source: BufferSource = bytes instanceof Uint8Array + ? new Uint8Array( + bytes.buffer as ArrayBuffer, + bytes.byteOffset, + bytes.byteLength, + ) + : bytes; + const digest = new Uint8Array(await crypto.subtle.digest("SHA-256", source)); return Array.from(digest, (byte) => byte.toString(16).padStart(2, "0")).join(""); } @@ -146,10 +205,85 @@ async function fetchBytes(url: string, label: string): Promise { return response.arrayBuffer(); } +async function rejectionMessage( + operation: Promise, + label: string, +): Promise { + try { + await operation; + } catch (error) { + return error instanceof Error ? error.message : String(error); + } + throw new Error(`${label} unexpectedly succeeded`); +} + +async function withTimeout( + operation: Promise, + label: string, + timeoutMs = 5_000, +): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + operation, + new Promise((_resolve, reject) => { + timer = setTimeout( + () => reject(new Error(`${label} timed out after ${timeoutMs}ms`)), + timeoutMs, + ); + }), + ]); + } finally { + if (timer !== undefined) clearTimeout(timer); + } +} + +async function exportRootfsWhenQuiescent( + kernel: BrowserKernel, + timeoutMs = 5_000, +): Promise { + const deadline = Date.now() + timeoutMs; + for (;;) { + try { + return await kernel.exportRootfsImage(); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if ( + !message.includes("no live or tearing-down processes") || + Date.now() >= deadline + ) { + throw error; + } + // WHY: the public process-exit promise resolves when the worker reports + // exit, before the worker-owned teardown promise necessarily settles. + // Retry only that documented transient rejection; the export API remains + // the authority for when the browser kernel is actually quiescent. + await new Promise((resolve) => setTimeout(resolve, 20)); + } + } +} + +function vfsPathExists(fs: MemoryFileSystem, path: string): boolean { + try { + fs.lstat(path); + return true; + } catch { + return false; + } +} + async function init(): Promise { const kernelBytes = await fetchBytes(kernelWasmUrl, "kernel.wasm"); const kernelSha256 = await sha256(kernelBytes); + window.__runHomebrewGuestLifecycleAcceptance = (fixture) => + runHomebrewGuestLifecycleInBrowser({ + fixture, + kernelWasm: kernelBytes, + corsProxyUrl, + afterMachineDestroy: settleWebKitReclaim, + }); + window.__runHomebrewVfsAcceptance = async (request) => { if (!Array.isArray(request.argv) || request.argv.length === 0) { throw new Error("argv must contain at least one entry"); @@ -282,6 +416,243 @@ async function init(): Promise { } }; + window.__runRootfsExportAcceptance = async (request) => { + const initialImage = new Uint8Array( + await fetchBytes(request.vfsUrl, "rootfs export VFS image"), + ); + const liveProcessBytes = await fetchBytes( + request.liveProcessUrl, + "live-process fixture", + ); + const teardownProcessBytes = await fetchBytes( + request.teardownProcessUrl, + "teardown-process fixture", + ); + const diagnostics: Array<{ source: string; message: string }> = []; + let firstKernel: BrowserKernel | null = new BrowserKernel({ + kernelOwnedFs: true, + onHostDiagnostic: (diagnostic) => { + diagnostics.push({ + source: diagnostic.source, + message: diagnostic.message, + }); + }, + }); + let firstExport: Uint8Array; + let liveProcessExitCode: number; + let liveProcessExportError: string; + let teardownProcessExitCode: number; + let teardownExportError: string; + let overlappingExportError: string; + let overlappingWriteError: string; + let lazyReadText: string; + let writeAfterExportText: string; + try { + await firstKernel.initFromImage({ + kernelWasm: kernelBytes, + vfsImage: initialImage, + }); + await firstKernel.writeFileToVfs( + request.writePath, + new TextEncoder().encode(request.writeText), + 0o640, + ); + + let resolveLivePid!: (pid: number) => void; + let rejectLivePid!: (error: unknown) => void; + const livePid = new Promise((resolve, reject) => { + resolveLivePid = resolve; + rejectLivePid = reject; + }); + const liveExit = firstKernel.spawn( + liveProcessBytes, + ["block-forever"], + { + onStarted: resolveLivePid, + }, + ).catch((error) => { + rejectLivePid(error); + throw error; + }); + const pid = await withTimeout( + livePid, + "live process start", + ); + liveProcessExportError = await withTimeout( + rejectionMessage( + firstKernel.exportRootfsImage(), + "rootfs export with a live process", + ), + "live-process rootfs rejection", + ); + await firstKernel.terminateProcess(pid, 143); + liveProcessExitCode = await withTimeout( + liveExit, + "live process termination", + ); + + teardownProcessExitCode = await withTimeout( + firstKernel.spawn( + teardownProcessBytes, + ["thread-exit-group"], + ), + "threaded process exit", + ); + // WHY: thread-exit-group exits from its child thread. The public exit + // promise resolves before the browser worker's tracked 250 ms thread and + // process-worker teardown settles, giving this request a deterministic + // real teardown window without exposing an internal test hook. + teardownExportError = await withTimeout( + rejectionMessage( + firstKernel.exportRootfsImage(), + "rootfs export during process-worker teardown", + ), + "teardown rootfs rejection", + ); + await exportRootfsWhenQuiescent(firstKernel); + + let resolveLazyStart!: () => void; + const lazyStarted = new Promise((resolve) => { + resolveLazyStart = resolve; + }); + const unsubscribeLazy = firstKernel.subscribeLazyDownloads((event) => { + if ( + event.url === request.lazyReadUrl && + event.status === "started" + ) { + resolveLazyStart(); + } + }); + const lazyRead = firstKernel.readFileFromVfs(request.lazyReadPath); + let gatedExport: Promise | undefined; + try { + await withTimeout(lazyStarted, "lazy rootfs read start"); + // WHY: the lazy read has entered the worker's mutation gate but its + // routed response is deliberately held by Playwright. FIFO worker + // messages make the first export close the gate while it waits for + // that read; the following export and write must therefore reject. + gatedExport = firstKernel.exportRootfsImage(); + [overlappingExportError, overlappingWriteError] = await withTimeout( + Promise.all([ + rejectionMessage( + firstKernel.exportRootfsImage(), + "overlapping rootfs export", + ), + rejectionMessage( + firstKernel.writeFileToVfs( + request.lateWritePath, + new TextEncoder().encode(request.lateWriteText), + 0o640, + ), + "rootfs write during export", + ), + ]), + "rootfs export exclusion", + ); + } finally { + unsubscribeLazy(); + // The callback is Playwright transport coordination only. It releases + // the real fetch used by MemoryFileSystem; it does not mutate worker + // state or bypass the production snapshot gate. + await window.__releaseRootfsExportLazyResponse(); + } + const lazyBytes = await withTimeout( + lazyRead, + "lazy rootfs read completion", + ); + if (lazyBytes === null) { + throw new Error(`lazy rootfs read lost ${request.lazyReadPath}`); + } + lazyReadText = new TextDecoder().decode(lazyBytes); + if (gatedExport === undefined) { + throw new Error("rootfs export exclusion did not start an export"); + } + firstExport = await withTimeout( + gatedExport, + "rootfs export after lazy mutation", + ); + + await firstKernel.writeFileToVfs( + request.lateWritePath, + new TextEncoder().encode(request.lateWriteText), + 0o640, + ); + const writeAfterExport = await firstKernel.readFileFromVfs( + request.lateWritePath, + ); + if (writeAfterExport === null) { + throw new Error(`post-export write lost ${request.lateWritePath}`); + } + writeAfterExportText = new TextDecoder().decode(writeAfterExport); + } finally { + await firstKernel?.destroy().catch(() => {}); + firstKernel = null; + await settleWebKitReclaim(); + } + + const parsed = MemoryFileSystem.fromImage(firstExport); + const lazyEntries = parsed.exportLazyEntries().map((entry) => ({ + path: entry.path, + url: entry.url, + size: entry.size, + })); + const exportedLazyRead = new TextDecoder().decode( + readVfsFile(parsed, request.lazyReadPath), + ); + if (exportedLazyRead !== request.lazyReadText) { + throw new Error( + `exported rootfs changed ${request.lazyReadPath}`, + ); + } + const lateWritePresentInExport = vfsPathExists( + parsed, + request.lateWritePath, + ); + + let secondKernel: BrowserKernel | null = new BrowserKernel({ + kernelOwnedFs: true, + onHostDiagnostic: (diagnostic) => { + diagnostics.push({ + source: diagnostic.source, + message: diagnostic.message, + }); + }, + }); + try { + await secondKernel.initFromImage({ + kernelWasm: kernelBytes, + vfsImage: firstExport, + }); + const persisted = await secondKernel.readFileFromVfs(request.writePath); + if (persisted === null) { + throw new Error(`exported rootfs lost ${request.writePath}`); + } + const secondExport = await secondKernel.exportRootfsImage(); + return { + persistedText: new TextDecoder().decode(persisted), + firstExportSha256: await sha256(firstExport), + secondExportSha256: await sha256(secondExport), + firstExportBytes: firstExport.byteLength, + secondExportBytes: secondExport.byteLength, + liveProcessExitCode, + liveProcessExportError, + teardownProcessExitCode, + teardownExportError, + overlappingExportError, + overlappingWriteError, + lazyReadText, + lateWritePresentInExport, + writeAfterExportText, + diagnostics, + lazyEntries, + }; + } finally { + await secondKernel?.destroy().catch(() => {}); + secondKernel = null; + await settleWebKitReclaim(); + } + }; + window.__destroyPackageLayerAcceptance = async () => { const machine = packageLayerMachine; packageLayerMachine = null; diff --git a/apps/browser-demos/pages/kandelo/kernel-host/live-setup.ts b/apps/browser-demos/pages/kandelo/kernel-host/live-setup.ts index 98782a9837..c0322294c7 100644 --- a/apps/browser-demos/pages/kandelo/kernel-host/live-setup.ts +++ b/apps/browser-demos/pages/kandelo/kernel-host/live-setup.ts @@ -23,6 +23,10 @@ import { import { MemoryFileSystem } from "../../../../../host/src/vfs/memory-fs"; import { loadHomebrewBottleMirrorClosedAssets } from "../../../../../host/src/homebrew-bottle-mirror-browser"; import { HOMEBREW_BOTTLE_MIRROR_PLAN_VFS_PATH } from "../../../../../host/src/homebrew-bottle-mirror-plan"; +import { + loadClosedLazyAssetSources, + type ClosedLazyAsset, +} from "../../../../../host/src/vfs/closed-lazy-assets"; import { composeBootDescriptorVfs, homebrewRuntimeLayerReferences, @@ -49,9 +53,7 @@ import { type DemoPresentation, type GalleryItem, } from "../../../../../web-libs/kandelo-session/src/kernel-host"; -import { - validateBootDescriptor, -} from "../../../../../web-libs/kandelo-session/src/boot-descriptor"; +import { validateBootDescriptor } from "../../../../../web-libs/kandelo-session/src/boot-descriptor"; import { genericDemoPresentation, resolveDemoAssets, @@ -60,9 +62,7 @@ import { type DemoAssetConfig, type KandeloDemoConfig, } from "../../../../../web-libs/kandelo-session/src/demo-config"; -import { - readKandeloDemoConfigFromVfs, -} from "../../../../../web-libs/kandelo-session/src/demo-config-vfs"; +import { readKandeloDemoConfigFromVfs } from "../../../../../web-libs/kandelo-session/src/demo-config-vfs"; import { KANDELO_SHELL_CONFIG_PATH, MAX_KANDELO_SHELL_CONFIG_BYTES, @@ -108,27 +108,51 @@ const DEFAULT_SOFTWARE_MANIFEST_URLS = [ ]; const OPTIONAL_BINARY_URLS = { - ...import.meta.glob("../../../../../local-binaries/programs/wasm32/fbtest.wasm", { - query: "?url", import: "default", - }), + ...import.meta.glob( + "../../../../../local-binaries/programs/wasm32/fbtest.wasm", + { + query: "?url", + import: "default", + }, + ), ...import.meta.glob("../../../../../binaries/programs/wasm32/fbtest.wasm", { - query: "?url", import: "default", - }), - ...import.meta.glob("../../../../../local-binaries/programs/wasm32/nginx-vfs.vfs.zst", { - query: "?url", import: "default", - }), - ...import.meta.glob("../../../../../binaries/programs/wasm32/nginx-vfs.vfs.zst", { - query: "?url", import: "default", - }), - ...import.meta.glob("../../../../../local-binaries/programs/wasm32/nginx-php-vfs.vfs.zst", { - query: "?url", import: "default", - }), - ...import.meta.glob("../../../../../binaries/programs/wasm32/nginx-php-vfs.vfs.zst", { - query: "?url", import: "default", + query: "?url", + import: "default", }), + ...import.meta.glob( + "../../../../../local-binaries/programs/wasm32/nginx-vfs.vfs.zst", + { + query: "?url", + import: "default", + }, + ), + ...import.meta.glob( + "../../../../../binaries/programs/wasm32/nginx-vfs.vfs.zst", + { + query: "?url", + import: "default", + }, + ), + ...import.meta.glob( + "../../../../../local-binaries/programs/wasm32/nginx-php-vfs.vfs.zst", + { + query: "?url", + import: "default", + }, + ), + ...import.meta.glob( + "../../../../../binaries/programs/wasm32/nginx-php-vfs.vfs.zst", + { + query: "?url", + import: "default", + }, + ), } as Record Promise>; -async function optionalBinaryUrl(relPaths: string[], label: string): Promise { +async function optionalBinaryUrl( + relPaths: string[], + label: string, +): Promise { for (const relPath of relPaths) { const loader = OPTIONAL_BINARY_URLS[relPath]; if (loader) return loader(); @@ -226,12 +250,7 @@ class BootSuperseded extends Error { } type LiveVfsImage = - | "shell" - | "node" - | "nginx" - | "nginx-php" - | "wordpress" - | "lamp"; + "shell" | "node" | "nginx" | "nginx-php" | "wordpress" | "lamp"; type LiveVfsSource = | { kind: "url"; url: string } @@ -291,7 +310,13 @@ const VFS_SOURCES: Record = { lamp: { kind: "optional-demo", image: "lamp" }, }; -const DINIT_NGINX_ARGV = ["/sbin/dinit", "--container", "-p", "/tmp/dinitctl", "nginx"]; +const DINIT_NGINX_ARGV = [ + "/sbin/dinit", + "--container", + "-p", + "/tmp/dinitctl", + "nginx", +]; const LIVE_DEMO_IDS = [ "shell", @@ -304,7 +329,7 @@ const LIVE_DEMO_IDS = [ "modeset", ] as const; -type LiveDemoId = typeof LIVE_DEMO_IDS[number]; +type LiveDemoId = (typeof LIVE_DEMO_IDS)[number]; // Boot-resource reclamation (worker-owned live filesystems and transient // image-build buffers) lives in the shared helper so every kernel-owned demo @@ -521,7 +546,11 @@ const SHELL_PROFILES: Record = { const INIT_ENV_PROFILES: Record string[]> = { service: () => SERVICE_ENV, - wordpress: () => [...SERVICE_ENV, `WP_APP_PATH=${APP_PATH}`, `WP_PROTO=${PROTO}`], + wordpress: () => [ + ...SERVICE_ENV, + `WP_APP_PATH=${APP_PATH}`, + `WP_PROTO=${PROTO}`, + ], }; export type FbDemo = "none" | "test"; @@ -532,7 +561,9 @@ export interface CreateLiveHostOptions { fb?: FbDemo; } -export async function createLiveHost(opts: CreateLiveHostOptions = {}): Promise { +export async function createLiveHost( + opts: CreateLiveHostOptions = {}, +): Promise { let currentKernel: BrowserKernel | null = null; let bootSeq = 0; let serviceWorkerReady: Promise | null = null; @@ -548,7 +579,9 @@ export async function createLiveHost(opts: CreateLiveHostOptions = {}): Promise< }, }); - const requireServiceWorker = (tick?: (msg: string) => void): Promise => { + const requireServiceWorker = ( + tick?: (msg: string) => void, + ): Promise => { if (!serviceWorkerReady) { tick?.("preparing service worker..."); serviceWorkerReady = ensureServiceWorkerReady(SW_URL) @@ -562,18 +595,22 @@ export async function createLiveHost(opts: CreateLiveHostOptions = {}): Promise< sessionStorage.removeItem(COI_RELOAD_SESSION_KEY); throw new Error( "Kandelo could not enable cross-origin isolation after the service worker became active. " + - "Reload the page; if this persists, clear site data for this site and check whether a browser extension is blocking service workers or COOP/COEP headers.", + "Reload the page; if this persists, clear site data for this site and check whether a browser extension is blocking service workers or COOP/COEP headers.", ); } sessionStorage.setItem(COI_RELOAD_SESSION_KEY, "1"); - tick?.("service worker active; reloading to enable cross-origin isolation..."); + tick?.( + "service worker active; reloading to enable cross-origin isolation...", + ); window.location.reload(); return new Promise((_, reject) => { window.setTimeout(() => { - reject(new Error( - "Kandelo requested a reload to enable cross-origin isolation, but the page did not unload.", - )); + reject( + new Error( + "Kandelo requested a reload to enable cross-origin isolation, but the page did not unload.", + ), + ); }, 5_000); }); }) @@ -584,12 +621,18 @@ export async function createLiveHost(opts: CreateLiveHostOptions = {}): Promise< } const ready = serviceWorkerReady; if (!ready) { - throw new Error("Kandelo service worker readiness promise was not initialized."); + throw new Error( + "Kandelo service worker readiness promise was not initialized.", + ); } return ready; }; - void startBoot(host, profileForDescriptor(initialDescriptor, opts.fb), initialDescriptor); + void startBoot( + host, + profileForDescriptor(initialDescriptor, opts.fb), + initialDescriptor, + ); void requireServiceWorker() .then(() => refreshSoftwareGallery(host, localGalleryItems)) .catch((err) => { @@ -694,17 +737,21 @@ function descriptorForBootQuery( const liveId = liveDemoIdForVfsImageUrl(normalizedVfsUrl); const base = descriptorFor(liveId ?? "shell"); - return descriptorWithVfsImageUrl(base, normalizedVfsUrl, liveId - ? { - id: liveId, - title: base.title, - packages: base.packages, - } - : { - id: demoIdFromVfsImageUrl(normalizedVfsUrl), - title: titleFromVfsImageUrl(normalizedVfsUrl), - packages: [], - }); + return descriptorWithVfsImageUrl( + base, + normalizedVfsUrl, + liveId + ? { + id: liveId, + title: base.title, + packages: base.packages, + } + : { + id: demoIdFromVfsImageUrl(normalizedVfsUrl), + title: titleFromVfsImageUrl(normalizedVfsUrl), + packages: [], + }, + ); } function profileForDescriptor(desc: BootDescriptor, fb?: FbDemo): LiveProfile { @@ -774,7 +821,8 @@ function profileFor(id: string, fb?: FbDemo): LiveProfile { descriptor: desc, shell: spec.shell ?? "default", includeNodeUtility: spec.includeNodeUtility ?? false, - maxVfsByteLength: spec.maxVfsByteLength ?? + maxVfsByteLength: + spec.maxVfsByteLength ?? (spec.image === "shell" ? MAIN_SHELL_VFS_PROFILE_MAX_BYTES : DEFAULT_VFS_PROFILE_MAX_BYTES), @@ -813,7 +861,10 @@ function shellCwdFor(profile: ShellProfile): string { return SHELL_PROFILES[profile].cwd; } -function shellIdentityForProfile(profile: LiveProfile, boot?: BootDescriptor["boot"]): { +function shellIdentityForProfile( + profile: LiveProfile, + boot?: BootDescriptor["boot"], +): { env: string[]; cwd: string; uid: number; @@ -821,11 +872,29 @@ function shellIdentityForProfile(profile: LiveProfile, boot?: BootDescriptor["bo } { let identity: { env: string[]; cwd: string; uid: number; gid: number }; if (profile.shell === "node") { - identity = { env: shellEnvFor(profile.shell), cwd: shellCwdFor(profile.shell), uid: DEMO_UID, gid: DEMO_GID }; - } else if (profile.software?.shellEnv && profile.software.shellEnv !== SERVICE_ENV) { - identity = { env: profile.software.shellEnv, cwd: DEMO_HOME, uid: DEMO_UID, gid: DEMO_GID }; + identity = { + env: shellEnvFor(profile.shell), + cwd: shellCwdFor(profile.shell), + uid: DEMO_UID, + gid: DEMO_GID, + }; + } else if ( + profile.software?.shellEnv && + profile.software.shellEnv !== SERVICE_ENV + ) { + identity = { + env: profile.software.shellEnv, + cwd: DEMO_HOME, + uid: DEMO_UID, + gid: DEMO_GID, + }; } else { - identity = { env: shellEnvFor(profile.shell), cwd: shellCwdFor(profile.shell), uid: DEMO_UID, gid: DEMO_GID }; + identity = { + env: shellEnvFor(profile.shell), + cwd: shellCwdFor(profile.shell), + uid: DEMO_UID, + gid: DEMO_GID, + }; } if (!boot) return identity; return { @@ -875,8 +944,8 @@ function presentationForProfile( runningPrimary: [ "web", "syslog", - ...presentation.runningPrimary.filter((surface) => - surface !== "web" && surface !== "syslog" + ...presentation.runningPrimary.filter( + (surface) => surface !== "web" && surface !== "syslog", ), ], }; @@ -912,9 +981,15 @@ class DinitBootStatusTracker { observeProcessOutput(text: string, stream: string): void { if (!text) return; - const normalized = `${this.outputTails.get(stream) ?? ""}${text}`.replace(/\r/g, ""); + const normalized = `${this.outputTails.get(stream) ?? ""}${text}`.replace( + /\r/g, + "", + ); const lines = normalized.split("\n"); - this.outputTails.set(stream, text.endsWith("\n") ? "" : lines.pop() ?? ""); + this.outputTails.set( + stream, + text.endsWith("\n") ? "" : (lines.pop() ?? ""), + ); for (const line of lines) { const serviceName = parseDinitCompletionLine(line); if (!serviceName) continue; @@ -944,7 +1019,9 @@ class DinitBootStatusTracker { } function parseDinitCompletionLine(line: string): string | null { - const match = stripAnsi(line).trim().match(/^\[(?:\s*OK\s*|FAILED)\]\s+(.+)$/); + const match = stripAnsi(line) + .trim() + .match(/^\[(?:\s*OK\s*|FAILED)\]\s+(.+)$/); return match?.[1]?.trim() || null; } @@ -1000,17 +1077,16 @@ function startDinitStartingPoller(options: { async function readDinitctlList(kernel: BrowserKernel): Promise { const chunks: Uint8Array[] = []; - const { pid, exit } = await kernel.spawnFromVfs(DINITCTL_PATH, [ + const { pid, exit } = await kernel.spawnFromVfs( DINITCTL_PATH, - "-p", - DINITCTL_SOCKET_PATH, - "list", - ], { - cwd: "/", - uid: ROOT_UID, - gid: ROOT_GID, - pty: true, - }); + [DINITCTL_PATH, "-p", DINITCTL_SOCKET_PATH, "list"], + { + cwd: "/", + uid: ROOT_UID, + gid: ROOT_GID, + pty: true, + }, + ); kernel.onPtyOutput(pid, (data) => { chunks.push(data.slice()); }); @@ -1062,7 +1138,9 @@ async function bootProfile( requestedDescriptor: BootDescriptor, bootStartedAt: number, isCurrent: () => boolean, - requireServiceWorker: (tick?: (msg: string) => void) => Promise, + requireServiceWorker: ( + tick?: (msg: string) => void, + ) => Promise, ): Promise { const assertCurrent = () => { if (!isCurrent()) throw new BootSuperseded(); @@ -1084,19 +1162,26 @@ async function bootProfile( host.setDescriptor({ ...profile.descriptor, title: requestedDescriptor.title || profile.descriptor.title, - packages: requestedDescriptor.packages.length > 0 - ? requestedDescriptor.packages - : profile.descriptor.packages, + packages: + requestedDescriptor.packages.length > 0 + ? requestedDescriptor.packages + : profile.descriptor.packages, mounts: requestedDescriptor.mounts, boot: effectiveBoot, }); - const genericPresentation = profile.fallbackPresentation ?? genericPresentationForProfile(profile); + const genericPresentation = + profile.fallbackPresentation ?? genericPresentationForProfile(profile); host.setPresentation(genericPresentation); host.setStatus("booting"); const tick = (msg: string) => { if (!isCurrent()) return; - host.pushDmesg({ t: bootElapsedMs(bootStartedAt), level: "info", facility: "kandelo", msg }); + host.pushDmesg({ + t: bootElapsedMs(bootStartedAt), + level: "info", + facility: "kandelo", + msg, + }); }; let maybeUpdateWebReadiness = () => {}; const dinitBootTracker = new DinitBootStatusTracker(tick, () => { @@ -1114,13 +1199,17 @@ async function bootProfile( tick("service worker active and cross-origin isolated"); tick(`loading ${profile.id} profile...`); const [kernelBytes, vfsBytes, softwareBinaries] = await Promise.all([ - fetch(kernelWasmUrl).then(failOn("kernel.wasm")).then((r) => r.arrayBuffer()), + fetch(kernelWasmUrl) + .then(failOn("kernel.wasm")) + .then((r) => r.arrayBuffer()), loadVfsImageBytes(profile), loadSoftwareBinaries(profile.software), ]); assertCurrent(); - tick(`kernel: ${kib(kernelBytes.byteLength)} · vfs: ${kib(vfsBytes.byteLength)}`); + tick( + `kernel: ${kib(kernelBytes.byteLength)} · vfs: ${kib(vfsBytes.byteLength)}`, + ); const fetchedVfsImageBytes = new Uint8Array(vfsBytes); const vfsMetadata = MemoryFileSystem.readImageMetadata(fetchedVfsImageBytes); assertVfsImageFitsProfile( @@ -1144,9 +1233,11 @@ async function bootProfile( const runtimeLayers = homebrewRuntimeLayerReferences(requestedDescriptor); let buildFs: MemoryFileSystem; if (runtimeLayers.length > 0) { - tick(`verifying ${runtimeLayers.length} selected runtime layer${ - runtimeLayers.length === 1 ? "" : "s" - }...`); + tick( + `verifying ${runtimeLayers.length} selected runtime layer${ + runtimeLayers.length === 1 ? "" : "s" + }...`, + ); const composed = await composeBootDescriptorVfs({ descriptor: requestedDescriptor, baseImageBytes: fetchedVfsImageBytes, @@ -1156,7 +1247,9 @@ async function bootProfile( }); buildFs = composed.fs; assertCurrent(); - tick("runtime layer files registered; archives remain lazy until first use"); + tick( + "runtime layer files registered; archives remain lazy until first use", + ); } else { buildFs = MemoryFileSystem.fromImage(fetchedVfsImageBytes, { maxByteLength: profile.maxVfsByteLength, @@ -1204,8 +1297,12 @@ async function bootProfile( assertImageShellExecutable(buildFs, shellConfig.path); } else { const [bashBytes, dashBytes] = await Promise.all([ - fetch(bashWasmUrl).then(failOn("bash.wasm")).then((r) => r.arrayBuffer()), - fetch(dashWasmUrl).then(failOn("dash.wasm")).then((r) => r.arrayBuffer()), + fetch(bashWasmUrl) + .then(failOn("bash.wasm")) + .then((r) => r.arrayBuffer()), + fetch(dashWasmUrl) + .then(failOn("dash.wasm")) + .then((r) => r.arrayBuffer()), ]); assertCurrent(); stageShellUtilities(buildFs, dashBytes, bashBytes); @@ -1214,16 +1311,21 @@ async function bootProfile( stageSoftwareBinaries(buildFs, softwareBinaries); const hasDinitctl = vfsPathExists(buildFs, DINITCTL_PATH); const imageConfig = readImageConfig(buildFs); - const rawPresentation = (imageConfig ? resolveDemoPresentation(imageConfig, profile.id) : null) - ?? builtinDemoPresentation(profile.id) - ?? genericPresentation; + const rawPresentation = + (imageConfig ? resolveDemoPresentation(imageConfig, profile.id) : null) ?? + builtinDemoPresentation(profile.id) ?? + genericPresentation; const presentation = presentationForProfile(profile, rawPresentation); host.setPresentation(presentation); - const demoGuide = (imageConfig ? resolveDemoGuide(imageConfig, profile.id) : null) - ?? builtinDemoGuide(profile.id); + const demoGuide = + (imageConfig ? resolveDemoGuide(imageConfig, profile.id) : null) ?? + builtinDemoGuide(profile.id); host.setDemoGuide(demoGuide); - const imageAssets = imageConfig ? resolveDemoAssets(imageConfig, profile.id) : []; - const assets = imageAssets.length > 0 ? imageAssets : builtinDemoAssets(profile.id); + const imageAssets = imageConfig + ? resolveDemoAssets(imageConfig, profile.id) + : []; + const assets = + imageAssets.length > 0 ? imageAssets : builtinDemoAssets(profile.id); await stageConfiguredAssets(buildFs, assets, tick); assertCurrent(); @@ -1278,17 +1380,20 @@ async function bootProfile( msg: diagnostic.message, }); }, - onProcessEvent: (event) => { if (isCurrent()) host.emitProcessEvent(event); }, + onProcessEvent: (event) => { + if (isCurrent()) host.emitProcessEvent(event); + }, onHttpBridgePendingRequests: (count) => { if (isCurrent()) host.setWebPreviewPendingRequests(count); }, onListenTcp: (pid, _fd, port) => { if (!isCurrent()) return; seenPorts.add(port); - void reportTcpListener(kernel!, pid, port, tick, isCurrent) - .finally(() => { + void reportTcpListener(kernel!, pid, port, tick, isCurrent).finally( + () => { maybeUpdateWebReadiness(); - }); + }, + ); }, }); await kernel.initFromImage({ @@ -1298,7 +1403,10 @@ async function bootProfile( }); assertCurrent(); host.attachKernel(kernel); - const shellIdentity = shellIdentityForProfile(profile, profile.init ? undefined : effectiveBoot); + const shellIdentity = shellIdentityForProfile( + profile, + profile.init ? undefined : effectiveBoot, + ); host.setDefaultShell({ programPath: shellConfig?.path ?? "/bin/bash", ...(shellProgramBytes ? { programBytes: shellProgramBytes } : {}), @@ -1323,13 +1431,20 @@ async function bootProfile( // fresh random id per boot; when machines become persistable this is // where their durable id would be passed instead. const sessionId = crypto.randomUUID(); - await setupServiceWorkerFetchBridge(SW_URL, APP_PREFIX, kernel, HTTP_PORT, sessionId, { - timeoutMs: 90_000, - debugLog: (line) => tick(line), - onPendingRequests: (count) => { - if (isCurrent()) host.setWebPreviewPendingRequests(count); + await setupServiceWorkerFetchBridge( + SW_URL, + APP_PREFIX, + kernel, + HTTP_PORT, + sessionId, + { + timeoutMs: 90_000, + debugLog: (line) => tick(line), + onPendingRequests: (count) => { + if (isCurrent()) host.setWebPreviewPendingRequests(count); + }, }, - }); + ); assertCurrent(); bridgeSent = true; maybeUpdateWebReadiness(); @@ -1347,17 +1462,25 @@ async function bootProfile( } if (profile.init) { - const initArgv = effectiveBoot.argv.length > 0 ? effectiveBoot.argv : profile.init.argv; + const initArgv = + effectiveBoot.argv.length > 0 ? effectiveBoot.argv : profile.init.argv; tick(`spawning ${initArgv[0]}...`); // The init binary lives in the kernel-owned VFS; spawn it by path rather // than shipping bytes the kernel already has. - const { exit: initExit } = await kernel.spawnFromVfs(initArgv[0], initArgv, { - env: mergeEnvArrays(profile.init.env ?? [], envArray(effectiveBoot.env)), - cwd: effectiveBoot.cwd || profile.init.cwd || ROOT_HOME, - uid: effectiveBoot.uid ?? profile.init.uid ?? ROOT_UID, - gid: effectiveBoot.gid ?? profile.init.gid ?? ROOT_GID, - stdin: new Uint8Array(), - }); + const { exit: initExit } = await kernel.spawnFromVfs( + initArgv[0], + initArgv, + { + env: mergeEnvArrays( + profile.init.env ?? [], + envArray(effectiveBoot.env), + ), + cwd: effectiveBoot.cwd || profile.init.cwd || ROOT_HOME, + uid: effectiveBoot.uid ?? profile.init.uid ?? ROOT_UID, + gid: effectiveBoot.gid ?? profile.init.gid ?? ROOT_GID, + stdin: new Uint8Array(), + }, + ); stopDinitStartingPoller = startDinitStartingPoller({ kernel, hasDinitctl, @@ -1394,20 +1517,33 @@ async function bootProfile( maybeUpdateWebReadiness(); if (profile.framebufferTest) { - const fbtestWasmUrl = await optionalBinaryUrl([ - "../../../../../local-binaries/programs/wasm32/fbtest.wasm", - "../../../../../binaries/programs/wasm32/fbtest.wasm", - ], "fbtest.wasm"); - void spawnLazy(kernel, "/usr/local/bin/fbtest", fbtestWasmUrl, ["fbtest"], tick); + const fbtestWasmUrl = await optionalBinaryUrl( + [ + "../../../../../local-binaries/programs/wasm32/fbtest.wasm", + "../../../../../binaries/programs/wasm32/fbtest.wasm", + ], + "fbtest.wasm", + ); + void spawnLazy( + kernel, + "/usr/local/bin/fbtest", + fbtestWasmUrl, + ["fbtest"], + tick, + ); } else if (presentation?.autoCommand) { tick("starting configured command from the default shell..."); void host.runShellCommand(presentation.autoCommand).catch((err) => { - tick(`configured command failed: ${err instanceof Error ? err.message : String(err)}`); + tick( + `configured command failed: ${err instanceof Error ? err.message : String(err)}`, + ); }); } else if (profile.autoCommand) { tick(`running ${profile.autoCommand}...`); void host.runShellCommand(profile.autoCommand).catch((err) => { - tick(`command failed: ${err instanceof Error ? err.message : String(err)}`); + tick( + `command failed: ${err instanceof Error ? err.message : String(err)}`, + ); }); } @@ -1428,7 +1564,10 @@ function genericPresentationForProfile(profile: LiveProfile): DemoPresentation { if (profile.descriptor.runtime.features.includes("kms")) { return genericDemoPresentation("kms"); } - if (profile.framebufferTest || profile.descriptor.runtime.features.includes("framebuffer")) { + if ( + profile.framebufferTest || + profile.descriptor.runtime.features.includes("framebuffer") + ) { return genericDemoPresentation("framebuffer"); } return genericDemoPresentation("terminal"); @@ -1443,11 +1582,27 @@ function stageShellUtilities( ensureDirRecursive(fs, "/bin"); ensureDirRecursive(fs, "/usr/bin"); writeVfsBinary(fs, "/bin/dash", new Uint8Array(dashBytes), 0o755); - try { fs.symlink("/bin/dash", "/bin/sh"); } catch { /* exists */ } - try { fs.symlink("/bin/dash", "/usr/bin/dash"); } catch { /* exists */ } - try { fs.symlink("/bin/dash", "/usr/bin/sh"); } catch { /* exists */ } + try { + fs.symlink("/bin/dash", "/bin/sh"); + } catch { + /* exists */ + } + try { + fs.symlink("/bin/dash", "/usr/bin/dash"); + } catch { + /* exists */ + } + try { + fs.symlink("/bin/dash", "/usr/bin/sh"); + } catch { + /* exists */ + } writeVfsBinary(fs, "/bin/bash", new Uint8Array(bashBytes), 0o755); - try { fs.symlink("/bin/bash", "/usr/bin/bash"); } catch { /* exists */ } + try { + fs.symlink("/bin/bash", "/usr/bin/bash"); + } catch { + /* exists */ + } } function rewriteNodeLazyFileUrl(fs: MemoryFileSystem): void { @@ -1482,16 +1637,34 @@ function patchWordPressRuntimeConfig( kind: WordPressDatabaseKind, ): void { writeVfsFile(fs, "/etc/wp-config-init.sh", WORDPRESS_CONFIG_INIT_SCRIPT); - writeVfsFile(fs, "/etc/wp-config-template.php", wordpressConfigTemplate(kind)); - writeVfsFile(fs, "/var/www/html/wp-config.php", renderWordPressConfig(kind, APP_PATH, PROTO)); + writeVfsFile( + fs, + "/etc/wp-config-template.php", + wordpressConfigTemplate(kind), + ); + writeVfsFile( + fs, + "/var/www/html/wp-config.php", + renderWordPressConfig(kind, APP_PATH, PROTO), + ); if (kind === "sqlite") { - ensureOwnedDir(fs, "/var/www/html/wp-content/database", 0o775, PHP_FPM_UID, PHP_FPM_GID); + ensureOwnedDir( + fs, + "/var/www/html/wp-content/database", + 0o775, + PHP_FPM_UID, + PHP_FPM_GID, + ); } else if (kind === "mariadb") { for (const dir of ["/data", "/data/mysql", "/data/tmp", "/data/test"]) { ensureOwnedDir(fs, dir, 0o775, MYSQL_UID, MYSQL_GID); } patchWordPressPersistentMysqli(fs); - writeVfsFile(fs, "/var/www/html/kandelo-mysql-bench.php", MYSQL_BENCHMARK_PHP); + writeVfsFile( + fs, + "/var/www/html/kandelo-mysql-bench.php", + MYSQL_BENCHMARK_PHP, + ); } ensureDirRecursive(fs, "/var/www/html/wp-content/mu-plugins"); writeVfsFile( @@ -1532,7 +1705,8 @@ function patchMariaDbUnixSocketConfig(fs: MemoryFileSystem): void { const patched = mariadbService .replace(/--socket=(?:\S*)?/g, `--socket=${MARIADB_SOCKET_PATH}`) .replace(/\s*--thread-handling=no-threads\b/g, ""); - if (patched !== mariadbService) writeVfsFile(fs, mariadbServicePath, patched); + if (patched !== mariadbService) + writeVfsFile(fs, mariadbServicePath, patched); } ensureMariaDbReadyService(fs); @@ -1541,7 +1715,10 @@ function patchMariaDbUnixSocketConfig(fs: MemoryFileSystem): void { function ensureMariaDbReadyService(fs: MemoryFileSystem): void { ensureDirRecursive(fs, dirname(MARIADB_READY_SCRIPT_PATH)); - writeVfsFile(fs, MARIADB_READY_SCRIPT_PATH, `#!/bin/sh + writeVfsFile( + fs, + MARIADB_READY_SCRIPT_PATH, + `#!/bin/sh set -u i=0 @@ -1555,19 +1732,29 @@ done echo "MariaDB readiness timed out waiting for ${MARIADB_SOCKET_PATH}" >&2 exit 1 -`, 0o755); - writeVfsFile(fs, `/etc/dinit.d/${MARIADB_READY_SERVICE}`, `type = scripted +`, + 0o755, + ); + writeVfsFile( + fs, + `/etc/dinit.d/${MARIADB_READY_SERVICE}`, + `type = scripted command = /bin/sh ${MARIADB_READY_SCRIPT_PATH} depends-on = mariadb restart = false -`); +`, + ); } function patchPhpFpmMariaDbDependency(fs: MemoryFileSystem): void { const phpFpmServicePath = "/etc/dinit.d/php-fpm"; const phpFpmService = readOptionalVfsText(fs, phpFpmServicePath); if (phpFpmService === null) return; - if (new RegExp(`^depends-on\\s*=\\s*${MARIADB_READY_SERVICE}$`, "m").test(phpFpmService)) { + if ( + new RegExp(`^depends-on\\s*=\\s*${MARIADB_READY_SERVICE}$`, "m").test( + phpFpmService, + ) + ) { return; } const patched = phpFpmService.replace( @@ -1600,7 +1787,9 @@ function patchWordPressPersistentMysqli(fs: MemoryFileSystem): void { async function loadVfsImageBytes(profile: LiveProfile): Promise { if (!profile.software) { const vfsUrl = await resolveProfileVfsUrl(profile); - return fetch(vfsUrl).then(failOn(`${profile.id}.vfs.zst`)).then((r) => r.arrayBuffer()); + return fetch(vfsUrl) + .then(failOn(`${profile.id}.vfs.zst`)) + .then((r) => r.arrayBuffer()); } const vfsImage = await loadArchiveArtifact( profile.software.vfsArchiveUrl, @@ -1617,7 +1806,10 @@ async function resolveProfileVfsUrl(profile: LiveProfile): Promise { return resolveOptionalDemoVfsUrl(profile.vfsSource.image); } if (profile.vfsSource?.kind === "optional-binary") { - return optionalBinaryUrl(profile.vfsSource.relPaths, profile.vfsSource.label); + return optionalBinaryUrl( + profile.vfsSource.relPaths, + profile.vfsSource.label, + ); } if (profile.vfsUrl) return profile.vfsUrl; throw new Error(`No VFS image URL configured for ${profile.id}`); @@ -1627,10 +1819,12 @@ async function loadSoftwareBinaries( software: SoftwareProfile | undefined, ): Promise> { if (!software) return []; - return Promise.all(software.binaries.map(async (spec) => ({ - spec, - bytes: await loadArchiveArtifact(spec.archiveUrl, spec.artifactPath), - }))); + return Promise.all( + software.binaries.map(async (spec) => ({ + spec, + bytes: await loadArchiveArtifact(spec.archiveUrl, spec.artifactPath), + })), + ); } function stageSoftwareBinaries( @@ -1642,7 +1836,11 @@ function stageSoftwareBinaries( writeVfsBinary(fs, spec.installPath, bytes, 0o755); for (const symlinkPath of spec.symlinks ?? []) { ensureDirRecursive(fs, dirname(symlinkPath)); - try { fs.symlink(spec.installPath, symlinkPath); } catch { /* exists */ } + try { + fs.symlink(spec.installPath, symlinkPath); + } catch { + /* exists */ + } } } } @@ -1664,7 +1862,10 @@ async function reportTcpListener( tick(`${processName ?? "service"} listening on :${port}`); } -async function processNameForPid(kernel: BrowserKernel, pid: number): Promise { +async function processNameForPid( + kernel: BrowserKernel, + pid: number, +): Promise { if (pid <= 0) return null; const proc = (await kernel.enumProcs()).find((entry) => entry.pid === pid); if (!proc) return null; @@ -1679,7 +1880,10 @@ function basename(path: string): string { return idx < 0 ? path : path.slice(idx + 1); } -async function loadArchiveArtifact(archiveUrl: string, artifactPath: string): Promise { +async function loadArchiveArtifact( + archiveUrl: string, + artifactPath: string, +): Promise { const archiveBytes = await fetchBytesNoStore(archiveUrl); const tarBytes = decompressZstd(archiveBytes); const artifact = extractTarFile(tarBytes, artifactPath); @@ -1689,7 +1893,10 @@ async function loadArchiveArtifact(archiveUrl: string, artifactPath: string): Pr return artifact; } -function extractTarFile(tarBytes: Uint8Array, wantedPath: string): Uint8Array | undefined { +function extractTarFile( + tarBytes: Uint8Array, + wantedPath: string, +): Uint8Array | undefined { for (let offset = 0; offset + 512 <= tarBytes.length;) { const header = tarBytes.subarray(offset, offset + 512); if (header.every((byte) => byte === 0)) return undefined; @@ -1713,7 +1920,9 @@ function extractTarFile(tarBytes: Uint8Array, wantedPath: string): Uint8Array | } function tarString(block: Uint8Array, offset: number, length: number): string { - return tarDecoder.decode(block.subarray(offset, offset + length)).replace(/\0.*$/, ""); + return tarDecoder + .decode(block.subarray(offset, offset + length)) + .replace(/\0.*$/, ""); } async function spawnLazy( @@ -1725,7 +1934,9 @@ async function spawnLazy( ): Promise { try { tick(`fetching ${argv[0]}...`); - const bytes = await fetch(url).then(failOn(argv[0])).then((r) => r.arrayBuffer()); + const bytes = await fetch(url) + .then(failOn(argv[0])) + .then((r) => r.arrayBuffer()); tick(`spawning ${argv[0]}...`); await kernel.spawn(bytes, argv, { env: SHELL_ENV, @@ -1735,7 +1946,9 @@ async function spawnLazy( }); tick(`${argv[0]} exited`); } catch (err) { - tick(`${argv[0]} failed: ${err instanceof Error ? err.message : String(err)}`); + tick( + `${argv[0]} failed: ${err instanceof Error ? err.message : String(err)}`, + ); } } @@ -1753,7 +1966,9 @@ async function stageConfiguredAssets( if (asset.sha256) { const digest = await sha256Hex(buffer); if (digest !== asset.sha256) { - throw new Error(`${asset.path} sha256 mismatch: expected ${asset.sha256}, got ${digest}`); + throw new Error( + `${asset.path} sha256 mismatch: expected ${asset.sha256}, got ${digest}`, + ); } } writeVfsBinary(fs, asset.path, bytes, asset.mode ?? 0o644); @@ -1787,10 +2002,13 @@ function maybeMarkWebReady( const web = profile.init?.web; if (!web) return; const portsReady = web.requiredPorts.every((p) => seenPorts.has(p)); - const servicesReady = (web.requiredServices ?? []) - .every((serviceName) => dinitBootTracker.hasCompleted(serviceName)); + const servicesReady = (web.requiredServices ?? []).every((serviceName) => + dinitBootTracker.hasCompleted(serviceName), + ); if (!portsReady || !servicesReady || !bridgeSent) return; - const readyMessage = web.probeHttp ? "HTTP bridge ready" : "Service stack ready"; + const readyMessage = web.probeHttp + ? "HTTP bridge ready" + : "Service stack ready"; if (readiness.ready) { if (!isCurrent()) return; host.setWebPreview({ @@ -1819,35 +2037,41 @@ function maybeMarkWebReady( label: web.label, url: APP_PREFIX, status: "starting", - message: web.probePath ? "Waiting for application readiness" : "Waiting for HTTP response", + message: web.probePath + ? "Waiting for application readiness" + : "Waiting for HTTP response", }); - void waitForHttpPreview(probeUrl, 90_000, { requireOk: Boolean(web.probePath) }).then( - () => { - if (!isCurrent()) return; - readiness.ready = true; - tick("HTTP preview ready"); - host.setWebPreview({ - label: web.label, - url: APP_PREFIX, - status: "running", - message: "HTTP bridge ready", - }); - }, - (err) => { + void waitForHttpPreview(probeUrl, 90_000, { + requireOk: Boolean(web.probePath), + }) + .then( + () => { + if (!isCurrent()) return; + readiness.ready = true; + tick("HTTP preview ready"); + host.setWebPreview({ + label: web.label, + url: APP_PREFIX, + status: "running", + message: "HTTP bridge ready", + }); + }, + (err) => { + if (!isCurrent()) return; + const message = err instanceof Error ? err.message : String(err); + host.setWebPreview({ + label: web.label, + url: APP_PREFIX, + status: "error", + message: "HTTP preview did not become ready", + }); + tick(`HTTP preview readiness failed: ${message}`); + }, + ) + .finally(() => { if (!isCurrent()) return; - const message = err instanceof Error ? err.message : String(err); - host.setWebPreview({ - label: web.label, - url: APP_PREFIX, - status: "error", - message: "HTTP preview did not become ready", - }); - tick(`HTTP preview readiness failed: ${message}`); - }, - ).finally(() => { - if (!isCurrent()) return; - readiness.probing = false; - }); + readiness.probing = false; + }); } async function waitForHttpPreview( @@ -1880,7 +2104,10 @@ function previewUrlForPath(path: string): string { return new URL(normalized || ".", root).href; } -async function fetchWithTimeout(url: string, timeoutMs: number): Promise { +async function fetchWithTimeout( + url: string, + timeoutMs: number, +): Promise { const controller = new AbortController(); const timer = window.setTimeout(() => controller.abort(), timeoutMs); try { @@ -1903,42 +2130,71 @@ function descriptorBootIdentity( software: SoftwareProfile | undefined, shell: ShellProfile, ): { env: string[]; cwd: string; uid: number; gid: number } { - const serviceIds = new Set(["nginx", "nginx-php", "wordpress-sqlite", "wordpress-mariadb"]); - if (software?.init || serviceIds.has(id) || software?.shellEnv === SERVICE_ENV) { - return { env: software?.shellEnv ?? SERVICE_ENV, cwd: ROOT_HOME, uid: ROOT_UID, gid: ROOT_GID }; + const serviceIds = new Set([ + "nginx", + "nginx-php", + "wordpress-sqlite", + "wordpress-mariadb", + ]); + if ( + software?.init || + serviceIds.has(id) || + software?.shellEnv === SERVICE_ENV + ) { + return { + env: software?.shellEnv ?? SERVICE_ENV, + cwd: ROOT_HOME, + uid: ROOT_UID, + gid: ROOT_GID, + }; } if (id === "node" || shell === "node") { - return { env: shellEnvFor(shell), cwd: shellCwdFor(shell), uid: DEMO_UID, gid: DEMO_GID }; + return { + env: shellEnvFor(shell), + cwd: shellCwdFor(shell), + uid: DEMO_UID, + gid: DEMO_GID, + }; } - return { env: software?.shellEnv ?? shellEnvFor(shell), cwd: shellCwdFor(shell), uid: DEMO_UID, gid: DEMO_GID }; + return { + env: software?.shellEnv ?? shellEnvFor(shell), + cwd: shellCwdFor(shell), + uid: DEMO_UID, + gid: DEMO_GID, + }; } function envRecord(env: string[]): Record { - return Object.fromEntries(env.map((kv) => { - const idx = kv.indexOf("="); - return [kv.slice(0, idx), kv.slice(idx + 1)]; - })); + return Object.fromEntries( + env.map((kv) => { + const idx = kv.indexOf("="); + return [kv.slice(0, idx), kv.slice(idx + 1)]; + }), + ); } function descriptorFor(id: string): BootDescriptor { const software = SOFTWARE_PROFILES.get(id); - const normalized = software ? "shell" : normalizeDemoId(id) ?? "shell"; + const normalized = software ? "shell" : (normalizeDemoId(id) ?? "shell"); const spec = LIVE_PROFILE_SPECS[normalized]; const item = software ? liveGalleryItems().find((p) => p.id === "shell")! - : liveGalleryItems().find((p) => p.id === normalized) ?? liveGalleryItems()[0]; + : (liveGalleryItems().find((p) => p.id === normalized) ?? + liveGalleryItems()[0]); const shell = spec.shell ?? "default"; - const network = software ? false : spec.network ?? false; + const network = software ? false : (spec.network ?? false); const bootIdentity = descriptorBootIdentity(normalized, software, shell); return { version: 1, id: software?.id ?? item.id, - title: software ? software.id.replace(/^kandelo-software-/, "") : item.title, + title: software + ? software.id.replace(/^kandelo-software-/, "") + : item.title, base: software ? `kandelo:shell@abi${ABI_VERSION}` : item.base, runtime: { arch: "wasm32", kernel: "kernel@local", - memoryPages: software ? 4096 : spec.memoryPages ?? 2048, + memoryPages: software ? 4096 : (spec.memoryPages ?? 2048), features: [ "shared-array-buffer", "pty", @@ -1949,11 +2205,20 @@ function descriptorFor(id: string): BootDescriptor { }, packages: software ? [] : item.packages, mounts: [ - { path: "/", source: "image", ref: `${software?.id ?? item.id}.vfs@local`, readonly: false }, + { + path: "/", + source: "image", + ref: `${software?.id ?? item.id}.vfs@local`, + readonly: false, + }, { path: "/tmp", source: "scratch", ephemeral: true }, ], boot: { - argv: software?.init ? software.init.argv : software ? ["bash", "-l", "-i"] : item.bootCommand, + argv: software?.init + ? software.init.argv + : software + ? ["bash", "-l", "-i"] + : item.bootCommand, cwd: bootIdentity.cwd, env: envRecord(bootIdentity.env), uid: bootIdentity.uid, @@ -1997,7 +2262,10 @@ function vfsImageUrlResolverForPreset( const source = VFS_SOURCES[LIVE_PROFILE_SPECS[liveId].image]; if (source.kind !== "optional-demo") return undefined; return async () => { - const url = new URL(await resolveOptionalDemoVfsUrl(source.image), location.href); + const url = new URL( + await resolveOptionalDemoVfsUrl(source.image), + location.href, + ); url.hash = liveId; return url.href; }; @@ -2052,18 +2320,25 @@ async function refreshSoftwareGallery( } async function loadKandeloSoftwareGalleryItems(): Promise { - const groups = await Promise.all(softwareManifestUrls().map(async (manifestUrl) => { - try { - return await loadSoftwareGalleryItemsFromManifest(manifestUrl); - } catch (err) { - console.warn(`Could not load Kandelo software gallery manifest ${manifestUrl}:`, err); - return []; - } - })); + const groups = await Promise.all( + softwareManifestUrls().map(async (manifestUrl) => { + try { + return await loadSoftwareGalleryItemsFromManifest(manifestUrl); + } catch (err) { + console.warn( + `Could not load Kandelo software gallery manifest ${manifestUrl}:`, + err, + ); + return []; + } + }), + ); return groups.flat(); } -async function loadSoftwareGalleryItemsFromManifest(manifestUrl: string): Promise { +async function loadSoftwareGalleryItemsFromManifest( + manifestUrl: string, +): Promise { const resolvedManifestUrl = new URL(manifestUrl, location.href).href; const manifestText = await fetchTextNoStore(resolvedManifestUrl); const manifest = JSON.parse(manifestText) as SoftwareGalleryManifest; @@ -2089,15 +2364,19 @@ async function loadSoftwareGalleryItemsFromManifest(manifestUrl: string): Promis function softwareManifestUrls(): string[] { const params = new URLSearchParams(location.search); - const queryUrls = params.getAll("softwareManifest").flatMap(splitManifestUrls); + const queryUrls = params + .getAll("softwareManifest") + .flatMap(splitManifestUrls); const envUrls = splitManifestUrls( - (import.meta.env.VITE_KANDELO_SOFTWARE_MANIFEST_URLS as string | undefined) ?? "", + (import.meta.env.VITE_KANDELO_SOFTWARE_MANIFEST_URLS as + string | undefined) ?? "", ); - const urls = queryUrls.length > 0 - ? queryUrls - : envUrls.length > 0 - ? envUrls - : DEFAULT_SOFTWARE_MANIFEST_URLS; + const urls = + queryUrls.length > 0 + ? queryUrls + : envUrls.length > 0 + ? envUrls + : DEFAULT_SOFTWARE_MANIFEST_URLS; return [...new Set(urls)]; } @@ -2108,12 +2387,21 @@ function splitManifestUrls(value: string): string[] { .filter(Boolean); } -function sourceIdForManifest(manifest: SoftwareGalleryManifest, manifestUrl: string): string { - const raw = manifest.source_id - ?? manifest.repository?.split("/").pop() - ?? new URL(manifestUrl, location.href).pathname.split("/").filter(Boolean)[0] - ?? "software"; - const normalized = raw.toLowerCase().replace(/[^a-z0-9._-]+/g, "-").replace(/^-+|-+$/g, ""); +function sourceIdForManifest( + manifest: SoftwareGalleryManifest, + manifestUrl: string, +): string { + const raw = + manifest.source_id ?? + manifest.repository?.split("/").pop() ?? + new URL(manifestUrl, location.href).pathname + .split("/") + .filter(Boolean)[0] ?? + "software"; + const normalized = raw + .toLowerCase() + .replace(/[^a-z0-9._-]+/g, "-") + .replace(/^-+|-+$/g, ""); return normalized || "software"; } @@ -2127,7 +2415,13 @@ function softwareEntryToGalleryItem( const archiveUrl = archiveUrlFor(index, indexUrl, primaryPackage); if (!primaryPackage || !archiveUrl) return null; const id = `${sourceId}-${entry.id}`; - const profile = softwareProfileForEntry(id, entry, index, indexUrl, archiveUrl); + const profile = softwareProfileForEntry( + id, + entry, + index, + indexUrl, + archiveUrl, + ); if (!profile) return null; SOFTWARE_PROFILES.set(id, profile); return { @@ -2169,19 +2463,26 @@ function softwareProfileForEntry( if (!runtimeArchiveUrl) return null; return { ...base, - binaries: [{ - archiveUrl: runtimeArchiveUrl, - artifactPath: "artifacts/python.wasm", - installPath: "/usr/bin/python", - symlinks: ["/usr/bin/python3", "/usr/local/bin/python", "/usr/local/bin/python3"], - }], + binaries: [ + { + archiveUrl: runtimeArchiveUrl, + artifactPath: "artifacts/python.wasm", + installPath: "/usr/bin/python", + symlinks: [ + "/usr/bin/python3", + "/usr/local/bin/python", + "/usr/local/bin/python3", + ], + }, + ], shellEnv: [ ...SHELL_ENV, "PYTHONHOME=/usr", "PYTHONDONTWRITEBYTECODE=1", "PYTHONNOUSERSITE=1", ], - autoCommand: "python3 -c \"import sys, json; print('Python', sys.version.split()[0]); print(json.dumps({'kandelo': 'software'}))\"", + autoCommand: + "python3 -c \"import sys, json; print('Python', sys.version.split()[0]); print(json.dumps({'kandelo': 'software'}))\"", }; } @@ -2191,12 +2492,14 @@ function softwareProfileForEntry( if (!runtimeArchiveUrl) return null; return { ...base, - binaries: [{ - archiveUrl: runtimeArchiveUrl, - artifactPath: "artifacts/perl.wasm", - installPath: "/usr/bin/perl", - symlinks: ["/usr/local/bin/perl"], - }], + binaries: [ + { + archiveUrl: runtimeArchiveUrl, + artifactPath: "artifacts/perl.wasm", + installPath: "/usr/bin/perl", + symlinks: ["/usr/local/bin/perl"], + }, + ], shellEnv: [...SHELL_ENV, "PERL5LIB=/usr/lib/perl5"], autoCommand: "perl -e 'print \"Perl $^V from kandelo-software\\n\"'", }; @@ -2208,12 +2511,14 @@ function softwareProfileForEntry( if (!runtimeArchiveUrl) return null; return { ...base, - binaries: [{ - archiveUrl: runtimeArchiveUrl, - artifactPath: "artifacts/erlang.wasm", - installPath: "/usr/bin/erlang", - symlinks: ["/usr/bin/erl", "/usr/local/bin/erl"], - }], + binaries: [ + { + archiveUrl: runtimeArchiveUrl, + artifactPath: "artifacts/erlang.wasm", + installPath: "/usr/bin/erlang", + symlinks: ["/usr/bin/erl", "/usr/local/bin/erl"], + }, + ], shellEnv: [ ...SHELL_ENV, "ROOTDIR=/usr/local/lib/erlang", @@ -2248,7 +2553,8 @@ function softwareProfileForEntry( terminalAccess: "primary", internalsAccess: "drawer", }, - autoCommand: "echo 'Redis VFS from kandelo-software'; ls -l /usr/local/bin/redis-server /etc/dinit.d/redis", + autoCommand: + "echo 'Redis VFS from kandelo-software'; ls -l /usr/local/bin/redis-server /etc/dinit.d/redis", }; } @@ -2272,9 +2578,11 @@ function packageAvailable( requirement: GalleryPackageRequirement, ): boolean { const wasm32 = index.packages.get(packageKey(requirement))?.binary.wasm32; - return stringTomlValue(wasm32?.status) === "success" && + return ( + stringTomlValue(wasm32?.status) === "success" && Boolean(stringTomlValue(wasm32?.archive_url)) && - booleanTomlValue(wasm32?.browser_compatible) === true; + booleanTomlValue(wasm32?.browser_compatible) === true + ); } function archiveUrlFor( @@ -2356,7 +2664,10 @@ function parseIndexToml(text: string): SoftwareIndex { const value = parseTomlValue(rawValue); if (!currentPackage) { if (key === "abi_version") { - const parsed = typeof value === "number" ? value : Number.parseInt(String(value), 10); + const parsed = + typeof value === "number" + ? value + : Number.parseInt(String(value), 10); if (Number.isFinite(parsed)) abiVersion = parsed; } continue; @@ -2368,7 +2679,10 @@ function parseIndexToml(text: string): SoftwareIndex { if (!stringValue) continue; currentPackage[key] = stringValue; if (currentPackage.name && currentPackage.version) { - packages.set(`${currentPackage.name}@${currentPackage.version}`, currentPackage); + packages.set( + `${currentPackage.name}@${currentPackage.version}`, + currentPackage, + ); } } } @@ -2378,13 +2692,15 @@ function parseIndexToml(text: string): SoftwareIndex { async function fetchTextNoStore(url: string): Promise { const response = await fetch(url, { cache: "no-store" }); - if (!response.ok) throw new Error(`${response.status} ${response.statusText}`); + if (!response.ok) + throw new Error(`${response.status} ${response.statusText}`); return await response.text(); } async function fetchBytesNoStore(url: string): Promise { const response = await fetch(url, { cache: "no-store" }); - if (!response.ok) throw new Error(`${response.status} ${response.statusText}`); + if (!response.ok) + throw new Error(`${response.status} ${response.statusText}`); return new Uint8Array(await response.arrayBuffer()); } @@ -2397,7 +2713,8 @@ function accentForSoftwareEntry(id: string): string { } function glyphForSoftwareEntry(entry: SoftwareGalleryEntry): string { - const packageName = entry.packages[entry.packages.length - 1]?.name ?? entry.id; + const packageName = + entry.packages[entry.packages.length - 1]?.name ?? entry.id; const parts = packageName.split(/[-_]/).filter(Boolean); if (parts.length >= 2) return `${parts[0][0]}${parts[1][0]}`.toLowerCase(); return packageName.slice(0, 3).toLowerCase(); @@ -2434,7 +2751,9 @@ function readImageShellConfig(fs: MemoryFileSystem): KandeloShellConfig | null { ); const config = parseKandeloShellConfig(json); if (!config) { - throw new Error(`VFS image has unsupported ${KANDELO_SHELL_CONFIG_PATH} version`); + throw new Error( + `VFS image has unsupported ${KANDELO_SHELL_CONFIG_PATH} version`, + ); } return config; } @@ -2445,7 +2764,8 @@ async function loadProfileClosedLazyAssets( tick: (message: string) => void, ) { const bundleRoot = ( - import.meta.env.VITE_KANDELO_HOMEBREW_CLOSED_ACCEPTANCE_ROOT as string | undefined + import.meta.env.VITE_KANDELO_HOMEBREW_CLOSED_ACCEPTANCE_ROOT as + string | undefined )?.trim(); if (!bundleRoot || profile.image !== "shell") return undefined; if (import.meta.env.PROD) { @@ -2461,8 +2781,95 @@ async function loadProfileClosedLazyAssets( embeddedPlanBytes, bundleRoot, }); - tick(`verified ${bundle.assets.length} exact deferred bottle payloads`); - return bundle.assets; + const packageAssets = await loadHomebrewBootstrapClosedAssets(fs); + tick( + `verified ${bundle.assets.length} exact deferred bottle payloads and ` + + `${packageAssets.length} package source tree`, + ); + return [...bundle.assets, ...packageAssets]; +} + +async function loadHomebrewBootstrapClosedAssets( + fs: MemoryFileSystem, +): Promise { + const metadata = fs.getImageMetadata(); + if (metadata === null || !Array.isArray(metadata.packageDeferredTrees)) { + throw new Error("closed shell image omits packageDeferredTrees metadata"); + } + const matches = metadata.packageDeferredTrees.filter( + ( + value, + ): value is { + package: { name: string; output: string }; + archive: { output: string; url: string; sha256: string; bytes: number }; + state: string; + } => { + if ( + !isPlainRecord(value) || + !isPlainRecord(value.package) || + !isPlainRecord(value.archive) + ) + return false; + return value.package.name === "homebrew-bootstrap"; + }, + ); + if (matches.length !== 1) { + throw new Error( + `closed shell image has ${matches.length} Homebrew bootstrap bindings`, + ); + } + const binding = matches[0]!; + if ( + binding.state !== "deferred" || + binding.package.output !== "homebrew-bootstrap.zip" || + binding.archive.output !== binding.package.output || + binding.archive.url !== binding.package.output || + !/^[0-9a-f]{64}$/.test(binding.archive.sha256) || + !Number.isSafeInteger(binding.archive.bytes) || + binding.archive.bytes <= 0 + ) { + throw new Error( + "closed shell image has an invalid Homebrew bootstrap binding", + ); + } + const sourceUrl = resolveShellLazyArchiveUrl(binding.archive.url); + const closedUrl = + `https://closed-lazy.kandelo.invalid/homebrew-bootstrap/` + + `${binding.archive.sha256}/${binding.package.output}`; + // WHY: rewriteLazyArchiveUrls is filesystem-wide. Refuse an ambiguous source + // URL so binding this one verified package cannot retarget another lazy tree. + const pendingForSource = fs + .exportLazyArchiveEntries() + .filter((tree) => + tree.content?.transports.some((transport) => transport === sourceUrl), + ); + if ( + pendingForSource.length !== 1 || + pendingForSource[0]!.content?.sha256 !== binding.archive.sha256 || + pendingForSource[0]!.content?.bytes !== binding.archive.bytes || + pendingForSource[0]!.content?.transports.length !== 1 + ) { + throw new Error( + "closed shell image does not bind the Homebrew bootstrap source " + + "to exactly one matching pending tree", + ); + } + // WHY: the worker's closed fetcher intentionally rejects every unbound URL. + // Keep the Vite file as an acceptance-only transport source, then bind its + // verified bytes to one canonical HTTPS identity before worker ownership. + fs.rewriteLazyArchiveUrls((url) => (url === sourceUrl ? closedUrl : url)); + return loadClosedLazyAssetSources([ + { + url: closedUrl, + sourceUrl, + sha256: binding.archive.sha256, + size: binding.archive.bytes, + }, + ]); +} + +function isPlainRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); } function assertImageShellExecutable(fs: MemoryFileSystem, path: string): void { @@ -2489,7 +2896,10 @@ function readImageConfig(fs: MemoryFileSystem): KandeloDemoConfig | null { return readKandeloDemoConfigFromVfs(fs); } -function readOptionalVfsText(fs: MemoryFileSystem, path: string): string | null { +function readOptionalVfsText( + fs: MemoryFileSystem, + path: string, +): string | null { try { return new TextDecoder().decode(new Uint8Array(readVfsFile(fs, path))); } catch (err) { @@ -2527,7 +2937,8 @@ function readVfsFile(fs: MemoryFileSystem, path: string): ArrayBuffer { function failOn(label: string): (r: Response) => Response { return (r) => { - if (!r.ok) throw new Error(`fetch failed for ${label}: ${r.status} ${r.statusText}`); + if (!r.ok) + throw new Error(`fetch failed for ${label}: ${r.status} ${r.statusText}`); return r; }; } diff --git a/apps/browser-demos/playwright-server-policy.test.ts b/apps/browser-demos/playwright-server-policy.test.ts new file mode 100644 index 0000000000..48fd810fd7 --- /dev/null +++ b/apps/browser-demos/playwright-server-policy.test.ts @@ -0,0 +1,30 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + shouldReuseExistingPlaywrightServer, +} from "./playwright-server-policy"; + +test("exact Homebrew browser proofs never reuse another worktree's server", () => { + assert.equal(shouldReuseExistingPlaywrightServer({}), true); + assert.equal(shouldReuseExistingPlaywrightServer({ CI: "1" }), false); + assert.equal( + shouldReuseExistingPlaywrightServer({ + KANDELO_HOMEBREW_MAIN_SHELL_STRICT: "1", + }), + false, + ); + assert.equal( + shouldReuseExistingPlaywrightServer({ + KANDELO_HOMEBREW_GUEST_BROWSER_LIFECYCLE_LIVE: "1", + }), + false, + ); + assert.equal( + shouldReuseExistingPlaywrightServer({ + KANDELO_HOMEBREW_GUEST_BROWSER_LIFECYCLE_LIVE: "0", + KANDELO_HOMEBREW_MAIN_SHELL_STRICT: "0", + }), + true, + ); +}); diff --git a/apps/browser-demos/playwright-server-policy.ts b/apps/browser-demos/playwright-server-policy.ts index bb19dce9ec..c600908c16 100644 --- a/apps/browser-demos/playwright-server-policy.ts +++ b/apps/browser-demos/playwright-server-policy.ts @@ -1,5 +1,6 @@ export interface PlaywrightServerEnvironment { CI?: string; + KANDELO_HOMEBREW_GUEST_BROWSER_LIFECYCLE_LIVE?: string; KANDELO_HOMEBREW_MAIN_SHELL_STRICT?: string; } @@ -12,5 +13,7 @@ export interface PlaywrightServerEnvironment { export function shouldReuseExistingPlaywrightServer( env: PlaywrightServerEnvironment, ): boolean { - return !env.CI && env.KANDELO_HOMEBREW_MAIN_SHELL_STRICT !== "1"; + return !env.CI && + env.KANDELO_HOMEBREW_MAIN_SHELL_STRICT !== "1" && + env.KANDELO_HOMEBREW_GUEST_BROWSER_LIFECYCLE_LIVE !== "1"; } diff --git a/apps/browser-demos/test/homebrew-guest-lifecycle.spec.ts b/apps/browser-demos/test/homebrew-guest-lifecycle.spec.ts new file mode 100644 index 0000000000..3583075713 --- /dev/null +++ b/apps/browser-demos/test/homebrew-guest-lifecycle.spec.ts @@ -0,0 +1,165 @@ +import { expect, test } from "@playwright/test"; +import { lstatSync, readFileSync } from "node:fs"; +import { resolve } from "node:path"; + +import { + projectHomebrewGuestLifecycleBrowserFixture, + type HomebrewGuestLifecycleBrowserFixture, +} from "../../../homebrew/test/homebrew_guest_lifecycle_browser_fixture"; +import type { + HomebrewGuestLifecycleBrowserResult, +} from "../../../homebrew/test/homebrew_guest_lifecycle_browser"; + +declare global { + interface Window { + __homebrewVfsTestReady: boolean; + __runHomebrewGuestLifecycleAcceptance: ( + fixture: HomebrewGuestLifecycleBrowserFixture, + ) => Promise; + } +} + +const LIVE_ENV = "KANDELO_HOMEBREW_GUEST_BROWSER_LIFECYCLE_LIVE"; +const FIXTURE_ENV = + "KANDELO_HOMEBREW_GUEST_BROWSER_LIFECYCLE_FIXTURE_PATH"; + +test( + "Chromium rejects a guest lifecycle fixture without live-network opt-in", + async ({ page, baseURL, browserName }) => { + test.skip( + browserName !== "chromium", + "the stock Homebrew lifecycle initially targets Chromium", + ); + if (!baseURL) throw new Error("Playwright baseURL is required"); + await page.goto(new URL("/pages/homebrew-vfs-test/", baseURL).href); + await expect.poll( + () => page.evaluate(() => window.__homebrewVfsTestReady), + { timeout: 120_000 }, + ).toBe(true); + + const externalRequests: string[] = []; + page.on("request", (request) => { + const url = new URL(request.url()); + if (url.origin !== new URL(baseURL).origin) { + externalRequests.push(url.href); + } + }); + const message = await page.evaluate(async () => { + const fixture = { + schema: 1, + allowLiveNetwork: false, + transportMode: "public", + image: { + url: "https://example.test/main-shell.vfs.zst", + sha256: "1".repeat(64), + bytes: 1, + }, + bootstrap: { + spec: { + url: "https://example.test/main-shell-brew-package-tree.json", + sha256: "2".repeat(64), + bytes: 1, + }, + archive: { + url: "https://example.test/homebrew-bootstrap.zip", + sha256: "3".repeat(64), + bytes: 1, + }, + environment: { + url: "https://example.test/homebrew-brew.env", + sha256: "4".repeat(64), + bytes: 1, + }, + }, + bottleMirror: { + plan: { + url: + "https://example.test/kandelo-homebrew-bottle-mirror-plan.json", + sha256: "7".repeat(64), + bytes: 1, + }, + }, + revisions: { + coreRevision: "5".repeat(40), + canaryRevision: "6".repeat(40), + }, + timeoutMs: 1_000, + }; + try { + await window.__runHomebrewGuestLifecycleAcceptance( + fixture as unknown as HomebrewGuestLifecycleBrowserFixture, + ); + } catch (error) { + return error instanceof Error ? error.message : String(error); + } + throw new Error("fixture without live-network opt-in unexpectedly ran"); + }); + expect(message).toContain("explicit live-network opt-in"); + expect(externalRequests).toEqual([]); + }, +); + +test( + "the exact stock Homebrew lifecycle survives a Chromium rootfs reboot", + async ({ page, baseURL, browserName }) => { + test.skip( + browserName !== "chromium", + "the stock Homebrew lifecycle initially targets Chromium", + ); + const liveValue = process.env[LIVE_ENV]; + const fixturePath = process.env[FIXTURE_ENV]; + const partiallyConfigured = + liveValue !== undefined || fixturePath !== undefined; + if (liveValue !== "1" || fixturePath === undefined) { + if (partiallyConfigured) { + throw new Error( + `${LIVE_ENV}=1 and ${FIXTURE_ENV} are both required for the live proof`, + ); + } + test.skip( + true, + "exact published Homebrew lifecycle fixture is not configured", + ); + } + if (!baseURL) throw new Error("Playwright baseURL is required"); + + const absoluteFixturePath = resolve(fixturePath!); + const stat = lstatSync(absoluteFixturePath); + if (!stat.isFile() || stat.isSymbolicLink()) { + throw new Error( + `${FIXTURE_ENV} must name a regular non-symlink JSON file`, + ); + } + const fixture = projectHomebrewGuestLifecycleBrowserFixture( + JSON.parse(readFileSync(absoluteFixturePath, "utf8")), + ); + test.setTimeout(fixture.timeoutMs + 180_000); + + await page.goto(new URL("/pages/homebrew-vfs-test/", baseURL).href); + await expect.poll( + () => page.evaluate(() => window.__homebrewVfsTestReady), + { timeout: 120_000 }, + ).toBe(true); + const result = await page.evaluate( + (exactFixture) => + window.__runHomebrewGuestLifecycleAcceptance(exactFixture), + fixture, + ); + + expect(result.coreRevision).toBe(fixture.revisions.coreRevision); + expect(result.canaryRevision).toBe(fixture.revisions.canaryRevision); + expect(result.exportedImageSha256).toMatch(/^[0-9a-f]{64}$/); + expect(result.exportedImageBytes).toBeGreaterThan(0); + expect(result.phaseOneCompletedUrls.length).toBeGreaterThan(0); + expect( + result.phaseOneLazyDownloads.some( + (event) => event.status === "error", + ), + ).toBe(false); + expect( + result.phaseTwoLazyDownloads.some( + (event) => event.status === "error", + ), + ).toBe(false); + }, +); diff --git a/apps/browser-demos/test/kandelo-homebrew-main-shell.spec.ts b/apps/browser-demos/test/kandelo-homebrew-main-shell.spec.ts index 6fe79e4ff4..8d15cc77d2 100644 --- a/apps/browser-demos/test/kandelo-homebrew-main-shell.spec.ts +++ b/apps/browser-demos/test/kandelo-homebrew-main-shell.spec.ts @@ -1,8 +1,13 @@ +import { createHash } from "node:crypto"; import { expect, test, type Page } from "@playwright/test"; import { MAIN_SHELL_LANGUAGE_RUNTIME_INVOCATIONS } from "../../../scripts/homebrew-language-runtime-contract"; const strict = process.env.KANDELO_HOMEBREW_MAIN_SHELL_STRICT === "1"; const expectedImageSha256 = process.env.KANDELO_HOMEBREW_MAIN_SHELL_SHA256; +const expectedBootstrapSha256 = + process.env.KANDELO_HOMEBREW_MAIN_SHELL_BOOTSTRAP_SHA256; +const expectedBootstrapBytes = + process.env.KANDELO_HOMEBREW_MAIN_SHELL_BOOTSTRAP_BYTES; const closedMirrorRoot = process.env.VITE_KANDELO_HOMEBREW_CLOSED_ACCEPTANCE_ROOT; const transportMode = process.env.KANDELO_HOMEBREW_MAIN_SHELL_TRANSPORT_MODE; @@ -26,8 +31,61 @@ interface LazyDownloadRow { eventCount: string | null; } +interface ExactAcceptanceConfig { + imageSha256: string; + bootstrapSha256: string; + bootstrapBytes: string; + transportMode: "closed" | "public"; + mirrorPlanUrl: string; + closedMirrorRoot: string | undefined; +} + +interface BootstrapPayloadResponse { + url: string; + status: number; + sha256: string; + bytes: number; +} + +interface ExactShellPage { + config: ExactAcceptanceConfig; + mirrorPlan: { assets: MirrorAsset[] }; + legacyArtifactDownloads: string[]; + bootstrapPayloadRequests: string[]; + bootstrapPayloadResponses: Array>; +} + +const BASE_EXPECTED_PACKAGES = [ + "kandelo-dev/tap-core/dash", + "kandelo-dev/tap-core/git", + "kandelo-dev/tap-core/nethack", +] as const; + +const BREW_EXPECTED_PACKAGES = [ + "kandelo-dev/tap-core/coreutils", + "kandelo-dev/tap-core/posix-utils-lite", + "kandelo-dev/tap-core/ruby", + "kandelo-dev/tap-core/zlib", +] as const; + +function isHomebrewBootstrapUrl(url: string): boolean { + const path = url.split(/[?#]/, 1)[0] ?? url; + return /(?:^|\/)homebrew-bootstrap(?:-[A-Za-z0-9_-]+)?\.zip$/.test(path); +} + +function isHomebrewBootstrapRow(row: LazyDownloadRow): boolean { + return row.source !== null && isHomebrewBootstrapUrl(row.source); +} + +function bottleRows(rows: readonly LazyDownloadRow[]): LazyDownloadRow[] { + return rows.filter((row) => !isHomebrewBootstrapRow(row)); +} + async function terminalText(page: Page): Promise { - return page.locator(".xterm-rows").first().evaluate((node) => node.textContent ?? ""); + return page + .locator(".xterm-rows") + .first() + .evaluate((node) => node.textContent ?? ""); } async function waitForTerminalContent( @@ -39,9 +97,10 @@ async function waitForTerminalContent( let text = ""; while (Date.now() < deadline) { text = await terminalText(page); - const matched = typeof expected === "string" - ? text.includes(expected) - : expected.test(text); + const matched = + typeof expected === "string" + ? text.includes(expected) + : expected.test(text); if (matched) return; if (/bash: \/bin\/sh: I\/O error/.test(text)) { throw new Error( @@ -58,13 +117,15 @@ async function waitForTerminalContent( async function lazyDownloadDiagnostics(page: Page): Promise { await page.getByRole("button", { name: "Internals" }).click(); await page.getByRole("tab", { name: "Lazy Load" }).click(); - const rows = await page.locator(".kdownload-table tbody tr").evaluateAll((elements) => - elements.map((element) => ({ - status: element.getAttribute("data-download-status"), - source: element.getAttribute("data-source"), - text: element.textContent?.replace(/\s+/g, " ").trim(), - })) - ); + const rows = await page + .locator(".kdownload-table tbody tr") + .evaluateAll((elements) => + elements.map((element) => ({ + status: element.getAttribute("data-download-status"), + source: element.getAttribute("data-source"), + text: element.textContent?.replace(/\s+/g, " ").trim(), + })), + ); return JSON.stringify(rows); } @@ -86,23 +147,31 @@ async function runTerminalCommand( await waitForTerminalContent(page, expected, timeout); } +function bashCommand(script: string): string { + return `/bin/bash -c '${script.replaceAll("'", `'"'"'`)}'`; +} + async function readLazyDownloadRows(page: Page): Promise { const internals = page.getByRole("button", { name: "Internals" }); - if (await internals.getAttribute("aria-pressed") !== "true") { + if ((await internals.getAttribute("aria-pressed")) !== "true") { await internals.click(); } await page.getByRole("tab", { name: "Lazy Load" }).click(); - const rows = await page.locator(".kdownload-table tbody tr").evaluateAll((elements) => - elements.map((element) => ({ - asset: element.querySelector(".kdownload-asset-name")?.textContent?.trim() ?? "", - status: element.getAttribute("data-download-status"), - kind: element.getAttribute("data-download-kind"), - source: element.getAttribute("data-source"), - loadedBytes: element.getAttribute("data-loaded-bytes"), - totalBytes: element.getAttribute("data-total-bytes"), - eventCount: element.getAttribute("data-download-events"), - })) - ); + const rows = await page + .locator(".kdownload-table tbody tr") + .evaluateAll((elements) => + elements.map((element) => ({ + asset: + element.querySelector(".kdownload-asset-name")?.textContent?.trim() ?? + "", + status: element.getAttribute("data-download-status"), + kind: element.getAttribute("data-download-kind"), + source: element.getAttribute("data-source"), + loadedBytes: element.getAttribute("data-loaded-bytes"), + totalBytes: element.getAttribute("data-total-bytes"), + eventCount: element.getAttribute("data-download-events"), + })), + ); await internals.click(); return rows; } @@ -111,14 +180,21 @@ function packageNamesForRows( rows: readonly LazyDownloadRow[], mirrorPlan: { assets: MirrorAsset[] }, ): string[] { - const packageByUrl = new Map(mirrorPlan.assets.map((asset) => [asset.url, asset.package])); - return rows.map((row) => { - const packageName = row.source === null ? undefined : packageByUrl.get(row.source); - if (packageName === undefined) { - throw new Error(`lazy row source is absent from the mirror plan: ${String(row.source)}`); - } - return packageName; - }).sort(); + const packageByUrl = new Map( + mirrorPlan.assets.map((asset) => [asset.url, asset.package]), + ); + return bottleRows(rows) + .map((row) => { + const packageName = + row.source === null ? undefined : packageByUrl.get(row.source); + if (packageName === undefined) { + throw new Error( + `lazy row source is absent from the mirror plan: ${String(row.source)}`, + ); + } + return packageName; + }) + .sort(); } async function waitForLazyPackageRows( @@ -137,9 +213,11 @@ async function waitForLazyPackageRows( const added = rows.filter(({ source }) => !priorSources.has(source)); const addedPackages = new Set(packageNamesForRows(added, mirrorPlan)); const expectedPackagesPresent = expectedPackages.every((packageName) => - addedPackages.has(packageName) + addedPackages.has(packageName), + ); + const addedRowsComplete = added.every( + ({ status }) => status === "complete", ); - const addedRowsComplete = added.every(({ status }) => status === "complete"); if (expectedPackagesPresent && addedRowsComplete) { // Guest completion can precede the last React ledger update. Require a // quiet completed window so a delayed row cannot escape this phase. @@ -166,13 +244,40 @@ async function waitForLazyPackageRows( ); } -test("the exact public-bottle shell preserves shell, language, and NetHack behavior", async ({ page }) => { - test.skip(!strict, "exact Homebrew main-shell CI configures this acceptance test"); +async function waitForHomebrewBootstrapRow( + page: Page, +): Promise<{ rows: LazyDownloadRow[]; bootstrap: LazyDownloadRow }> { + const deadline = Date.now() + 30_000; + let rows: LazyDownloadRow[] = []; + while (Date.now() < deadline) { + rows = await readLazyDownloadRows(page); + const matches = rows.filter(isHomebrewBootstrapRow); + if (matches.length === 1 && matches[0]?.status === "complete") { + return { rows, bootstrap: matches[0] }; + } + await page.waitForTimeout(100); + } + throw new Error( + `timed out waiting for one completed Homebrew source tree: ${JSON.stringify(rows)}`, + ); +} + +function exactAcceptanceConfig(): ExactAcceptanceConfig { if (!expectedImageSha256 || !/^[0-9a-f]{64}$/.test(expectedImageSha256)) { throw new Error( "KANDELO_HOMEBREW_MAIN_SHELL_SHA256 must be the exact lowercase image digest", ); } + if ( + !expectedBootstrapSha256 || + !/^[0-9a-f]{64}$/.test(expectedBootstrapSha256) || + !expectedBootstrapBytes || + !/^[1-9][0-9]*$/.test(expectedBootstrapBytes) + ) { + throw new Error( + "the exact Homebrew bootstrap SHA-256 and byte count must be configured", + ); + } if (transportMode !== "closed" && transportMode !== "public") { throw new Error( "KANDELO_HOMEBREW_MAIN_SHELL_TRANSPORT_MODE must be closed or public", @@ -184,11 +289,26 @@ test("the exact public-bottle shell preserves shell, language, and NetHack behav (!closedMirrorRoot || !closedMirrorRoot.startsWith("/"))) || (transportMode === "public" && closedMirrorRoot !== undefined) ) { - throw new Error("main-shell transport mode has inconsistent mirror configuration"); + throw new Error( + "main-shell transport mode has inconsistent mirror configuration", + ); } - test.setTimeout(420_000); + return { + imageSha256: expectedImageSha256, + bootstrapSha256: expectedBootstrapSha256, + bootstrapBytes: expectedBootstrapBytes, + transportMode, + mirrorPlanUrl, + closedMirrorRoot, + }; +} +async function bootExactShellPage(page: Page): Promise { + const config = exactAcceptanceConfig(); const legacyArtifactDownloads: string[] = []; + const bootstrapPayloadRequests: string[] = []; + const bootstrapPayloadResponses: Array> = + []; const closedPayloadResponses: Array<{ url: string; status: number }> = []; await page.addInitScript(() => { const evidence = { @@ -206,16 +326,21 @@ test("the exact public-bottle shell preserves shell, language, and NetHack behav window.fetch = async (...args) => { const response = await nativeFetch(...args); if (/shell[^/]*\.vfs\.zst(?:\?|$)/.test(response.url)) { - void response.clone().arrayBuffer() + void response + .clone() + .arrayBuffer() .then((bytes) => crypto.subtle.digest("SHA-256", bytes)) .then((digest) => { evidence.digests.push( - Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")) - .join(""), + Array.from(new Uint8Array(digest), (byte) => + byte.toString(16).padStart(2, "0"), + ).join(""), ); }) .catch((error: unknown) => { - evidence.errors.push(error instanceof Error ? error.message : String(error)); + evidence.errors.push( + error instanceof Error ? error.message : String(error), + ); }); } return response; @@ -223,26 +348,44 @@ test("the exact public-bottle shell preserves shell, language, and NetHack behav }); page.on("request", (request) => { const url = request.url(); + if (request.resourceType() === "fetch" && isHomebrewBootstrapUrl(url)) { + bootstrapPayloadRequests.push(url); + return; + } if ( request.resourceType() === "fetch" && - ( - (/\.(?:wasm|zip)(?:\?|$)/.test(url) && - !/kernel[^/]*\.wasm(?:\?|$)/.test(url)) || + ((/\.(?:wasm|zip)(?:\?|$)/.test(url) && + !/kernel[^/]*\.wasm(?:\?|$)/.test(url)) || (/\.vfs(?:\.zst)?(?:\?|$)/.test(url) && - !/shell[^/]*\.vfs\.zst(?:\?|$)/.test(url)) - ) + !/shell[^/]*\.vfs\.zst(?:\?|$)/.test(url))) ) { legacyArtifactDownloads.push(url); } }); page.on("response", (response) => { - if (!closedMirrorRoot) return; + if ( + response.request().resourceType() === "fetch" && + isHomebrewBootstrapUrl(response.url()) + ) { + bootstrapPayloadResponses.push( + response.body().then((bytes) => ({ + url: response.url(), + status: response.status(), + sha256: createHash("sha256").update(bytes).digest("hex"), + bytes: bytes.byteLength, + })), + ); + } + if (!config.closedMirrorRoot) return; const url = new URL(response.url()); if ( - url.pathname.startsWith(`${closedMirrorRoot}/`) && + url.pathname.startsWith(`${config.closedMirrorRoot}/`) && url.pathname.endsWith("-layer.bin") ) { - closedPayloadResponses.push({ url: response.url(), status: response.status() }); + closedPayloadResponses.push({ + url: response.url(), + status: response.status(), + }); } }); @@ -250,58 +393,240 @@ test("the exact public-bottle shell preserves shell, language, and NetHack behav await page.waitForTimeout(2_000); const overlay = page.locator("vite-error-overlay"); if (await overlay.count()) { - const detail = await overlay.evaluate((element) => - element.shadowRoot?.querySelector(".message-body")?.textContent?.trim() - || element.shadowRoot?.textContent?.trim() - || element.textContent?.trim() - || "unknown Vite import error" + const detail = await overlay.evaluate( + (element) => + element.shadowRoot + ?.querySelector(".message-body") + ?.textContent?.trim() || + element.shadowRoot?.textContent?.trim() || + element.textContent?.trim() || + "unknown Vite import error", + ); + throw new Error( + `Homebrew main-shell smoke hit a Vite error overlay: ${detail}`, ); - throw new Error(`Homebrew main-shell smoke hit a Vite error overlay: ${detail}`); } - await page.waitForFunction(() => { - const evidence = (window as typeof window & { - __kandeloHomebrewMainShellImageEvidence?: { - digests: string[]; - errors: string[]; - }; - }).__kandeloHomebrewMainShellImageEvidence; - return Boolean(evidence && (evidence.digests.length > 0 || evidence.errors.length > 0)); - }, undefined, { timeout: 180_000 }); - const imageEvidence = await page.evaluate(() => - (window as typeof window & { - __kandeloHomebrewMainShellImageEvidence: { - digests: string[]; - errors: string[]; - }; - }).__kandeloHomebrewMainShellImageEvidence + await page.waitForFunction( + () => { + const evidence = ( + window as typeof window & { + __kandeloHomebrewMainShellImageEvidence?: { + digests: string[]; + errors: string[]; + }; + } + ).__kandeloHomebrewMainShellImageEvidence; + return Boolean( + evidence && (evidence.digests.length > 0 || evidence.errors.length > 0), + ); + }, + undefined, + { timeout: 180_000 }, + ); + const imageEvidence = await page.evaluate( + () => + ( + window as typeof window & { + __kandeloHomebrewMainShellImageEvidence: { + digests: string[]; + errors: string[]; + }; + } + ).__kandeloHomebrewMainShellImageEvidence, ); expect(imageEvidence.errors).toEqual([]); - expect(new Set(imageEvidence.digests)).toEqual(new Set([expectedImageSha256])); + expect(new Set(imageEvidence.digests)).toEqual(new Set([config.imageSha256])); - await expect(page.locator(".xterm-rows").first()).toBeVisible({ timeout: 180_000 }); + await expect(page.locator(".xterm-rows").first()).toBeVisible({ + timeout: 180_000, + }); await waitForTerminalContent(page, /kandelo\$\s*$/, 240_000); const mirrorPlan = await page.evaluate(async (url) => { - const response = await fetch( - url, - { cache: "no-store", credentials: "omit", redirect: "error" }, - ); - if (!response.ok) throw new Error(`mirror plan fetch failed: HTTP ${response.status}`); + const response = await fetch(url, { + cache: "no-store", + credentials: "omit", + redirect: "error", + }); + if (!response.ok) + throw new Error(`mirror plan fetch failed: HTTP ${response.status}`); return response.json() as Promise<{ assets: MirrorAsset[] }>; - }, mirrorPlanUrl); + }, config.mirrorPlanUrl); expect(mirrorPlan.assets.length).toBeGreaterThan(0); - if (transportMode === "closed") { + expect(mirrorPlan.assets).toHaveLength(39); + if (config.transportMode === "closed") { expect(closedPayloadResponses).toHaveLength(mirrorPlan.assets.length); - expect(closedPayloadResponses.every(({ status }) => status === 200)).toBe(true); + expect(closedPayloadResponses.every(({ status }) => status === 200)).toBe( + true, + ); } await expect(page.getByRole("heading", { name: "Shell demo" })).toBeVisible({ timeout: 60_000, }); + + return { + config, + mirrorPlan, + legacyArtifactDownloads, + bootstrapPayloadRequests, + bootstrapPayloadResponses, + }; +} + +async function assertBootstrapStillDeferred( + shell: ExactShellPage, + rows: readonly LazyDownloadRow[], +): Promise { + expect(rows.filter(isHomebrewBootstrapRow)).toEqual([]); + const responses = await Promise.all(shell.bootstrapPayloadResponses); + if (shell.config.transportMode === "closed") { + // WHY: closed acceptance verifies and snapshots the Vite-owned package + // bytes during setup, but that preload must not materialize the guest tree. + expect(shell.bootstrapPayloadRequests).toHaveLength(1); + expect(responses).toEqual([ + expect.objectContaining({ + status: 200, + sha256: shell.config.bootstrapSha256, + bytes: Number(shell.config.bootstrapBytes), + }), + ]); + } else { + expect(shell.bootstrapPayloadRequests).toEqual([]); + expect(responses).toEqual([]); + } +} + +async function assertBootstrapMaterialized( + shell: ExactShellPage, + row: LazyDownloadRow, +): Promise { + await expect + .poll(() => shell.bootstrapPayloadRequests.length, { + timeout: 30_000, + }) + .toBe(1); + expect(shell.bootstrapPayloadResponses).toHaveLength(1); + expect(await Promise.all(shell.bootstrapPayloadResponses)).toEqual([ + expect.objectContaining({ + status: 200, + sha256: shell.config.bootstrapSha256, + bytes: Number(shell.config.bootstrapBytes), + }), + ]); + expect(row).toEqual( + expect.objectContaining({ + kind: "tree", + status: "complete", + loadedBytes: shell.config.bootstrapBytes, + totalBytes: shell.config.bootstrapBytes, + }), + ); + expect(Number(row.eventCount)).toBeGreaterThanOrEqual(3); +} + +function assertBottleLedger( + rows: readonly LazyDownloadRow[], + mirrorPlan: { assets: MirrorAsset[] }, +): void { + // The kernel worker's lazy-download ledger is the transport authority. Raw + // browser request events are only diagnostic: service-worker delivery can + // notify Playwright after the guest has consumed the verified response. + const assetByUrl = new Map( + mirrorPlan.assets.map((asset) => [asset.url, asset]), + ); + for (const row of bottleRows(rows)) { + const asset = row.source === null ? undefined : assetByUrl.get(row.source); + expect(asset, `unplanned lazy row ${row.asset}`).toBeDefined(); + expect(row.asset).toBe(asset!.asset); + expect(row.kind).toBe("tree"); + expect(row.status).toBe("complete"); + expect(row.loadedBytes).toBe(String(asset!.bytes)); + expect(row.totalBytes).toBe(String(asset!.bytes)); + expect(Number(row.eventCount)).toBeGreaterThanOrEqual(3); + } +} + +test("a fresh exact shell materializes brew and its runtime only on first use", async ({ + page, +}) => { + test.skip( + !strict, + "exact Homebrew main-shell CI configures this acceptance test", + ); + test.setTimeout(420_000); + + const shell = await bootExactShellPage(page); await runTerminalCommand( page, - "printf 'HOMEBREW_MAIN_SHELL_PATH:%s:%s\\n' \"$0\" \"${PATH%%:*}\"", + 'printf \'HOMEBREW_MAIN_SHELL_PATH:%s:%s\\n\' "$0" "${PATH%%:*}"', "HOMEBREW_MAIN_SHELL_PATH:bash:/home/linuxbrew/.linuxbrew/bin", ); + let lazyRows = await readLazyDownloadRows(page); + expect(lazyRows).toEqual([]); + await assertBootstrapStillDeferred(shell, lazyRows); + + // WHY: stock brew itself starts bottled Ruby. Keep this as the first runtime + // command on a pristine machine so Ruby, zlib, and brew's utility bottles + // are proven to be consequences of ordinary /usr/bin/brew first use. + const brewPriorSources = new Set(lazyRows.map(({ source }) => source)); + const brewScript = [ + "set -eu", + 'test "$(command -v brew)" = /home/linuxbrew/.linuxbrew/bin/brew', + "test -x /usr/bin/brew", + 'brew_version="$(/usr/bin/brew --version 2>&1)"', + 'case "$brew_version" in "Homebrew "*) ;; *) ' + + 'printf "unexpected brew version: %s\\n" "$brew_version" >&2; exit 1;; esac', + 'test "$(/usr/bin/brew --prefix 2>&1)" = /home/linuxbrew/.linuxbrew', + 'test "$(/usr/bin/brew --repository 2>&1)" = /home/linuxbrew/.linuxbrew', + 'test "$(/usr/bin/brew --cellar 2>&1)" = /home/linuxbrew/.linuxbrew/Cellar', + 'test "$(/usr/bin/brew --cache 2>&1)" = /home/user/.cache/Homebrew', + "mkdir -p /home/linuxbrew/.linuxbrew/etc/homebrew /home/user/.homebrew", + "printf 'HOMEBREW_KANDELO_BOTTLE_TAG=wasm64_kandelo\\n' " + + "> /home/linuxbrew/.linuxbrew/etc/homebrew/brew.env", + "printf 'HOMEBREW_KANDELO_BOTTLE_TAG=wasm64_kandelo\\n' " + + "> /home/user/.homebrew/brew.env", + `test "$(/usr/bin/brew ruby -e 'print ENV.fetch("HOMEBREW_KANDELO_BOTTLE_TAG")' 2>&1)" = wasm32_kandelo`, + 'printf "HOMEBREW_BREW_COMMAND_OK\\n"', + ].join("; "); + await runTerminalCommand( + page, + bashCommand(brewScript), + "HOMEBREW_BREW_COMMAND_OK", + 240_000, + ); + lazyRows = await waitForLazyPackageRows( + page, + brewPriorSources, + BREW_EXPECTED_PACKAGES, + shell.mirrorPlan, + ); + const brewResult = await waitForHomebrewBootstrapRow(page); + lazyRows = brewResult.rows; + expect( + packageNamesForRows( + lazyRows.filter(({ source }) => !brewPriorSources.has(source)), + shell.mirrorPlan, + ), + ).toEqual([...BREW_EXPECTED_PACKAGES]); + await assertBootstrapMaterialized(shell, brewResult.bootstrap); + + const repeatBrewPriorSources = new Set(lazyRows.map(({ source }) => source)); + await runTerminalCommand( + page, + 'test "$(/usr/bin/brew --prefix)" = /home/linuxbrew/.linuxbrew && ' + + "printf 'HOMEBREW_BREW_REUSE_OK\\n'", + "HOMEBREW_BREW_REUSE_OK", + 240_000, + ); + lazyRows = await readLazyDownloadRows(page); + expect( + lazyRows.filter(({ source }) => !repeatBrewPriorSources.has(source)), + ).toEqual([]); + expect(lazyRows.filter(isHomebrewBootstrapRow)).toHaveLength(1); + expect(shell.bootstrapPayloadRequests).toHaveLength(1); + expect(shell.bootstrapPayloadResponses).toHaveLength(1); + + const basePriorSources = new Set(lazyRows.map(({ source }) => source)); await runTerminalCommand( page, "/bin/sh -c 'set -e; " + @@ -310,9 +635,9 @@ test("the exact public-bottle shell preserves shell, language, and NetHack behav "bunzip2 bzcat netcat git-remote-http git-remote-https git-remote-ftp " + "git-remote-ftps nano vim nethack fbdoom modeset " + "python python3 python3.13 perl erl ruby gem bundle bundler; " + - "do command -v \"$cmd\" >/dev/null; done; " + - "test \"$(git config --get user.name)\" = User; " + - "printf \"HOMEBREW_MAIN_SHELL_OK:%s\\n\" \"$(git --version)\"'", + 'do command -v "$cmd" >/dev/null; done; ' + + 'test "$(git config --get user.name)" = User; ' + + 'printf "HOMEBREW_MAIN_SHELL_OK:%s\\n" "$(git --version)"\'', "HOMEBREW_MAIN_SHELL_OK:git version 2.47.1", 240_000, ); @@ -321,26 +646,49 @@ test("the exact public-bottle shell preserves shell, language, and NetHack behav "/bin/bash -c 'set -e; " + ": > /home/.nethack/record; " + "if ! nethack_output=$(nethack -s all 2>&1); then " + - "printf \"%s\\n\" \"$nethack_output\" >&2; exit 1; fi; " + - "case \"$nethack_output\" in *\"Cannot open record file\"*) " + - "printf \"%s\\n\" \"$nethack_output\" >&2; exit 1;; esac; " + - "printf \"HOMEBREW_NETHACK_STATE_OK\\n\"'", + 'printf "%s\\n" "$nethack_output" >&2; exit 1; fi; ' + + 'case "$nethack_output" in *"Cannot open record file"*) ' + + 'printf "%s\\n" "$nethack_output" >&2; exit 1;; esac; ' + + 'printf "HOMEBREW_NETHACK_STATE_OK\\n"\'', "HOMEBREW_NETHACK_STATE_OK", 180_000, ); - const basePackages = [ - "kandelo-dev/tap-core/dash", - "kandelo-dev/tap-core/git", - "kandelo-dev/tap-core/nethack", - ]; - expect(mirrorPlan.assets).toHaveLength(39); - let lazyRows = await waitForLazyPackageRows( + lazyRows = await waitForLazyPackageRows( page, - new Set(), - basePackages, - mirrorPlan, + basePriorSources, + BASE_EXPECTED_PACKAGES, + shell.mirrorPlan, + ); + expect( + packageNamesForRows( + lazyRows.filter(({ source }) => !basePriorSources.has(source)), + shell.mirrorPlan, + ), + ).toEqual([...BASE_EXPECTED_PACKAGES].sort()); + + expect(packageNamesForRows(lazyRows, shell.mirrorPlan)).toEqual( + [...BASE_EXPECTED_PACKAGES, ...BREW_EXPECTED_PACKAGES].sort(), ); - expect(packageNamesForRows(lazyRows, mirrorPlan)).toEqual([...basePackages].sort()); + expect( + shell.mirrorPlan.assets.length - bottleRows(lazyRows).length, + ).toBeGreaterThan(0); + assertBottleLedger(lazyRows, shell.mirrorPlan); + expect(shell.legacyArtifactDownloads).toEqual([]); +}); + +test("a separate fresh shell keeps each language bottle independent of brew", async ({ + page, +}) => { + test.skip( + !strict, + "exact Homebrew main-shell CI configures this acceptance test", + ); + test.setTimeout(420_000); + + const shell = await bootExactShellPage(page); + let lazyRows = await readLazyDownloadRows(page); + expect(lazyRows).toEqual([]); + await assertBootstrapStillDeferred(shell, lazyRows); for (const invocation of MAIN_SHELL_LANGUAGE_RUNTIME_INVOCATIONS) { const priorSources = new Set(lazyRows.map(({ source }) => source)); @@ -354,57 +702,48 @@ test("the exact public-bottle shell preserves shell, language, and NetHack behav page, priorSources, [invocation.packageName], - mirrorPlan, + shell.mirrorPlan, ); const newRows = nextRows.filter(({ source }) => !priorSources.has(source)); - const fetchedPackages = packageNamesForRows(newRows, mirrorPlan); + const fetchedPackages = packageNamesForRows(newRows, shell.mirrorPlan); expect(fetchedPackages).toContain(invocation.packageName); const allowedPackages = new Set([ invocation.packageName, ...invocation.dependencyPackages, + ...invocation.launcherPackages, ]); - expect(fetchedPackages.every((name) => allowedPackages.has(name))).toBe(true); - const otherLanguages = fetchedPackages.filter((name) => - name !== invocation.packageName && - MAIN_SHELL_LANGUAGE_RUNTIME_INVOCATIONS.some( - (candidate) => candidate.packageName === name, - ) + expect(fetchedPackages.every((name) => allowedPackages.has(name))).toBe( + true, + ); + const otherLanguages = fetchedPackages.filter( + (name) => + name !== invocation.packageName && + MAIN_SHELL_LANGUAGE_RUNTIME_INVOCATIONS.some( + (candidate) => candidate.packageName === name, + ), ); expect(otherLanguages).toEqual([]); lazyRows = nextRows; } - const requiredPackages = [ - ...basePackages, - ...MAIN_SHELL_LANGUAGE_RUNTIME_INVOCATIONS.map(({ packageName }) => packageName), - ]; - const fetchedPackages = packageNamesForRows(lazyRows, mirrorPlan); - for (const packageName of requiredPackages) { - expect(fetchedPackages).toContain(packageName); - } - const allowedPackages = new Set([ - ...requiredPackages, + const expectedPackages = new Set([ + ...MAIN_SHELL_LANGUAGE_RUNTIME_INVOCATIONS.map( + ({ packageName }) => packageName, + ), ...MAIN_SHELL_LANGUAGE_RUNTIME_INVOCATIONS.flatMap( ({ dependencyPackages }) => dependencyPackages, ), + ...MAIN_SHELL_LANGUAGE_RUNTIME_INVOCATIONS.flatMap( + ({ launcherPackages }) => launcherPackages, + ), ]); - expect(fetchedPackages.every((name) => allowedPackages.has(name))).toBe(true); - expect(mirrorPlan.assets.length - lazyRows.length).toBeGreaterThan(0); - - // The kernel worker's lazy-download ledger is the transport authority. Raw - // browser request events are only diagnostic: service-worker delivery can - // notify Playwright after the guest has consumed the verified response. - const assetByUrl = new Map(mirrorPlan.assets.map((asset) => [asset.url, asset])); - for (const row of lazyRows) { - const asset = row.source === null ? undefined : assetByUrl.get(row.source); - expect(asset, `unplanned lazy row ${row.asset}`).toBeDefined(); - expect(row.asset).toBe(asset!.asset); - expect(row.kind).toBe("tree"); - expect(row.status).toBe("complete"); - expect(row.loadedBytes).toBe(String(asset!.bytes)); - expect(row.totalBytes).toBe(String(asset!.bytes)); - expect(Number(row.eventCount)).toBeGreaterThanOrEqual(3); - } - - expect(legacyArtifactDownloads).toEqual([]); + expect(packageNamesForRows(lazyRows, shell.mirrorPlan)).toEqual( + [...expectedPackages].sort(), + ); + await assertBootstrapStillDeferred(shell, lazyRows); + expect( + shell.mirrorPlan.assets.length - bottleRows(lazyRows).length, + ).toBeGreaterThan(0); + assertBottleLedger(lazyRows, shell.mirrorPlan); + expect(shell.legacyArtifactDownloads).toEqual([]); }); diff --git a/apps/browser-demos/test/rootfs-export.spec.ts b/apps/browser-demos/test/rootfs-export.spec.ts new file mode 100644 index 0000000000..03689fa908 --- /dev/null +++ b/apps/browser-demos/test/rootfs-export.spec.ts @@ -0,0 +1,189 @@ +import { mkdir, rm, writeFile } from "node:fs/promises"; +import { fileURLToPath } from "node:url"; + +import { expect, test } from "@playwright/test"; + +import { MemoryFileSystem } from "../../../host/src/vfs/memory-fs"; + +interface RootfsExportAcceptanceResult { + persistedText: string; + firstExportSha256: string; + secondExportSha256: string; + firstExportBytes: number; + secondExportBytes: number; + liveProcessExitCode: number; + liveProcessExportError: string; + teardownProcessExitCode: number; + teardownExportError: string; + overlappingExportError: string; + overlappingWriteError: string; + lazyReadText: string; + lateWritePresentInExport: boolean; + writeAfterExportText: string; + diagnostics: Array<{ source: string; message: string }>; + lazyEntries: Array<{ path: string; url: string; size: number }>; +} + +declare global { + interface Window { + __homebrewVfsTestReady: boolean; + __runRootfsExportAcceptance: (request: { + vfsUrl: string; + writePath: string; + writeText: string; + liveProcessUrl: string; + teardownProcessUrl: string; + lazyReadPath: string; + lazyReadUrl: string; + lazyReadText: string; + lateWritePath: string; + lateWriteText: string; + }) => Promise; + __releaseRootfsExportLazyResponse: () => Promise; + } +} + +const fixtureRoot = new URL( + "../public/__kandelo-acceptance/rootfs-export/", + import.meta.url, +); +const liveProcessPath = fileURLToPath( + new URL("../../../examples/block-forever.wasm", import.meta.url), +); +const teardownProcessPath = fileURLToPath( + new URL("../../../examples/thread-exit-group.wasm", import.meta.url), +); +const lazyRaceUrl = + "https://rootfs-export-race.invalid/lazy-read-payload"; +const lazyRacePath = "/opt/lazy-export-race"; +const lazyRaceText = "lazy mutation completed before export\n"; +const lazyRaceBytes = new TextEncoder().encode(lazyRaceText); +const lateWritePath = "/state/rejected-during-export.txt"; +const lateWriteText = "write succeeds only after export\n"; + +function projectFixtureDir(projectName: string): URL { + if (!/^[a-z0-9-]+$/.test(projectName)) { + throw new Error(`unsafe Playwright project name: ${projectName}`); + } + return new URL(`${projectName}/`, fixtureRoot); +} + +test.beforeAll(async ({}, testInfo) => { + const fs = MemoryFileSystem.create(new SharedArrayBuffer(8 * 1024 * 1024)); + fs.mkdir("/state", 0o755); + fs.registerLazyFile( + "/opt/lazy-export-sentinel", + "https://packages.example.test/lazy-export-sentinel.wasm", + 123_456, + 0o755, + ); + fs.registerLazyFile( + lazyRacePath, + lazyRaceUrl, + lazyRaceBytes.byteLength, + 0o644, + ); + const fixtureDir = projectFixtureDir(testInfo.project.name); + const fixture = new URL("rootfs-export.vfs", fixtureDir); + await mkdir(fixtureDir, { recursive: true }); + await writeFile(fixture, await fs.saveImage()); +}); + +test.afterAll(async ({}, testInfo) => { + const fixtureDir = projectFixtureDir(testInfo.project.name); + await rm(fixtureDir, { recursive: true, force: true }); +}); + +test("browser rootfs export rejects unsafe races and reboots its snapshot", async ({ + page, + baseURL, +}, testInfo) => { + test.setTimeout(180_000); + if (!baseURL) throw new Error("Playwright baseURL is required"); + + let releaseLazyResponse!: () => void; + const lazyResponseReleased = new Promise((resolve) => { + releaseLazyResponse = resolve; + }); + let lazyRequestCount = 0; + let releaseCallCount = 0; + // WHY: holding the real fetch keeps one VFS mutation active long enough to + // prove that export waits for it while excluding later export and writes. + await page.route(lazyRaceUrl, async (route) => { + lazyRequestCount += 1; + await lazyResponseReleased; + await route.fulfill({ + status: 200, + body: Buffer.from(lazyRaceBytes), + headers: { + "Access-Control-Allow-Origin": "*", + "Cache-Control": "no-store", + "Content-Type": "application/octet-stream", + "Cross-Origin-Resource-Policy": "cross-origin", + }, + }); + }); + await page.exposeFunction("__releaseRootfsExportLazyResponse", () => { + releaseCallCount += 1; + releaseLazyResponse(); + }); + + await page.goto(new URL("/pages/homebrew-vfs-test/", baseURL).href); + await expect + .poll(() => page.evaluate(() => window.__homebrewVfsTestReady), { + timeout: 120_000, + }) + .toBe(true); + + const projectName = testInfo.project.name; + const request = { + vfsUrl: new URL( + `/__kandelo-acceptance/rootfs-export/${projectName}/rootfs-export.vfs`, + baseURL, + ).href, + writePath: "/state/persisted.txt", + writeText: "browser rootfs state survives reboot\n", + liveProcessUrl: new URL(`/@fs/${liveProcessPath}`, baseURL).href, + teardownProcessUrl: new URL(`/@fs/${teardownProcessPath}`, baseURL).href, + lazyReadPath: lazyRacePath, + lazyReadUrl: lazyRaceUrl, + lazyReadText: lazyRaceText, + lateWritePath, + lateWriteText, + }; + const result = await page.evaluate((acceptanceRequest) => + window.__runRootfsExportAcceptance(acceptanceRequest), request); + + expect(result.persistedText).toBe( + "browser rootfs state survives reboot\n", + ); + expect(result.firstExportBytes).toBeGreaterThan(0); + expect(result.secondExportBytes).toBeGreaterThan(0); + expect(result.firstExportSha256).toMatch(/^[0-9a-f]{64}$/); + expect(result.secondExportSha256).toMatch(/^[0-9a-f]{64}$/); + expect(result.liveProcessExitCode).toBe(143); + expect(result.liveProcessExportError).toContain( + "no live or tearing-down processes", + ); + expect(result.teardownProcessExitCode).toBe(0); + expect(result.teardownExportError).toContain( + "no live or tearing-down processes", + ); + expect(result.overlappingExportError).toContain( + "rootfs export is already in progress", + ); + expect(result.overlappingWriteError).toContain( + "rootfs export is in progress; cannot write a rootfs file", + ); + expect(result.lazyReadText).toBe(lazyRaceText); + expect(result.lateWritePresentInExport).toBe(false); + expect(result.writeAfterExportText).toBe(lateWriteText); + expect(result.diagnostics).toEqual([]); + expect(lazyRequestCount).toBe(1); + expect(releaseCallCount).toBe(1); + expect(result.lazyEntries).toEqual([{ + path: "/opt/lazy-export-sentinel", + url: "https://packages.example.test/lazy-export-sentinel.wasm", + size: 123_456, + }]); +}); diff --git a/apps/browser-demos/vite.config.ts b/apps/browser-demos/vite.config.ts index c969d8763c..791509523b 100644 --- a/apps/browser-demos/vite.config.ts +++ b/apps/browser-demos/vite.config.ts @@ -112,6 +112,10 @@ const browserBinaryResolution = createBrowserBinaryResolution(binaryDevAccess); const crossOriginIsolationHeaders = { "Cross-Origin-Opener-Policy": "same-origin", "Cross-Origin-Embedder-Policy": "require-corp", + // WebKit revalidates a module worker when a kernel is rebooted on the same + // page. Mark every dev/preview response same-origin so that cached worker + // responses remain admissible under COEP, including a 304 revalidation. + "Cross-Origin-Resource-Policy": "same-origin", "Service-Worker-Allowed": "/", }; @@ -467,6 +471,41 @@ function injectCoiServiceWorker(): Plugin { }; } +/** + * Keep local module-worker reloads usable under COEP in WebKit. + * + * WebKit 26.5 rejects a second same-page module Worker load when it + * conditionally revalidates Vite's transformed worker response, even though + * both the original response and the page carry matching COEP/CORP headers. + * Removing only the worker request validators makes Vite return the same + * transformed bytes with a normal 200 response. Production assets do not use + * this middleware; the deployed service worker adds the isolation headers to + * its cached response itself. + */ +function forceFreshDevWorkerResponses(): Plugin { + function attachMiddleware( + middlewares: ViteDevServer["middlewares"] | PreviewServer["middlewares"], + ): void { + middlewares.use((req, _res, next) => { + if (req.headers["sec-fetch-dest"] === "worker") { + delete req.headers["if-none-match"]; + delete req.headers["if-modified-since"]; + } + next(); + }); + } + + return { + name: "force-fresh-dev-worker-responses", + configureServer(server) { + attachMiddleware(server.middlewares); + }, + configurePreviewServer(server) { + attachMiddleware(server.middlewares); + }, + }; +} + /** * Vite plugin: inject the service worker CORS proxy URL. Local dev/preview * uses the Vite same-origin proxy by default so the service worker can read @@ -675,6 +714,7 @@ export default defineConfig({ rewriteNavLinks(), injectGitRevision(), injectCoiServiceWorker(), + forceFreshDevWorkerResponses(), injectCorsProxyUrl(), devCorsProxyMiddleware(), ], diff --git a/docs/architecture.md b/docs/architecture.md index 674865791a..580f0fd962 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -901,6 +901,20 @@ changed by another worker. `materializeAll: true` resolves both standalone and archive-backed entries and fails instead of emitting an image that still depends on a deferred URL. +Kernel-owned machines expose that same durable boundary through +`NodeKernelHost.exportRootfsImage()` and +`BrowserKernel.exportRootfsImage()`. Export is available only after a +VFS-backed kernel has initialized and every guest process and worker teardown +has completed. The owning worker closes a snapshot gate before its first +asynchronous wait, drains host-side mutations that started earlier, and rejects +later spawns, lazy registration, materializing reads, writes, unlinks, and +concurrent exports until serialization settles. The returned image contains +only the `/` image backend; boot-scoped scratch, device, and shared-memory +mounts are recreated on the next boot. Lazy descriptors and image metadata +remain part of the root image, so a deferred package that was never opened +stays deferred after export and restore. Callers must await the export before +destroying the host. + **Restore from an image:** ```typescript diff --git a/docs/browser-support.md b/docs/browser-support.md index ef44c63e0c..ecafaa6877 100644 --- a/docs/browser-support.md +++ b/docs/browser-support.md @@ -57,6 +57,12 @@ Service Worker ──MessagePort──> Kernel Worker │ `writeFileToVfs`, and `unlinkFileFromVfs`). The owning worker performs those mutations through the mounted VFS; the main thread never receives the live VFS `SharedArrayBuffer`. + A quiescent machine can return durable root-image bytes through + `BrowserKernel.exportRootfsImage()`. The worker rejects export while a guest + process or teardown is live, serializes it against the same staging and lazy + materialization RPCs, and transfers only the `/` image backend. Scratch, + device, and shared-memory mounts are boot-local and are recreated when those + bytes start another machine. - **Legacy shared VFS** (`memfs:` constructor option + `kernel.spawn()`): main thread holds a `MemoryFileSystem` and shares the SAB with the kernel worker. Used by demos that fetch transient binaries at runtime (test runners, REPLs that load arbitrary user code, benchmark suites). The main thread transfers each program's bytes, but the Rust `ProcessTable` allocates the PID and the worker returns it. Top-level creation, guest fork/spawn, and thread clone all draw from that one authoritative task-ID sequence; no browser or host-side allocator exists. - **Exec reads from filesystem**: Like a real OS, `exec()` reads binaries from the kernel-side `MemoryFileSystem`. Programs are baked into the VFS image at build time (or written by the page in the legacy path before spawning). Symlinks are used for multicall binaries (e.g., coreutils). - **dinit for service supervision**: Multi-process demos (nginx, redis, mariadb, nginx-php, wordpress, lamp, mariadb-test) bake `/sbin/dinit` and per-service files under `/etc/dinit.d/` into the VFS image via `addDinitInit()` (`images/vfs/scripts/dinit-image-helpers.ts`). dinit is the first user process, not PID 1. It reaps its directly supervised children and handles `depends-on` ordering and bootstrap-then-daemon chains. Synthetic PID 1 has no wait loop, so Kandelo does not yet reap children reparented to it. Page code waits for service-ready via `onListenTcp` (port-bind) callbacks, then starts driving the demo over kernel-loopback TCP or the HTTP bridge. diff --git a/docs/homebrew-publishing.md b/docs/homebrew-publishing.md index 8bcb36066d..2e8735a33b 100644 --- a/docs/homebrew-publishing.md +++ b/docs/homebrew-publishing.md @@ -31,14 +31,15 @@ Homebrew's `bottle do` block. The patched Homebrew Ruby tree used by a guest is a dedicated Kandelo program package named `homebrew-bootstrap`; it is not a Formula bottle. The package -emits `homebrew-bootstrap.zip` from a sealed exact Homebrew checkout and the -reviewed guest-platform patch. Its source lock at +atomically emits `homebrew-bootstrap.zip` from a sealed exact Homebrew checkout +and the reviewed guest-platform patch plus `homebrew-brew.env`, which owns the +matching architecture and system-environment policy. Its source lock at `homebrew/homebrew-bootstrap-source-lock.json` binds all source, patch, prepared-tree, portable-Ruby, archive-producing Git, and final-archive -identities. The first package revision is sealed by the exact final ZIP -SHA-256 and byte count recorded in that lock. Consumer cutover is a separate -rollout step, so introducing the package does not change shell or -bootstrap-image consumers. +identities. The program-package generation binds both declared members to one +recipe, dependency closure, cache identity, and immutable release archive; +consumers must resolve the canonical nested member paths together rather than +recreating `brew.env` or resolving a mutable flat fallback. ## Repositories And Ownership @@ -464,14 +465,63 @@ offline after its disposable Ruby dependencies have been provisioned and sealed. Kandelo guests instead receive upstream Homebrew commit `4ead8619231cb15cbe15e8e8188081e347d6f7cd` through the dedicated `homebrew-bootstrap` program package. Guest acceptance must materialize that -package through the canonical ABI release index and verify its package-output -receipt, archive identity, cache key, and locked inner ZIP before booting it. +package through the canonical ABI release index and verify its complete +two-member generation, package-output receipt, archive identity, cache key, +and locked inner ZIP before booting it. The PR staging release is evidence for the package build, not a durable consumer URL. Until the package is activated in the canonical ABI index and the tap has adopted these Requirement declarations under a compatible pinned publisher, the full guest install lifecycle remains a rollout gate rather than a supported user-facing contract. +#### Exact Chromium guest-lifecycle fixture + +The Node.js and Chromium lifecycle runners use the same generated guest +scripts and host-neutral phase runner. They tap exact first- and third-party +revisions, install and reinstall bottles, export and reboot the rootfs, execute +the persisted packages, prove the pinned upgrade is a no-op, then uninstall and +untap. A browser fixture only supplies host transport identities; it cannot +replace or weaken those guest assertions. + +The live Playwright proof is disabled unless +`KANDELO_HOMEBREW_GUEST_BROWSER_LIFECYCLE_LIVE=1` and +`KANDELO_HOMEBREW_GUEST_BROWSER_LIFECYCLE_FIXTURE_PATH` are both set. The JSON +file uses schema 1 and declares: + +- `allowLiveNetwork: true`, an explicit opt-in checked before any fixture + request; +- `transportMode`, either `closed` or `public`; +- exact `url`, `sha256`, and `bytes` records for the main-shell image and the + bootstrap spec, archive, and environment; +- an exact bottle-mirror plan record; closed transport also requires one exact + payload record for every plan asset, while public transport forbids local + payload bytes; +- exact 40-character `coreRevision` and `canaryRevision` values; and +- a bounded `timeoutMs` from 1,000 through 1,800,000. + +All artifact URLs are canonical credential-free HTTPS identities. Chromium +may retrieve them through the same-origin test proxy, but digest validation and +the VFS keep the original immutable URL as authority. The downloaded mirror +plan must be byte-identical to the plan embedded in the image and must derive +its release tag and every payload URL from its complete collection digest. +Closed payloads are then handed to the worker as an exhaustive transport: +an undeclared request fails instead of falling back to ambient network. + +Run an exact reviewed fixture with: + +```bash +cd apps/browser-demos +bash ../../scripts/dev-shell.sh env \ + KANDELO_HOMEBREW_GUEST_BROWSER_LIFECYCLE_LIVE=1 \ + KANDELO_HOMEBREW_GUEST_BROWSER_LIFECYCLE_FIXTURE_PATH=/absolute/path/to/fixture.json \ + npx playwright test test/homebrew-guest-lifecycle.spec.ts --project=chromium +``` + +Without those two variables, CI still runs the browser admission test proving +that a fixture without live-network opt-in makes no external request. A skipped +live test is preparation evidence only; it is not evidence that a bottle was +published, poured, or executed. + The native launcher installs each selected direct dependency as an explicit `homebrew/core/` reference under an ephemeral native prefix. Each install uses Homebrew's normal dependency resolution and completes its full transitive @@ -1345,8 +1395,11 @@ the ABI from `crates/shared`, resolves the Node kernel, canonical rootfs package set, and Homebrew bootstrap programs through `xtask build-deps`, and calls `scripts/prepare-homebrew-bootstrap-source.sh` to prepare Homebrew. The dedicated `homebrew-bootstrap` package uses that same preparer and records -byte identity with this still-current image path; switching this image to -consume the package is a separate consumer change. Source preparation verifies +the source ZIP and `homebrew-brew.env` as one package generation. The main +shell resolves that exact generation, embeds the small environment policy, and +registers the source ZIP as a package-level lazy tree behind `/usr/bin/brew`; +the separate diagnostic bootstrap image above remains an eager integration +artifact. Source preparation verifies the reviewed patch SHA-256, refuses an upstream revision where the patch does not apply, limits the patch to its declared Homebrew files, and archives the patched Git tree with a fixed timestamp and UTC timezone. @@ -1407,11 +1460,16 @@ not prove shebang dispatch, `/usr/bin/brew` alias execution, an install, or a bottle download. ABI 41 raised every fork continuation reserve from 16 KiB to 60 KiB. The -earlier ABI 39 dispatcher and `/usr/bin/brew` alias-launcher measurements needed -20,012 and 29,212 bytes respectively. The exact candidate bootstrap also found -a 49,232-byte Bash child continuation in the recursive command evaluator, -which the 48 KiB draft reserve rejected truthfully. All three now fit without -weakening the overrun guard. Repeat the probe with +earlier ABI 39 dispatcher, `/usr/bin/brew` alias-launcher, and recursive Bash +measurements needed 20,012, 29,212, and 49,232 bytes respectively, so the +shallow source/bootstrap probe above fits. The complete main-shell proof found +that Ruby-backed Homebrew startup can require 64,256 and 66,092 bytes. ABI 41 +therefore does not support the full guest Homebrew lifecycle; the truthful +overrun is a platform failure, not a package defect. Full support waits for the +reviewed dynamic continuation design, allocation-failure recovery, ABI-current +bottle rebuild, and exact Node/browser proof. Do not increase a package-local +limit, bypass command substitution, or accept the diagnostic as success. +Repeat the shallow source probe with `--brew-script /usr/bin/brew` when validating the alias path; `$0` must remain `/usr/bin/brew`, the launcher must recognize the symlink, and the command must print the Homebrew version rather than silently falling back to `/Library`. diff --git a/docs/package-management.md b/docs/package-management.md index 3546fda489..636a7a2ffa 100644 --- a/docs/package-management.md +++ b/docs/package-management.md @@ -534,12 +534,15 @@ members or use HTTP ranges. Dependency bottles have separate identities and remain unfetched until a path owned by that dependency is used. The guest `brew` implementation is distributed separately from Formula -bottles as the `homebrew-bootstrap` program package. Its single declared -artifact, `homebrew-bootstrap.zip`, is a deterministic archive of one exact -upstream Homebrew commit plus Kandelo's reviewed guest-platform patch. Although -the artifact is not Wasm, it uses the ordinary program-package resolver, -program projection, cache key, and release archive contracts; its output -therefore declares `fork_instrumentation = "disabled"`. +bottles as the `homebrew-bootstrap` program package. One package generation +contains two declared outputs: `homebrew-bootstrap.zip`, a deterministic +archive of one exact upstream Homebrew commit plus Kandelo's reviewed +guest-platform patch, and `homebrew-brew.env`, the architecture tag and system +environment policy consumed with that exact tree. Consumers resolve both from +the same immutable generation; they must not reconstruct the environment file +or combine it with a ZIP from another build. Neither output is Wasm, but both +use the ordinary program-package resolver, projection, cache key, and release +archive contracts and therefore declare `fork_instrumentation = "disabled"`. `homebrew/homebrew-bootstrap-source-lock.json` is the reviewed source/output identity. It binds the upstream archive URL and SHA-256, sealed @@ -548,10 +551,12 @@ tree identities, portable Ruby version, archive-producing Git version, and final ZIP SHA-256/byte count. The package build imports the resolver-owned exact Git checkout into private scratch storage and performs no source fetch of its own. The lock also records the dedicated package output's exact SHA-256 -and byte count, so a rebuild cannot silently change guest Homebrew bytes. -Shell and bootstrap-image consumer cutover remains a separate change. Run the -build through `scripts/dev-shell.sh`; a different Git ZIP implementation fails -the exact output lock instead of publishing different bytes. +and byte count, so a rebuild cannot silently change guest Homebrew source +bytes. The package recipe emits the environment member in the same atomic +generation, and the program projection records both canonical nested member +paths. Run the build through `scripts/dev-shell.sh`; a different Git ZIP +implementation fails the exact output lock instead of publishing different +bytes. See [docs/homebrew-publishing.md](homebrew-publishing.md) for the Homebrew formula, sidecar, GHCR, VFS, and runtime validation contract. diff --git a/docs/plans/2026-07-21-homebrew-migration-execution-plan.md b/docs/plans/2026-07-21-homebrew-migration-execution-plan.md index d14aa64ef0..67e3dd9629 100644 --- a/docs/plans/2026-07-21-homebrew-migration-execution-plan.md +++ b/docs/plans/2026-07-21-homebrew-migration-execution-plan.md @@ -710,6 +710,144 @@ main-shell capability): existing sealed build/test plan and provenance. Unsupported dependency-skip flags, a synthetic core API, and a curated partial core tap remain rejected. +Working checkpoint (2026-07-23; not yet canonical): + +- The activation work is deliberately stacked behind the atomic package + generation and VFS-source contracts in PRs #1073, #1074, #1069, and #1076. + Land #1073's shared resolver boundary first. Then preserve the reviewed + commit boundaries for #1074, #1069, and #1076 in one combined candidate and + run one full exact-head gate, instead of paying three serial staging and + prepare-merge cycles for an already ordered stack. PR #1076 has merge + approval, but the combined candidate must still be replayed onto the exact + landed #1073 parent and pass exact-head CI before merge. This ordering keeps + the lazy Homebrew tree from bypassing package-generation freshness or source + integrity checks while reducing CI serialization rather than test scope. +- The bootstrap package now emits two members from one generation: the exact + Homebrew source archive and the environment file consumed with it. The + canonical resolver therefore proves they share one recipe, dependency + closure, cache identity, and immutable generation rather than resolving two + independently mutable files. +- Two clean candidate builds produced the same 6,025,043-byte compressed + 512 MiB-capacity image. They embed only `libcxx`, `ncurses`, and Bash; leave + 39 Formula trees and the Homebrew bootstrap deferred; install the ordinary + `/usr/bin/brew` entrypoint and `/etc/homebrew/brew.env`; and preserve the + unprivileged Homebrew ownership model. A derived eager build has the same + deferred-tree descriptors and bootstrap consumer identity, differing only + in which declared sources are materialized initially. +- The first exact runtime pass correctly rejected the candidate's stale tap + lock: it selected Bash revision 1, built without `compgen`, even though the + public tap already contains the corrected revision 2 rebuild 3. Do not + accept the warning or patch upstream `brew`. After the in-flight Tcl/Dinit + publication sequence finalizes one coherent catalog commit, advance both + tap pins to that commit, retain Bash revision 2, regenerate all contextual + package identities and the image artifact lock, and repeat Node.js and + Chromium proofs against the final bytes. +- An interim build against the already-finalized corrected Bash snapshot + produced a 6,044,463-byte image and proved that `brew --version`, `--prefix`, + `--repository`, `--cellar`, and `--cache` all pass through the ordinary lazy + `/usr/bin/brew` entrypoint. The first Ruby-backed command then exposed a real + ABI 41 platform ceiling: two valid fork continuations required 64,256 and + 66,092 bytes while the fixed buffer reserves 61,440. Do not weaken the test, + avoid command substitution, or patch Ruby/Homebrew around this. Audit and + finish the existing dynamic continuation work, including allocation-failure + recovery, Node/browser parity, ABI rollout, and exact rebuilt-bottle proof. +- The dynamic linked-chunk architecture in PR #1043 is the intended general + fix, but its current head is not mergeable evidence. PRs #979 and #1043 are + sibling ABI-42 transitions based far behind main and must become one ordered + unpublished ABI-42 tranche: authoritative kernel task identity first, then + transactional growable fork continuations. Preserve current-main wasm64 + `BigInt` address conversion, move the descriptor/import/export requirements + into generated ABI and publication guards, cover allocation recovery across + main, pthread, side-module, Node, and browser paths, and benchmark shallow + and overflowing forks before rollout. No ABI-42 binary or bottle release + exists, so both contracts may still ship as ABI 42 if they are composed and + published together; publishing either incomplete contract first forces the + combined transition to ABI 43. +- The ABI transition is one coordinated rebuild wave, not a series of mixed + package fixes: stage all selected programs with the final instrumenter, + publish one `binaries-abi-v42` set and one `bottles-abi-v42` catalog, rebuild + the shell VFS, and repeat exact Node/Chromium Homebrew plus conformance and + performance validation. Fixed-buffer growth or a Homebrew/Ruby workaround + is not an accepted intermediate endpoint. +- Erlang's `erl` entrypoint is a `/bin/sh` launcher, so Dash is an operational + runtime dependency even though the current tap Formula labels it test-only. + The main-shell proof names this launcher dependency separately from Erlang's + library closure. Before arbitrary mix-and-match installation is claimed, + promote Dash to a runtime Formula dependency and publish a rebuilt Erlang + bottle so the package owns that truth itself. + +Preparation checkpoint (2026-07-24; locally validated scaffolding, not yet a +successful public guest lifecycle): + +- Node.js and browser hosts can request an atomic image of the worker-owned + root filesystem only after the kernel has become quiescent. The worker closes + a snapshot gate before awaiting earlier filesystem mutations, rejects new + process/lazy-tree mutations while saving, and refuses export while a process + is live or still tearing down. The resulting image is durable root-mount + state. Boot-scoped `/tmp`, `/var/tmp`, `/var/log`, `/var/run`, `/home/user`, + `/root`, `/srv`, `/dev`, and `/dev/shm` mounts are intentionally + reconstructed for the next boot rather than serialized as durable package + state. +- The guest lifecycle harness consumes the same VFS-embedded mirror plan, + exact closed bottle bindings, deferred `homebrew-bootstrap` package tree, and + image-owned Bash as the main-shell smoke. It takes exact 40-character core + and independent-canary revisions rather than embedding mutable branch + defaults. The core revision must equal the canonical repository, tap, and + checkout in the image's `/etc/kandelo/homebrew-vfs.json`, checked through the + same authoritative catalog parser as the complete main-shell contract. It + uses stock `brew tap`, `install`, `reinstall`, `outdated`, `upgrade`, + `uninstall`, and `untap`; it does not copy support files, rewrite Formulae, + emulate Formula resolution on the host, or create `homebrew/core`. +- The lazy shell already has direct-composed receipts for Bzip2 and M4. To + distinguish a real stock install from Homebrew's "already installed" path, + the harness first removes only those two receipts through + `brew uninstall --ignore-dependencies`, then installs Bzip2 from the core tap + and M4 from the independent tap. Dash stays installed because it is both + M4's cross-tap runtime dependency and the shell's `/bin/sh`; the M4 receipt + must name that exact first-party dependency. The exported image is then + rebooted, its shell executable is resolved again from exported bytes, both + packages execute again, and the proof cleans up its own installs. Closed + phase-two transport omits every URL phase one completed, and all transport + modes reject any repeated event for one of those URLs, so a re-deferred tree + cannot hide an export durability regression by fetching the original bytes. + Unexpected host diagnostics are fatal rather than accepted alongside a + successful guest marker. +- Node.js and Chromium now share that lifecycle orchestration and the exact + generated phase scripts; their adapters differ only in host transport, + process launch, output capture, and rootfs export. The browser fixture is + rejected before any fixture network access unless it explicitly opts into a live run + and binds the image, bootstrap spec/archive/environment, embedded bottle + mirror plan, every closed payload when used, and both tap revisions to exact + immutable URLs, byte lengths, and SHA-256 values. Offline unit coverage and + a real Chromium admission test are green. This is prepared browser + scaffolding, not live bottle evidence: the lifecycle test remains skipped + unless both + `KANDELO_HOMEBREW_GUEST_BROWSER_LIFECYCLE_LIVE=1` and + `KANDELO_HOMEBREW_GUEST_BROWSER_LIFECYCLE_FIXTURE_PATH` name an exact + reviewed fixture. +- The first live run remains gated by one coherent ABI-42 generation. The + `homebrew-bootstrap` recipe still declares ABI 41, the main-shell mirror + requires the complete public ABI-42 closure, and the core and independent + taps need final compatible immutable revisions plus public ABI-42 Bzip2, M4, + and Dash bottles. Until those inputs exist, static contract tests can prove + SHA validation, generated shell syntax, canonical origins, absence of + Formula mutation, exact closed-asset binding, and export/reboot behavior, but + cannot truthfully claim a public bottle install. +- Loud ABI- and digest-mismatch evidence remains a separate negative live + fixture. It must bind an immutable intentionally wrong artifact or a closed + guest-network response to an exact reviewed expectation; it must not corrupt + a production package, rewrite a Formula after tapping it, or weaken + Homebrew's own checksum and Kandelo's ABI enforcement paths. +- `brew outdated` plus a no-op `brew upgrade` at the exact pinned revisions is + the first upgrade-state milestone. The no-op proof compares each selected + Formula's exact prefix, reported version, receipt digest, and complete keg + content digest before and after the command. A real old-to-new upgrade + requires two immutable bottle versions and remains a live fixture. + `brew update` also remains separate because the bootstrap is a reviewed + patched source archive rather than a Git checkout. Ambient source replacement + is not safe until an update contract preserves and revalidates the Kandelo + platform boundary. + Acceptance: - Stock upstream Homebrew, with only the documented Kandelo target/platform @@ -784,6 +922,21 @@ Formula files. files from the embedded closure (1,534,914 uncompressed bytes); measure the exact compressed base-image delta before considering one docs sidecar for the embedded set. +10. After the first mostly-lazy shell cutover, make bottles addressable by the + smallest correct **activation group**, beginning with the independent + programs in `posix-utils-lite`. Preserve one Formula and one reviewed bottle + as the build, test, receipt, and publication unit, but publish a + content-addressed activation-group inventory and independently retrievable + objects in that bottle's GitHub package. Each lazy command entry must name + its owning bottle, group, installed paths, digests, sizes, modes, links, and + inseparable runtime companions. A self-contained utility may occupy a + one-binary group, so invoking `cat` does not download every other + `posix-utils-lite` program. A data-rich application such as Vim instead + keeps its executable, runtime scripts, syntax data, defaults, and other + required files in one cohesive group; it must never appear runnable after + fetching only its executable. Implement this as a generic bottle-owned + activation contract rather than a command-name special case, and retain an + eager materialization path derived from the same inventory. Acceptance: @@ -802,6 +955,14 @@ Acceptance: deferred package. The deferred case fetches only the owning bottle, page links and `MANPATH` resolve through normal Homebrew layout, and the package inventory rejects an applicable Formula that silently drops its pages. +- Node.js and Chromium can invoke two previously untouched + `posix-utils-lite` commands independently; network evidence proves that each + first use retrieves only its content-addressed activation group from the + owning GitHub package, a second use is cache-only, and neither use downloads + the full bundle. A data-rich fixture proves that first use atomically + materializes all declared supporting files and never exposes an + executable-only partial installation. Tampered members, inventories, modes, + incomplete groups, and cross-bottle ownership are rejected before execution. ### Phase 7: Bottle-compose service, application, and selectable VFS layers diff --git a/homebrew/main-shell-brew-package-tree.json b/homebrew/main-shell-brew-package-tree.json new file mode 100644 index 0000000000..6cb68906b4 --- /dev/null +++ b/homebrew/main-shell-brew-package-tree.json @@ -0,0 +1,24 @@ +{ + "schema": 1, + "kind": "kandelo-package-deferred-zip-tree", + "id": "homebrew-bootstrap/source-tree", + "content_role": "source-tree", + "package": { + "name": "homebrew-bootstrap", + "output": "homebrew-bootstrap.zip" + }, + "archive": { + "url": "homebrew-bootstrap.zip", + "mode_policy": "portable-posix-v1" + }, + "mount_prefix": "/home/linuxbrew/.linuxbrew", + "owner": { + "uid": 1000, + "gid": 1000 + }, + "activation": { + "mode": "first-use", + "capabilities": ["homebrew:bootstrap"], + "roots": ["/home/linuxbrew/.linuxbrew/bin/brew"] + } +} diff --git a/homebrew/main-shell-lazy-artifact-lock.json b/homebrew/main-shell-lazy-artifact-lock.json index 421b98cd5f..3ebdcde74d 100644 --- a/homebrew/main-shell-lazy-artifact-lock.json +++ b/homebrew/main-shell-lazy-artifact-lock.json @@ -3,7 +3,7 @@ "kind": "kandelo-homebrew-lazy-shell-artifact-lock", "source_date_epoch": 0, "image": { - "sha256": "7a08d342b4e1d00976623d18da74a2d1a923f4eafe88c93253e2947ec0a49cce", - "bytes": 5885691 + "sha256": "1a106b0195254aed3800868c288ae2cac54be967b08b8688461c7b1dd5caf229", + "bytes": 6025043 } } diff --git a/homebrew/test/homebrew_guest_lifecycle_browser.ts b/homebrew/test/homebrew_guest_lifecycle_browser.ts new file mode 100644 index 0000000000..705aebd6a4 --- /dev/null +++ b/homebrew/test/homebrew_guest_lifecycle_browser.ts @@ -0,0 +1,286 @@ +import { BrowserKernel } from "../../host/src/browser-kernel-host"; +import { ABI_VERSION } from "../../host/src/generated/abi"; +import { + MemoryFileSystem, + type LazyDownloadEvent, +} from "../../host/src/vfs/memory-fs"; +import { + loadHomebrewGuestLifecycleBrowserFixture, + type HomebrewGuestLifecycleBrowserFixture, +} from "./homebrew_guest_lifecycle_browser_fixture"; +import { + HOMEBREW_GUEST_LIFECYCLE_ENV, + type HomebrewGuestLifecycleMachine, + runHomebrewGuestLifecycle, +} from "./homebrew_guest_lifecycle_runner"; +import { + deriveHomebrewGuestLifecycleRuntimeInputs, + type HomebrewGuestLifecycleRuntimeInputs, +} from "./homebrew_guest_lifecycle_runtime_inputs"; +import { + assertNoUnexpectedHostDiagnostics, +} from "./homebrew_guest_lifecycle_runtime_contract"; + +export interface HomebrewGuestLifecycleBrowserResult { + exportedImageSha256: string; + exportedImageBytes: number; + coreRevision: string; + canaryRevision: string; + phaseOneCompletedUrls: string[]; + phaseOneLazyDownloads: readonly LazyDownloadEvent[]; + phaseTwoLazyDownloads: readonly LazyDownloadEvent[]; +} + +type FetchLike = ( + input: string | URL, + init?: RequestInit, +) => Promise; + +const MAX_CAPTURED_OUTPUT_BYTES = 8 * 1024 * 1024; +const MAX_CAPTURED_DIAGNOSTICS = 1_000; + +/** + * Run the same stock-Homebrew lifecycle used by the Node acceptance runner in + * Chromium. This adapter owns only browser transport and worker mechanics; + * the guest scripts, reboot boundary, and assertions live in the shared + * host-neutral runner. + */ +export async function runHomebrewGuestLifecycleInBrowser(options: { + fixture: unknown; + kernelWasm: ArrayBuffer; + corsProxyUrl: string; + fetchImpl?: FetchLike; + afterMachineDestroy?: () => Promise; +}): Promise { + const loaded = await loadHomebrewGuestLifecycleBrowserFixture( + options.fixture, + { + fetchImpl: options.fetchImpl, + sourceUrl: (canonicalUrl) => + createCorsProxySourceUrl(options.corsProxyUrl, canonicalUrl), + }, + ); + const fixture = loaded.fixture; + MemoryFileSystem.assertImageKernelAbi( + loaded.imageBytes, + ABI_VERSION, + "Homebrew guest lifecycle browser image", + ); + const publicTransport = fixture.transportMode === "public"; + const runtime = deriveHomebrewGuestLifecycleRuntimeInputs({ + imageBytes: loaded.imageBytes, + bootstrapSpecBytes: loaded.bootstrapSpecBytes, + bootstrapArchiveBytes: loaded.bootstrapArchiveBytes, + bootstrapArchiveSha256: fixture.bootstrap.archive.sha256, + bootstrapEnvironmentBytes: loaded.bootstrapEnvironmentBytes, + coreRevision: fixture.revisions.coreRevision, + transportMode: fixture.transportMode, + expectedEmbeddedBottlePlanBytes: loaded.bottleMirrorPlanBytes, + lazyUrlBase: publicTransport + ? new URL(".", fixture.bootstrap.archive.url).href + : "https://closed.kandelo.invalid/homebrew-guest-lifecycle/", + ...(publicTransport + ? { + expectedBootstrapTransportUrl: fixture.bootstrap.archive.url, + } + : { + closedBottleAssets: loaded.closedBottleAssets!, + }), + }); + + const result = await runHomebrewGuestLifecycle({ + runtime, + revisions: fixture.revisions, + timeoutMs: fixture.timeoutMs, + createMachine: (machineRuntime) => + createBrowserLifecycleMachine({ + runtime: machineRuntime, + kernelWasm: options.kernelWasm, + corsProxyUrl: options.corsProxyUrl, + afterDestroy: options.afterMachineDestroy, + }), + }); + return { + exportedImageSha256: await sha256(result.exportedImage), + exportedImageBytes: result.exportedImage.byteLength, + coreRevision: fixture.revisions.coreRevision, + canaryRevision: fixture.revisions.canaryRevision, + phaseOneCompletedUrls: [...result.phaseOneCompletedUrls].sort(), + phaseOneLazyDownloads: result.phaseOneLazyDownloads, + phaseTwoLazyDownloads: result.phaseTwoLazyDownloads, + }; +} + +export function createCorsProxySourceUrl( + corsProxyUrl: string, + canonicalUrl: string, +): string { + const proxy = new URL(corsProxyUrl, globalThis.location?.href); + if ( + ( + proxy.protocol !== "http:" && + proxy.protocol !== "https:" + ) || + proxy.username !== "" || + proxy.password !== "" || + proxy.hash !== "" + ) { + throw new Error("Homebrew browser lifecycle CORS proxy URL is invalid"); + } + proxy.searchParams.set("url", canonicalUrl); + return proxy.href; +} + +function createBrowserLifecycleMachine(options: { + runtime: HomebrewGuestLifecycleRuntimeInputs; + kernelWasm: ArrayBuffer; + corsProxyUrl: string; + afterDestroy?: () => Promise; +}): HomebrewGuestLifecycleMachine { + const lazyDownloads: LazyDownloadEvent[] = []; + const diagnostics: string[] = []; + let stdout = ""; + let stderr = ""; + let outputBytes = 0; + let outputLimitExceeded = false; + const stdoutDecoder = new TextDecoder(); + const stderrDecoder = new TextDecoder(); + const capture = (bytes: Uint8Array, stream: "stdout" | "stderr"): void => { + outputBytes += bytes.byteLength; + if (outputBytes > MAX_CAPTURED_OUTPUT_BYTES) { + outputLimitExceeded = true; + return; + } + if (stream === "stdout") { + stdout += stdoutDecoder.decode(bytes, { stream: true }); + } else { + stderr += stderrDecoder.decode(bytes, { stream: true }); + } + }; + const kernel = new BrowserKernel({ + kernelOwnedFs: true, + maxWorkers: 8, + corsProxyUrl: options.corsProxyUrl, + onStdout: (bytes) => capture(bytes, "stdout"), + onStderr: (bytes) => capture(bytes, "stderr"), + onHostDiagnostic: (diagnostic) => { + if (diagnostics.length < MAX_CAPTURED_DIAGNOSTICS) { + diagnostics.push(diagnostic.message); + } + }, + onLazyDownload: (event) => lazyDownloads.push(event), + }); + + return { + lazyDownloads, + diagnostics, + start: () => + kernel.initFromImage({ + kernelWasm: options.kernelWasm, + vfsImage: options.runtime.imageBytes, + lazyUrlBase: options.runtime.lazyUrlBase, + ...(options.runtime.lazyAssets === undefined + ? {} + : { closedLazyAssets: options.runtime.lazyAssets }), + }), + runShellScript: async (scriptOptions) => { + const stdoutStart = stdout.length; + const stderrStart = stderr.length; + const diagnosticStart = diagnostics.length; + let pid: number | undefined; + let timeout: ReturnType | undefined; + try { + const exit = kernel.spawn( + toArrayBuffer(scriptOptions.shellBytes), + [scriptOptions.shellArgv0, "-c", scriptOptions.script], + { + env: [...HOMEBREW_GUEST_LIFECYCLE_ENV], + cwd: "/home/user", + uid: 1000, + gid: 1000, + stdin: new Uint8Array(), + onStarted: (startedPid) => { + pid = startedPid; + }, + }, + ); + const timedOut = new Promise((_resolve, reject) => { + timeout = setTimeout( + () => + reject( + new Error( + `${scriptOptions.label} timed out after ` + + `${scriptOptions.timeoutMs}ms`, + ), + ), + scriptOptions.timeoutMs, + ); + }); + const exitCode = await Promise.race([exit, timedOut]); + const scriptStdout = stdout.slice(stdoutStart); + const scriptStderr = stderr.slice(stderrStart); + if (exitCode !== 0) { + throw new Error( + `${scriptOptions.label} exited ${exitCode}; stdout=` + + `${JSON.stringify(scriptStdout)}; stderr=` + + `${JSON.stringify(scriptStderr)}; diagnostics=` + + `${JSON.stringify(diagnostics)}`, + ); + } + if (!scriptStdout.split(/\r?\n/).includes(scriptOptions.marker)) { + throw new Error( + `${scriptOptions.label} marker is missing; stdout=` + + `${JSON.stringify(scriptStdout)}; stderr=` + + `${JSON.stringify(scriptStderr)}`, + ); + } + assertNoUnexpectedHostDiagnostics( + diagnostics.slice(diagnosticStart), + scriptOptions.label, + ); + if (outputLimitExceeded) { + throw new Error( + `${scriptOptions.label} exceeded the ` + + `${MAX_CAPTURED_OUTPUT_BYTES}-byte output limit`, + ); + } + } catch (error) { + if (pid !== undefined) { + await kernel.terminateProcess(pid, 124).catch(() => {}); + } + throw error; + } finally { + if (timeout !== undefined) clearTimeout(timeout); + } + }, + exportRootfsImage: () => kernel.exportRootfsImage(), + destroy: async () => { + try { + await kernel.destroy(); + } finally { + await options.afterDestroy?.(); + } + }, + }; +} + +function toArrayBuffer(bytes: Uint8Array): ArrayBuffer { + const result = new ArrayBuffer(bytes.byteLength); + new Uint8Array(result).set(bytes); + return result; +} + +async function sha256(bytes: Uint8Array): Promise { + const owned = toArrayBuffer(bytes); + const digest = new Uint8Array( + await crypto.subtle.digest("SHA-256", owned), + ); + return Array.from( + digest, + (byte) => byte.toString(16).padStart(2, "0"), + ).join(""); +} + +export type { + HomebrewGuestLifecycleBrowserFixture, +}; diff --git a/homebrew/test/homebrew_guest_lifecycle_browser_fixture.test.ts b/homebrew/test/homebrew_guest_lifecycle_browser_fixture.test.ts new file mode 100644 index 0000000000..70a0f05334 --- /dev/null +++ b/homebrew/test/homebrew_guest_lifecycle_browser_fixture.test.ts @@ -0,0 +1,253 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import test from "node:test"; + +import { + encodeHomebrewBottleMirrorCollectionIdentity, + encodeHomebrewBottleMirrorPlan, + HOMEBREW_BOTTLE_MIRROR_PLAN_ASSET, + HOMEBREW_BOTTLE_MIRROR_PLAN_KIND, + type HomebrewBottleMirrorPlan, +} from "../../host/src/homebrew-vfs-composer"; +import { homebrewRuntimeLayerPayloadAsset } from + "../../host/src/homebrew-runtime-layer-limits"; +import { + loadHomebrewGuestLifecycleBrowserFixture, + projectHomebrewGuestLifecycleBrowserFixture, +} from "./homebrew_guest_lifecycle_browser_fixture"; + +test("requires an explicit live-network opt-in before accepting URLs", () => { + const fixture = createFixture(); + assert.throws( + () => + projectHomebrewGuestLifecycleBrowserFixture({ + ...fixture.value, + allowLiveNetwork: false, + }), + /explicit live-network opt-in/, + ); +}); + +test("requires exact closed mirror inputs and forbids them in public mode", () => { + const fixture = createFixture(); + const { payloads: _payloads, ...planOnly } = fixture.value.bottleMirror; + assert.throws( + () => + projectHomebrewGuestLifecycleBrowserFixture({ + ...fixture.value, + bottleMirror: planOnly, + }), + /closed browser lifecycle transport requires exact bottle payloads/, + ); + assert.throws( + () => + projectHomebrewGuestLifecycleBrowserFixture({ + ...fixture.value, + transportMode: "public", + }), + /public transport forbids local payload bytes/, + ); + assert.doesNotThrow(() => + projectHomebrewGuestLifecycleBrowserFixture({ + ...fixture.value, + transportMode: "public", + bottleMirror: planOnly, + }) + ); +}); + +test("loads every exact fixture byte and binds payloads to the mirror plan", async () => { + const fixture = createFixture(); + const loaded = await loadHomebrewGuestLifecycleBrowserFixture( + fixture.value, + { + sourceUrl: (url) => url, + fetchImpl: createFixtureFetch(fixture.bytesByUrl), + }, + ); + + assert.deepEqual(loaded.imageBytes, fixture.imageBytes); + assert.deepEqual(loaded.bootstrapSpecBytes, fixture.specBytes); + assert.deepEqual(loaded.bootstrapArchiveBytes, fixture.archiveBytes); + assert.deepEqual(loaded.bootstrapEnvironmentBytes, fixture.environmentBytes); + assert.deepEqual(loaded.bottleMirrorPlanBytes, fixture.planBytes); + assert.deepEqual(loaded.closedBottleAssets, [{ + url: fixture.plan.assets[0]!.url, + sha256: fixture.plan.assets[0]!.sha256, + size: fixture.payloadBytes.byteLength, + bytes: fixture.payloadBytes, + }]); +}); + +test("rejects changed bytes and fixture identities that differ from the plan", async () => { + const fixture = createFixture(); + const changedBytesByUrl = new Map(fixture.bytesByUrl); + changedBytesByUrl.set( + fixture.value.bootstrap.archive.url, + new Uint8Array([99]), + ); + await assert.rejects( + () => + loadHomebrewGuestLifecycleBrowserFixture(fixture.value, { + sourceUrl: (url) => url, + fetchImpl: createFixtureFetch(changedBytesByUrl), + }), + /bootstrap archive.*changed SHA-256/, + ); + + const changedPayloadFixture = { + ...fixture.value, + bottleMirror: { + ...fixture.value.bottleMirror, + payloads: fixture.value.bottleMirror.payloads.map((payload, index) => + index === 0 ? { ...payload, sha256: "0".repeat(64) } : payload + ), + }, + }; + await assert.rejects( + () => + loadHomebrewGuestLifecycleBrowserFixture(changedPayloadFixture, { + sourceUrl: (url) => url, + fetchImpl: createFixtureFetch(fixture.bytesByUrl), + }), + /payload fixture differs from mirror asset/, + ); + + const inconsistentPlanBytes = encodeHomebrewBottleMirrorPlan({ + ...fixture.plan, + collection_sha256: "0".repeat(64), + }); + const inconsistentPlanFixture = { + ...fixture.value, + bottleMirror: { + ...fixture.value.bottleMirror, + plan: exact( + fixture.value.bottleMirror.plan.url, + inconsistentPlanBytes, + ), + }, + }; + const inconsistentBytesByUrl = new Map(fixture.bytesByUrl); + inconsistentBytesByUrl.set( + fixture.value.bottleMirror.plan.url, + inconsistentPlanBytes, + ); + await assert.rejects( + () => + loadHomebrewGuestLifecycleBrowserFixture(inconsistentPlanFixture, { + sourceUrl: (url) => url, + fetchImpl: createFixtureFetch(inconsistentBytesByUrl), + }), + /inconsistent derived identity/, + ); +}); + +function createFixture() { + const imageBytes = new Uint8Array([1, 2]); + const specBytes = new Uint8Array([3]); + const archiveBytes = new Uint8Array([4]); + const environmentBytes = new Uint8Array([5]); + const payloadBytes = new Uint8Array([6, 7, 8]); + const repository = "example/project"; + const identity = { + id: "bottle-test", + package: "example/tap/test", + asset: homebrewRuntimeLayerPayloadAsset("bottle-test"), + sha256: sha256(payloadBytes), + bytes: payloadBytes.byteLength, + }; + const collection = sha256( + encodeHomebrewBottleMirrorCollectionIdentity(repository, [identity]), + ); + const tag = `homebrew-shell-bottles-sha256-${collection}`; + const releaseRoot = + `https://github.com/${repository}/releases/download/${tag}`; + const plan: HomebrewBottleMirrorPlan = { + schema: 1, + kind: HOMEBREW_BOTTLE_MIRROR_PLAN_KIND, + repository, + collection_sha256: collection, + tag, + release_root: releaseRoot, + manifest_asset: HOMEBREW_BOTTLE_MIRROR_PLAN_ASSET, + assets: [{ + ...identity, + url: `${releaseRoot}/${identity.asset}`, + }], + }; + const planBytes = encodeHomebrewBottleMirrorPlan(plan); + const urls = { + image: "https://example.test/main-shell.vfs.zst", + spec: "https://example.test/main-shell-brew-package-tree.json", + archive: "https://example.test/homebrew-bootstrap.zip", + environment: "https://example.test/homebrew-brew.env", + plan: `${releaseRoot}/${HOMEBREW_BOTTLE_MIRROR_PLAN_ASSET}`, + }; + const value = { + schema: 1, + allowLiveNetwork: true, + transportMode: "closed", + image: exact(urls.image, imageBytes), + bootstrap: { + spec: exact(urls.spec, specBytes), + archive: exact(urls.archive, archiveBytes), + environment: exact(urls.environment, environmentBytes), + }, + bottleMirror: { + plan: exact(urls.plan, planBytes), + payloads: [{ + asset: identity.asset, + ...exact(plan.assets[0]!.url, payloadBytes), + }], + }, + revisions: { + coreRevision: "1".repeat(40), + canaryRevision: "2".repeat(40), + }, + timeoutMs: 900_000, + } as const; + const bytesByUrl = new Map([ + [urls.image, imageBytes], + [urls.spec, specBytes], + [urls.archive, archiveBytes], + [urls.environment, environmentBytes], + [urls.plan, planBytes], + [plan.assets[0]!.url, payloadBytes], + ]); + return { + value, + bytesByUrl, + imageBytes, + specBytes, + archiveBytes, + environmentBytes, + payloadBytes, + plan, + planBytes, + }; +} + +function createFixtureFetch(bytesByUrl: ReadonlyMap) { + return async (input: string | URL): Promise => { + const url = String(input); + const bytes = bytesByUrl.get(url); + if (bytes === undefined) return new Response(null, { status: 404 }); + const owned = bytes.slice(); + return new Response(owned.buffer, { + status: 200, + headers: { "content-length": String(owned.byteLength) }, + }); + }; +} + +function exact(url: string, bytes: Uint8Array) { + return { + url, + sha256: sha256(bytes), + bytes: bytes.byteLength, + }; +} + +function sha256(bytes: Uint8Array): string { + return createHash("sha256").update(bytes).digest("hex"); +} diff --git a/homebrew/test/homebrew_guest_lifecycle_browser_fixture.ts b/homebrew/test/homebrew_guest_lifecycle_browser_fixture.ts new file mode 100644 index 0000000000..79b4bc6d62 --- /dev/null +++ b/homebrew/test/homebrew_guest_lifecycle_browser_fixture.ts @@ -0,0 +1,418 @@ +import { + loadClosedLazyAssetSources, + MAX_CLOSED_LAZY_ASSET_BYTES, + type ClosedLazyAsset, + type ClosedLazyAssetSource, +} from "../../host/src/vfs/closed-lazy-assets"; +import { + assertHomebrewBottleMirrorPlanIdentity, + decodeHomebrewBottleMirrorPlan, + isRecord, +} from "../../scripts/homebrew-closed-lazy-assets-contract"; +import { + assertHomebrewGuestLifecycleRevisions, + type HomebrewGuestLifecycleRevisions, +} from "./homebrew_guest_lifecycle_contract"; +import type { + HomebrewGuestLifecycleTransportMode, +} from "./homebrew_guest_lifecycle_runtime_inputs"; + +export interface HomebrewGuestLifecycleExactAsset { + url: string; + sha256: string; + bytes: number; +} + +export interface HomebrewGuestLifecycleBottlePayloadFixture + extends HomebrewGuestLifecycleExactAsset { + asset: string; +} + +export interface HomebrewGuestLifecycleBrowserFixture { + schema: 1; + allowLiveNetwork: true; + transportMode: HomebrewGuestLifecycleTransportMode; + image: HomebrewGuestLifecycleExactAsset; + bootstrap: { + spec: HomebrewGuestLifecycleExactAsset; + archive: HomebrewGuestLifecycleExactAsset; + environment: HomebrewGuestLifecycleExactAsset; + }; + bottleMirror: { + plan: HomebrewGuestLifecycleExactAsset; + payloads?: HomebrewGuestLifecycleBottlePayloadFixture[]; + }; + revisions: HomebrewGuestLifecycleRevisions; + timeoutMs: number; +} + +export interface LoadedHomebrewGuestLifecycleBrowserFixture { + fixture: HomebrewGuestLifecycleBrowserFixture; + imageBytes: Uint8Array; + bootstrapSpecBytes: Uint8Array; + bootstrapArchiveBytes: Uint8Array; + bootstrapEnvironmentBytes: Uint8Array; + bottleMirrorPlanBytes: Uint8Array; + closedBottleAssets?: readonly ClosedLazyAsset[]; +} + +type FetchLike = ( + input: string | URL, + init?: RequestInit, +) => Promise; + +const TOP_LEVEL_KEYS = [ + "schema", + "allowLiveNetwork", + "transportMode", + "image", + "bootstrap", + "bottleMirror", + "revisions", + "timeoutMs", +] as const; +const BOOTSTRAP_KEYS = ["spec", "archive", "environment"] as const; +const MIRROR_KEYS = ["plan", "payloads"] as const; +const REVISION_KEYS = ["coreRevision", "canaryRevision"] as const; +const ASSET_KEYS = ["url", "sha256", "bytes"] as const; +const PAYLOAD_KEYS = ["asset", "url", "sha256", "bytes"] as const; +const SHA256_RE = /^[0-9a-f]{64}$/; + +/** + * Reject ambient or partially specified live inputs before any browser fetch. + * A lifecycle proof may use network transport, but every accepted byte source + * remains bound to an immutable URL, exact length, and SHA-256. + */ +export function projectHomebrewGuestLifecycleBrowserFixture( + value: unknown, +): HomebrewGuestLifecycleBrowserFixture { + if ( + !isRecord(value) || + !hasExactKeys(value, TOP_LEVEL_KEYS) + ) { + throw new Error( + "Homebrew browser lifecycle fixture has unknown or missing fields", + ); + } + if (value.schema !== 1 || value.allowLiveNetwork !== true) { + throw new Error( + "Homebrew browser lifecycle requires explicit live-network opt-in", + ); + } + if (value.transportMode !== "closed" && value.transportMode !== "public") { + throw new Error( + "Homebrew browser lifecycle transport mode must be closed or public", + ); + } + if ( + !isRecord(value.bootstrap) || + !hasExactKeys(value.bootstrap, BOOTSTRAP_KEYS) + ) { + throw new Error( + "Homebrew browser lifecycle bootstrap has unknown or missing fields", + ); + } + if ( + !isRecord(value.revisions) || + !hasExactKeys(value.revisions, REVISION_KEYS) || + typeof value.revisions.coreRevision !== "string" || + typeof value.revisions.canaryRevision !== "string" + ) { + throw new Error("Homebrew browser lifecycle revisions are invalid"); + } + const revisions = { + coreRevision: value.revisions.coreRevision, + canaryRevision: value.revisions.canaryRevision, + }; + assertHomebrewGuestLifecycleRevisions(revisions); + if ( + !Number.isSafeInteger(value.timeoutMs) || + (value.timeoutMs as number) < 1_000 || + (value.timeoutMs as number) > 30 * 60 * 1_000 + ) { + throw new Error( + "Homebrew browser lifecycle timeout must be 1000..1800000 milliseconds", + ); + } + + const bottleMirror = projectBottleMirror(value.bottleMirror); + if ( + ( + value.transportMode === "closed" && + bottleMirror.payloads === undefined + ) || + ( + value.transportMode === "public" && + bottleMirror.payloads !== undefined + ) + ) { + throw new Error( + "closed browser lifecycle transport requires exact bottle payloads, " + + "while public transport forbids local payload bytes", + ); + } + + return { + schema: 1, + allowLiveNetwork: true, + transportMode: value.transportMode, + image: projectExactAsset(value.image, "image"), + bootstrap: { + spec: projectExactAsset(value.bootstrap.spec, "bootstrap spec"), + archive: projectExactAsset( + value.bootstrap.archive, + "bootstrap archive", + ), + environment: projectExactAsset( + value.bootstrap.environment, + "bootstrap environment", + ), + }, + bottleMirror, + revisions, + timeoutMs: value.timeoutMs as number, + }; +} + +export async function loadHomebrewGuestLifecycleBrowserFixture( + value: unknown, + options: { + fetchImpl?: FetchLike; + sourceUrl: (canonicalUrl: string) => string; + }, +): Promise { + const fixture = projectHomebrewGuestLifecycleBrowserFixture(value); + const fetchImpl = options.fetchImpl ?? fetch; + const [ + imageBytes, + bootstrapSpecBytes, + bootstrapArchiveBytes, + bootstrapEnvironmentBytes, + ] = await Promise.all([ + loadExactAsset(fixture.image, "image", options.sourceUrl, fetchImpl), + loadExactAsset( + fixture.bootstrap.spec, + "bootstrap spec", + options.sourceUrl, + fetchImpl, + ), + loadExactAsset( + fixture.bootstrap.archive, + "bootstrap archive", + options.sourceUrl, + fetchImpl, + ), + loadExactAsset( + fixture.bootstrap.environment, + "bootstrap environment", + options.sourceUrl, + fetchImpl, + ), + ]); + + const bottleMirrorPlanBytes = await loadExactAsset( + fixture.bottleMirror.plan, + "bottle mirror plan", + options.sourceUrl, + fetchImpl, + ); + const plan = decodeHomebrewBottleMirrorPlan( + bottleMirrorPlanBytes, + "live Homebrew bottle mirror plan", + ); + await assertHomebrewBottleMirrorPlanIdentity(plan); + const expectedPlanUrl = `${plan.release_root}/${plan.manifest_asset}`; + if (fixture.bottleMirror.plan.url !== expectedPlanUrl) { + throw new Error( + "live bottle mirror plan URL differs from its canonical release URL", + ); + } + + if (fixture.bottleMirror.payloads === undefined) { + return { + fixture, + imageBytes, + bootstrapSpecBytes, + bootstrapArchiveBytes, + bootstrapEnvironmentBytes, + bottleMirrorPlanBytes, + }; + } + + const payloadFixtureByAsset = new Map( + fixture.bottleMirror.payloads.map((payload) => [payload.asset, payload]), + ); + if ( + payloadFixtureByAsset.size !== fixture.bottleMirror.payloads.length || + payloadFixtureByAsset.size !== plan.assets.length + ) { + throw new Error( + "live bottle payload fixtures must cover each mirror asset exactly once", + ); + } + const payloadSources: ClosedLazyAssetSource[] = plan.assets.map((asset) => { + const payload = payloadFixtureByAsset.get(asset.asset); + if ( + payload === undefined || + payload.url !== asset.url || + payload.sha256 !== asset.sha256 || + payload.bytes !== asset.bytes + ) { + throw new Error( + `live bottle payload fixture differs from mirror asset ${asset.asset}`, + ); + } + return { + url: asset.url, + sourceUrl: options.sourceUrl(asset.url), + sha256: asset.sha256, + size: asset.bytes, + }; + }); + const closedBottleAssets = await loadClosedLazyAssetSources(payloadSources, { + fetchImpl, + maxConcurrency: 4, + }); + + return { + fixture, + imageBytes, + bootstrapSpecBytes, + bootstrapArchiveBytes, + bootstrapEnvironmentBytes, + bottleMirrorPlanBytes, + closedBottleAssets, + }; +} + +function projectBottleMirror(value: unknown): + HomebrewGuestLifecycleBrowserFixture["bottleMirror"] { + if ( + !isRecord(value) || + ( + !hasExactKeys(value, ["plan"]) && + !hasExactKeys(value, MIRROR_KEYS) + ) + ) { + throw new Error( + "Homebrew browser lifecycle bottle mirror has unknown or missing fields", + ); + } + if ( + value.payloads !== undefined && + (!Array.isArray(value.payloads) || value.payloads.length === 0) + ) { + throw new Error( + "Homebrew browser lifecycle bottle payloads must be a non-empty array", + ); + } + return { + plan: projectExactAsset(value.plan, "bottle mirror plan"), + ...(value.payloads === undefined + ? {} + : { + payloads: value.payloads.map((payload, index) => + projectBottlePayload(payload, index) + ), + }), + }; +} + +function projectBottlePayload( + value: unknown, + index: number, +): HomebrewGuestLifecycleBottlePayloadFixture { + if ( + !isRecord(value) || + !hasExactKeys(value, PAYLOAD_KEYS) || + typeof value.asset !== "string" || + value.asset.length === 0 || + value.asset.includes("/") || + value.asset === "." || + value.asset === ".." + ) { + throw new Error(`Homebrew bottle payload fixture ${index} is invalid`); + } + return { + asset: value.asset, + ...projectExactAssetFields(value, `bottle payload ${index}`), + }; +} + +function projectExactAsset( + value: unknown, + label: string, +): HomebrewGuestLifecycleExactAsset { + if (!isRecord(value) || !hasExactKeys(value, ASSET_KEYS)) { + throw new Error(`${label} has unknown or missing fields`); + } + return projectExactAssetFields(value, label); +} + +function projectExactAssetFields( + value: Record, + label: string, +): HomebrewGuestLifecycleExactAsset { + if ( + typeof value.url !== "string" || + typeof value.sha256 !== "string" || + !SHA256_RE.test(value.sha256) || + !Number.isSafeInteger(value.bytes) || + (value.bytes as number) <= 0 || + (value.bytes as number) > MAX_CLOSED_LAZY_ASSET_BYTES + ) { + throw new Error(`${label} has invalid exact identity fields`); + } + let parsed: URL; + try { + parsed = new URL(value.url); + } catch (error) { + throw new Error(`${label} URL is invalid`, { cause: error }); + } + if ( + parsed.protocol !== "https:" || + parsed.username !== "" || + parsed.password !== "" || + parsed.hash !== "" || + value.url.includes("#") || + parsed.href !== value.url + ) { + throw new Error(`${label} must use one canonical credential-free HTTPS URL`); + } + return { + url: value.url, + sha256: value.sha256, + bytes: value.bytes as number, + }; +} + +async function loadExactAsset( + asset: HomebrewGuestLifecycleExactAsset, + label: string, + sourceUrl: (canonicalUrl: string) => string, + fetchImpl: FetchLike, +): Promise { + try { + const [loaded] = await loadClosedLazyAssetSources([{ + url: asset.url, + sourceUrl: sourceUrl(asset.url), + sha256: asset.sha256, + size: asset.bytes, + }], { fetchImpl }); + return loaded!.bytes; + } catch (error) { + throw new Error( + `failed to load exact Homebrew browser lifecycle ${label}: ${String(error)}`, + { cause: error }, + ); + } +} + +function hasExactKeys( + value: Record, + expected: readonly string[], +): boolean { + const actual = Object.keys(value); + return actual.length === expected.length && + expected.every((key) => Object.hasOwn(value, key)); +} diff --git a/homebrew/test/homebrew_guest_lifecycle_contract.test.ts b/homebrew/test/homebrew_guest_lifecycle_contract.test.ts new file mode 100644 index 0000000000..e005b1440b --- /dev/null +++ b/homebrew/test/homebrew_guest_lifecycle_contract.test.ts @@ -0,0 +1,131 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import test from "node:test"; + +import { + assertHomebrewGuestLifecycleRevisions, + createHomebrewGuestLifecyclePhaseOneScript, + createHomebrewGuestLifecyclePhaseTwoScript, + HOMEBREW_GUEST_LIFECYCLE_PHASE_ONE_MARKER, + HOMEBREW_GUEST_LIFECYCLE_PHASE_TWO_MARKER, +} from "./homebrew_guest_lifecycle_contract"; + +const revisions = { + coreRevision: "1".repeat(40), + canaryRevision: "2".repeat(40), +}; + +test("requires immutable lower-case tap revisions", () => { + assert.doesNotThrow(() => assertHomebrewGuestLifecycleRevisions(revisions)); + for (const candidate of [ + "", + "1".repeat(39), + "1".repeat(41), + "A".repeat(40), + `${"1".repeat(40)}; touch /tmp/injected`, + ]) { + assert.throws( + () => assertHomebrewGuestLifecycleRevisions({ + ...revisions, + coreRevision: candidate, + }), + /exact lowercase 40-character SHA/, + ); + } +}); + +test("phase one uses only stock Homebrew against clean canonical tap checkouts", () => { + const script = createHomebrewGuestLifecyclePhaseOneScript(revisions); + assertShellSyntax(script); + for (const expected of [ + "brew tap kandelo-dev/tap-core https://github.com/Kandelo-dev/homebrew-tap-core.git", + `checkout --detach ${revisions.coreRevision}`, + "brew install --no-ask --force-bottle kandelo-dev/tap-core/bzip2", + "brew reinstall --force-bottle kandelo-dev/tap-core/bzip2", + "brew tap brandonpayton/kandelo-canary https://github.com/brandonpayton/homebrew-kandelo-canary.git", + `checkout --detach ${revisions.canaryRevision}`, + "brew install --no-ask --force-bottle brandonpayton/kandelo-canary/m4", + 'dependency["full_name"] == ARGV.fetch(1)', + HOMEBREW_GUEST_LIFECYCLE_PHASE_ONE_MARKER, + ]) { + assert.ok(script.includes(expected), `missing lifecycle contract: ${expected}`); + } + assert.equal( + script.match(/brew uninstall --ignore-dependencies/g)?.length, + 2, + "only the Bzip2 and M4 direct-composer transitions may ignore dependents", + ); + for (const forbidden of [ + "File.binwrite", + "Formula/", + "Kandelo/formula_support", + "sed -i", + "homebrew/core/", + ]) { + assert.ok( + !script.includes(forbidden), + `phase one must not mutate or substitute package inputs: ${forbidden}`, + ); + } +}); + +test("phase two proves durable state and labels the pinned upgrade as a no-op", () => { + const script = createHomebrewGuestLifecyclePhaseTwoScript(revisions); + assertShellSyntax(script); + for (const expected of [ + "brew outdated --json=v2", + "snapshot_package_identity kandelo-dev/tap-core/bzip2 \"$before_bzip2\"", + "snapshot_package_identity brandonpayton/kandelo-canary/m4 \"$before_m4\"", + "brew upgrade --force-bottle kandelo-dev/tap-core/bzip2 brandonpayton/kandelo-canary/m4", + "snapshot_package_identity kandelo-dev/tap-core/bzip2 \"$after_bzip2\"", + "snapshot_package_identity brandonpayton/kandelo-canary/m4 \"$after_m4\"", + "receipt_sha256", + "content_sha256", + "/usr/bin/cmp \"$before_bzip2\" \"$after_bzip2\"", + "/usr/bin/cmp \"$before_m4\" \"$after_m4\"", + "brew uninstall brandonpayton/kandelo-canary/m4", + "brew uninstall kandelo-dev/tap-core/bzip2", + "brew untap brandonpayton/kandelo-canary", + "brew untap --force kandelo-dev/tap-core", + HOMEBREW_GUEST_LIFECYCLE_PHASE_TWO_MARKER, + ]) { + assert.ok(script.includes(expected), `missing reboot contract: ${expected}`); + } + assert.ok(!script.includes("brew update")); + assert.ok( + script.includes("base shell has receipts for the rest of the direct-composed core"), + "the forced temporary untap needs its maintenance rationale inline", + ); + assert.ok( + script.includes("successful brew upgrade does not prove it was a no-op"), + "the exact package snapshots need their maintenance rationale inline", + ); + const before = script.indexOf( + "snapshot_package_identity kandelo-dev/tap-core/bzip2 \"$before_bzip2\"", + ); + const upgrade = script.indexOf( + "brew upgrade --force-bottle kandelo-dev/tap-core/bzip2", + ); + const after = script.indexOf( + "snapshot_package_identity kandelo-dev/tap-core/bzip2 \"$after_bzip2\"", + ); + const comparison = script.indexOf( + "/usr/bin/cmp \"$before_bzip2\" \"$after_bzip2\"", + ); + assert.ok( + before < upgrade && upgrade < after && after < comparison, + "the exact installed identity must bracket and verify brew upgrade", + ); +}); + +function assertShellSyntax(script: string): void { + const result = spawnSync("/bin/bash", ["-n"], { + input: script, + encoding: "utf8", + }); + assert.equal( + result.status, + 0, + `generated lifecycle shell is invalid: ${result.stderr}`, + ); +} diff --git a/homebrew/test/homebrew_guest_lifecycle_contract.ts b/homebrew/test/homebrew_guest_lifecycle_contract.ts new file mode 100644 index 0000000000..7f3009e690 --- /dev/null +++ b/homebrew/test/homebrew_guest_lifecycle_contract.ts @@ -0,0 +1,370 @@ +export const HOMEBREW_GUEST_LIFECYCLE_PHASE_ONE_MARKER = + "KANDELO_HOMEBREW_GUEST_LIFECYCLE_PHASE_ONE_OK"; +export const HOMEBREW_GUEST_LIFECYCLE_PHASE_TWO_MARKER = + "KANDELO_HOMEBREW_GUEST_LIFECYCLE_PHASE_TWO_OK"; + +export const HOMEBREW_GUEST_LIFECYCLE_CORE_TAP = "kandelo-dev/tap-core"; +export const HOMEBREW_GUEST_LIFECYCLE_CORE_REPOSITORY = + "kandelo-dev/homebrew-tap-core"; +const CORE_TAP = HOMEBREW_GUEST_LIFECYCLE_CORE_TAP; +const CORE_ORIGIN = + "https://github.com/Kandelo-dev/homebrew-tap-core.git"; +const CANARY_TAP = "brandonpayton/kandelo-canary"; +const CANARY_ORIGIN = + "https://github.com/brandonpayton/homebrew-kandelo-canary.git"; +const EXACT_GIT_REVISION = /^[0-9a-f]{40}$/; + +export interface HomebrewGuestLifecycleRevisions { + coreRevision: string; + canaryRevision: string; +} + +export function assertHomebrewGuestLifecycleRevisions( + revisions: HomebrewGuestLifecycleRevisions, +): void { + for (const [label, revision] of [ + ["core", revisions.coreRevision], + ["canary", revisions.canaryRevision], + ] as const) { + if (!EXACT_GIT_REVISION.test(revision)) { + throw new Error( + `Homebrew guest lifecycle ${label} revision must be an exact lowercase 40-character SHA`, + ); + } + } +} + +/** + * Exercise stock Homebrew against exact first- and third-party tap revisions. + * + * The current lazy shell already contains direct-composed receipts for its + * complete closure. The two `uninstall --ignore-dependencies` operations + * below deliberately create empty Bzip2 and M4 targets before asking stock + * Homebrew to install them. This is a transition proof between composition + * models, not a package workaround: no Formula, bottle, or Homebrew source is + * modified. + */ +export function createHomebrewGuestLifecyclePhaseOneScript( + revisions: HomebrewGuestLifecycleRevisions, +): string { + assertHomebrewGuestLifecycleRevisions(revisions); + return String.raw` +set -euo pipefail +fail() { printf 'homebrew-guest-lifecycle: %s\n' "$*" >&2; exit 1; } +progress() { printf 'homebrew-guest-lifecycle: %s\n' "$*"; } +assert_poured() { + /usr/bin/ruby -rjson -e ' + receipt = JSON.parse(File.binread(File.join(ARGV.fetch(0), "INSTALL_RECEIPT.json"))) + abort "bottle was not poured" unless receipt.fetch("poured_from_bottle") == true + ' "$1" +} +assert_clean_tap() { + tap_root="$1" + expected_origin="$2" + expected_revision="$3" + [ "$(/usr/bin/git -C "$tap_root" remote get-url origin)" = "$expected_origin" ] || + fail "tap origin differs from the canonical public repository" + [ "$(/usr/bin/git -C "$tap_root" rev-parse HEAD)" = "$expected_revision" ] || + fail "tap checkout differs from the reviewed revision" + [ -z "$(/usr/bin/git -C "$tap_root" status --porcelain=v1 --untracked-files=all)" ] || + fail "tap checkout is dirty" +} +assert_bzip2_roundtrip() { + prefix="$1" + input=/tmp/kandelo-homebrew-bzip2.input + archive=/tmp/kandelo-homebrew-bzip2.bz2 + output=/tmp/kandelo-homebrew-bzip2.output + /usr/bin/printf 'Kandelo stock Homebrew lifecycle\n' >"$input" + "$prefix/bin/bzip2" -c "$input" >"$archive" + "$prefix/bin/bzip2" -dc "$archive" >"$output" + /usr/bin/cmp "$input" "$output" + /usr/bin/rm -f "$input" "$archive" "$output" +} +assert_m4_execution() { + prefix="$1" + expected="$2" + actual="$(/usr/bin/printf '%s\n' \ + 'changequote([,])dnl' \ + "define([KANDELO_LIFECYCLE_VALUE],[$expected])dnl" \ + 'KANDELO_LIFECYCLE_VALUE' | + "$prefix/bin/m4")" + [ "$actual" = "$expected" ] || fail "third-party M4 did not execute" +} + +export HOMEBREW_NO_ANALYTICS=1 +export HOMEBREW_NO_AUTO_UPDATE=1 +export HOMEBREW_NO_ENV_HINTS=1 +export HOMEBREW_NO_INSTALL_FROM_API=1 +export GIT_TERMINAL_PROMPT=0 + +repository="$(/usr/bin/brew --repository)" +core_repository="$repository/Library/Taps/homebrew/homebrew-core" +[ ! -e "$core_repository" ] || fail "homebrew/core existed before the lifecycle proof" + +progress "tapping the exact first-party repository" +/usr/bin/brew tap ${CORE_TAP} ${CORE_ORIGIN} +core_tap="$(/usr/bin/brew --repository ${CORE_TAP})" +/usr/bin/git -C "$core_tap" fetch --no-tags origin ${revisions.coreRevision} +/usr/bin/git -C "$core_tap" checkout --detach ${revisions.coreRevision} +assert_clean_tap "$core_tap" ${CORE_ORIGIN} ${revisions.coreRevision} +[ ! -e "$core_repository" ] || fail "first-party tap created homebrew/core" + +# WHY: the base shell is already composed from this bottle closure. Remove the +# existing receipt through stock Homebrew so the following command proves a +# genuine install rather than accepting Homebrew's "already installed" path. +composed_bzip2_prefix="$(/usr/bin/brew --prefix ${CORE_TAP}/bzip2)" +assert_poured "$composed_bzip2_prefix" +/usr/bin/brew uninstall --ignore-dependencies ${CORE_TAP}/bzip2 +[ ! -e "$composed_bzip2_prefix" ] || + fail "direct-composed Bzip2 prefix remains after transition uninstall" + +progress "installing and executing the first-party Bzip2 bottle" +/usr/bin/brew install --no-ask --force-bottle ${CORE_TAP}/bzip2 +bzip2_prefix="$(/usr/bin/brew --prefix ${CORE_TAP}/bzip2)" +assert_poured "$bzip2_prefix" +assert_bzip2_roundtrip "$bzip2_prefix" + +progress "reinstalling and executing the first-party Bzip2 bottle" +/usr/bin/brew reinstall --force-bottle ${CORE_TAP}/bzip2 +reinstalled_bzip2_prefix="$(/usr/bin/brew --prefix ${CORE_TAP}/bzip2)" +[ "$reinstalled_bzip2_prefix" = "$bzip2_prefix" ] || + fail "Bzip2 reinstall changed its versioned prefix" +assert_poured "$reinstalled_bzip2_prefix" +assert_bzip2_roundtrip "$reinstalled_bzip2_prefix" + +progress "tapping the exact independent third-party repository" +/usr/bin/brew tap ${CANARY_TAP} ${CANARY_ORIGIN} +canary_tap="$(/usr/bin/brew --repository ${CANARY_TAP})" +/usr/bin/git -C "$canary_tap" fetch --no-tags origin ${revisions.canaryRevision} +/usr/bin/git -C "$canary_tap" checkout --detach ${revisions.canaryRevision} +assert_clean_tap "$canary_tap" ${CANARY_ORIGIN} ${revisions.canaryRevision} +[ ! -e "$core_repository" ] || fail "third-party tap created homebrew/core" + +# WHY: core M4 and canary M4 have the same conventional Cellar identity. Use +# stock uninstall to create one truthful target before the independent tap +# pours its own bottle; do not rewrite either Formula to avoid the collision. +composed_m4_prefix="$(/usr/bin/brew --prefix ${CORE_TAP}/m4)" +assert_poured "$composed_m4_prefix" +/usr/bin/brew uninstall --ignore-dependencies ${CORE_TAP}/m4 +[ ! -e "$composed_m4_prefix" ] || + fail "direct-composed M4 prefix remains after transition uninstall" + +dash_prefix="$(/usr/bin/brew --prefix ${CORE_TAP}/dash)" +assert_poured "$dash_prefix" +progress "installing independent M4 with its first-party Dash dependency" +/usr/bin/brew install --no-ask --force-bottle ${CANARY_TAP}/m4 +m4_prefix="$(/usr/bin/brew --prefix ${CANARY_TAP}/m4)" +assert_poured "$m4_prefix" +assert_poured "$dash_prefix" +/usr/bin/ruby -rjson -e ' + receipt = JSON.parse(File.binread(File.join(ARGV.fetch(0), "INSTALL_RECEIPT.json"))) + dependencies = receipt.fetch("runtime_dependencies") + abort "M4 receipt does not bind first-party Dash" unless + dependencies.any? { |dependency| dependency["full_name"] == ARGV.fetch(1) } +' "$m4_prefix" ${CORE_TAP}/dash +"$m4_prefix/bin/m4" --version >/dev/null +assert_m4_execution "$m4_prefix" cross-tap-ok + +state="$repository/var/homebrew/kandelo-guest-lifecycle-state" +{ + /usr/bin/printf '%s\n' ${revisions.coreRevision} + /usr/bin/printf '%s\n' ${revisions.canaryRevision} +} >"$state" + +assert_clean_tap "$core_tap" ${CORE_ORIGIN} ${revisions.coreRevision} +assert_clean_tap "$canary_tap" ${CANARY_ORIGIN} ${revisions.canaryRevision} +[ ! -e "$core_repository" ] || fail "lifecycle install created homebrew/core" +progress "phase one is durable and ready for rootfs export" +/usr/bin/printf '%s\n' ${HOMEBREW_GUEST_LIFECYCLE_PHASE_ONE_MARKER} +`.trim(); +} + +/** + * Reboot the phase-one filesystem, execute its installed bottles, exercise the + * no-op upgrade path at the same pinned versions, and remove only the packages + * installed by the lifecycle proof. + * + * A real old-to-new bottle transition needs two immutable published versions + * and is intentionally a later live fixture. This phase does not call + * `brew update`: the guest bootstrap is a patched immutable source archive, + * so replacing that source through an ambient update would lose its reviewed + * Kandelo boundary. + */ +export function createHomebrewGuestLifecyclePhaseTwoScript( + revisions: HomebrewGuestLifecycleRevisions, +): string { + assertHomebrewGuestLifecycleRevisions(revisions); + return String.raw` +set -euo pipefail +fail() { printf 'homebrew-guest-lifecycle-reboot: %s\n' "$*" >&2; exit 1; } +progress() { printf 'homebrew-guest-lifecycle-reboot: %s\n' "$*"; } +assert_poured() { + /usr/bin/ruby -rjson -e ' + receipt = JSON.parse(File.binread(File.join(ARGV.fetch(0), "INSTALL_RECEIPT.json"))) + abort "bottle was not poured" unless receipt.fetch("poured_from_bottle") == true + ' "$1" +} +assert_clean_tap() { + tap_root="$1" + expected_origin="$2" + expected_revision="$3" + [ "$(/usr/bin/git -C "$tap_root" remote get-url origin)" = "$expected_origin" ] || + fail "tap origin changed across reboot" + [ "$(/usr/bin/git -C "$tap_root" rev-parse HEAD)" = "$expected_revision" ] || + fail "tap revision changed across reboot" + [ -z "$(/usr/bin/git -C "$tap_root" status --porcelain=v1 --untracked-files=all)" ] || + fail "tap checkout became dirty" +} +assert_bzip2_roundtrip() { + prefix="$1" + input=/tmp/kandelo-homebrew-bzip2-reboot.input + archive=/tmp/kandelo-homebrew-bzip2-reboot.bz2 + output=/tmp/kandelo-homebrew-bzip2-reboot.output + /usr/bin/printf 'Kandelo durable Homebrew state\n' >"$input" + "$prefix/bin/bzip2" -c "$input" >"$archive" + "$prefix/bin/bzip2" -dc "$archive" >"$output" + /usr/bin/cmp "$input" "$output" + /usr/bin/rm -f "$input" "$archive" "$output" +} +snapshot_package_identity() { + formula="$1" + destination="$2" + prefix="$(/usr/bin/brew --prefix "$formula")" + versions="$(/usr/bin/brew list --versions --full-name "$formula")" + [ -n "$versions" ] || fail "brew list omitted installed identity for $formula" + # WHY: a successful brew upgrade does not prove it was a no-op. Bind the + # exact Cellar path, reported version, receipt bytes, and complete keg tree + # so replacement, relinking, or receipt mutation cannot masquerade as one. + /usr/bin/ruby -rdigest -rjson -e ' + root = ARGV.fetch(0) + formula = ARGV.fetch(1) + versions = ARGV.fetch(2) + receipt_path = File.join(root, "INSTALL_RECEIPT.json") + receipt = File.binread(receipt_path) + entries = Dir.glob( + File.join(root, "**", "*"), + File::FNM_DOTMATCH, + ).reject { |path| [".", ".."].include?(File.basename(path)) }.sort.map do |path| + relative = path.delete_prefix("#{root}/") + stat = File.lstat(path) + payload = case stat.ftype + when "file" + Digest::SHA256.file(path).hexdigest + when "link" + File.readlink(path) + when "directory" + nil + else + abort "unsupported keg entry type #{stat.ftype}: #{relative}" + end + [relative, stat.ftype, stat.mode & 0o7777, stat.nlink, stat.size, payload] + end + identity = { + "full_name" => formula, + "prefix" => root, + "versions" => versions, + "receipt_sha256" => Digest::SHA256.hexdigest(receipt), + "content_sha256" => Digest::SHA256.hexdigest(JSON.generate(entries)), + } + STDOUT.write(JSON.generate(identity)) + STDOUT.write("\n") + ' "$prefix" "$formula" "$versions" >"$destination" +} + +export HOMEBREW_NO_ANALYTICS=1 +export HOMEBREW_NO_AUTO_UPDATE=1 +export HOMEBREW_NO_ENV_HINTS=1 +export HOMEBREW_NO_INSTALL_FROM_API=1 +export GIT_TERMINAL_PROMPT=0 + +repository="$(/usr/bin/brew --repository)" +state="$repository/var/homebrew/kandelo-guest-lifecycle-state" +[ -f "$state" ] || fail "durable lifecycle state is missing after reboot" +{ + IFS= read -r saved_core_revision + IFS= read -r saved_canary_revision +} <"$state" +[ "$saved_core_revision" = "${revisions.coreRevision}" ] || + fail "first-party tap revision state changed across reboot" +[ "$saved_canary_revision" = "${revisions.canaryRevision}" ] || + fail "third-party tap revision state changed across reboot" + +core_tap="$(/usr/bin/brew --repository ${CORE_TAP})" +canary_tap="$(/usr/bin/brew --repository ${CANARY_TAP})" +assert_clean_tap "$core_tap" ${CORE_ORIGIN} ${revisions.coreRevision} +assert_clean_tap "$canary_tap" ${CANARY_ORIGIN} ${revisions.canaryRevision} + +bzip2_prefix="$(/usr/bin/brew --prefix ${CORE_TAP}/bzip2)" +m4_prefix="$(/usr/bin/brew --prefix ${CANARY_TAP}/m4)" +dash_prefix="$(/usr/bin/brew --prefix ${CORE_TAP}/dash)" +assert_poured "$bzip2_prefix" +assert_poured "$m4_prefix" +assert_poured "$dash_prefix" + +progress "executing persisted bottles after rootfs reboot" +assert_bzip2_roundtrip "$bzip2_prefix" +"$m4_prefix/bin/m4" --version >/dev/null +m4_output="$(/usr/bin/printf '%s\n' \ + 'changequote([,])dnl' \ + 'define([KANDELO_LIFECYCLE_VALUE],[reboot-ok])dnl' \ + 'KANDELO_LIFECYCLE_VALUE' | + "$m4_prefix/bin/m4")" +[ "$m4_output" = reboot-ok ] || fail "M4 did not execute after reboot" + +progress "checking pinned upgrade state through stock Homebrew" +outdated=/tmp/kandelo-homebrew-outdated.json +before_bzip2=/tmp/kandelo-homebrew-bzip2.before.json +after_bzip2=/tmp/kandelo-homebrew-bzip2.after.json +before_m4=/tmp/kandelo-homebrew-m4.before.json +after_m4=/tmp/kandelo-homebrew-m4.after.json +/usr/bin/brew outdated --json=v2 >"$outdated" +/usr/bin/ruby -rjson -e ' + document = JSON.parse(File.binread(ARGV.fetch(0))) + abort "brew outdated omitted formulae" unless document["formulae"].is_a?(Array) + selected = document["formulae"].filter_map { |entry| entry["name"] } + forbidden = ARGV.drop(1) + abort "newly installed pinned Formula is unexpectedly outdated" unless + (selected & forbidden).empty? +' "$outdated" bzip2 m4 +snapshot_package_identity ${CORE_TAP}/bzip2 "$before_bzip2" +snapshot_package_identity ${CANARY_TAP}/m4 "$before_m4" +/usr/bin/brew upgrade --force-bottle ${CORE_TAP}/bzip2 ${CANARY_TAP}/m4 +snapshot_package_identity ${CORE_TAP}/bzip2 "$after_bzip2" +snapshot_package_identity ${CANARY_TAP}/m4 "$after_m4" +/usr/bin/cmp "$before_bzip2" "$after_bzip2" || + fail "pinned Bzip2 upgrade changed its exact installed identity" +/usr/bin/cmp "$before_m4" "$after_m4" || + fail "pinned M4 upgrade changed its exact installed identity" +assert_poured "$bzip2_prefix" +assert_poured "$m4_prefix" +assert_bzip2_roundtrip "$bzip2_prefix" +"$m4_prefix/bin/m4" --version >/dev/null +/usr/bin/rm -f \ + "$outdated" \ + "$before_bzip2" "$after_bzip2" \ + "$before_m4" "$after_m4" + +progress "uninstalling lifecycle bottles and untapping both repositories" +/usr/bin/brew uninstall ${CANARY_TAP}/m4 +[ ! -e "$m4_prefix" ] || fail "M4 prefix remains after uninstall" +[ -x "$dash_prefix/bin/dash" ] || + fail "uninstalling M4 removed its pre-existing first-party dependency" +/usr/bin/brew uninstall ${CORE_TAP}/bzip2 +[ ! -e "$bzip2_prefix" ] || fail "Bzip2 prefix remains after uninstall" + +/usr/bin/brew untap ${CANARY_TAP} +# WHY: the base shell has receipts for the rest of the direct-composed core +# closure. Force removes only this temporary tap checkout; it does not remove +# those packages or alter their receipts. +/usr/bin/brew untap --force ${CORE_TAP} +[ ! -e "$repository/Library/Taps/brandonpayton/homebrew-kandelo-canary" ] || + fail "third-party tap remains after untap" +[ ! -e "$repository/Library/Taps/kandelo-dev/homebrew-tap-core" ] || + fail "first-party tap remains after untap" +[ ! -e "$repository/Library/Taps/homebrew/homebrew-core" ] || + fail "lifecycle created homebrew/core" + +/usr/bin/rm -f "$state" +/usr/bin/printf '%s\n' ${HOMEBREW_GUEST_LIFECYCLE_PHASE_TWO_MARKER} +`.trim(); +} diff --git a/homebrew/test/homebrew_guest_lifecycle_node.ts b/homebrew/test/homebrew_guest_lifecycle_node.ts new file mode 100644 index 0000000000..8e71179c86 --- /dev/null +++ b/homebrew/test/homebrew_guest_lifecycle_node.ts @@ -0,0 +1,436 @@ +#!/usr/bin/env -S npx tsx + +import { createHash } from "node:crypto"; +import { lstatSync, readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; + +import { NodeKernelHost } from "../../host/src/node-kernel-host"; +import { + MemoryFileSystem, + type LazyDownloadEvent, +} from "../../host/src/vfs/memory-fs"; +import { assertHomebrewBottleMirrorPlan } from "../../host/src/homebrew-vfs-composer"; +import { + assertPackageDeferredZipTreeState, + derivePackageDeferredZipTree, +} from "../../host/src/vfs/package-deferred-tree"; +import { + loadHomebrewBottleMirrorBindings, +} from "../../scripts/homebrew-closed-lazy-assets"; +import { + assertHomebrewGuestLifecycleRevisions, + type HomebrewGuestLifecycleRevisions, +} from "./homebrew_guest_lifecycle_contract"; +import { + assertNoUnexpectedHostDiagnostics, +} from "./homebrew_guest_lifecycle_runtime_contract"; +import { + HOMEBREW_GUEST_LIFECYCLE_ENV, + type HomebrewGuestLifecycleMachine, + runHomebrewGuestLifecycle, +} from "./homebrew_guest_lifecycle_runner"; +import { + deriveHomebrewGuestLifecycleRuntimeInputs, + type HomebrewGuestLifecycleRuntimeInputs, +} from "./homebrew_guest_lifecycle_runtime_inputs"; + +interface Options extends HomebrewGuestLifecycleRevisions { + imagePath: string; + bootstrapSpecPath: string; + bootstrapArchivePath: string; + bootstrapEnvironmentPath: string; + transportMode: "closed" | "public"; + bottleMirrorPlanPath?: string; + timeoutMs: number; + traceProcessesFromPid?: number; +} + +interface CapturedHost { + host: NodeKernelHost; + lazyDownloads: LazyDownloadEvent[]; + output: { + stdout: string; + stderr: string; + diagnostics: string[]; + limitExceeded: boolean; + }; +} + +const MAX_CAPTURED_OUTPUT_BYTES = 8 * 1024 * 1024; +const MAX_CAPTURED_DIAGNOSTICS = 1_000; + +async function main(): Promise { + const options = parseOptions(process.argv.slice(2)); + const runtime = loadRootfsRuntimeInputs(options); + const revisions = { + coreRevision: options.coreRevision, + canaryRevision: options.canaryRevision, + }; + + await runHomebrewGuestLifecycle({ + runtime, + revisions, + timeoutMs: options.timeoutMs, + createMachine: (machineRuntime) => + createNodeLifecycleMachine(machineRuntime, options), + }); + + process.stdout.write( + "homebrew_guest_lifecycle_node: stock install, reinstall, cross-tap " + + "dependency, durable reboot, pinned upgrade state, uninstall, and " + + "untap proof passed\n", + ); +} + +function loadRootfsRuntimeInputs( + options: Options, +): HomebrewGuestLifecycleRuntimeInputs { + const imageBytes = readRegularFile(options.imagePath, "main-shell VFS image"); + const bootstrapArchiveBytes = readRegularFile( + options.bootstrapArchivePath, + "Homebrew bootstrap archive", + ); + const bootstrapEnvironmentBytes = readRegularFile( + options.bootstrapEnvironmentPath, + "Homebrew bootstrap environment", + ); + const bootstrapSpecBytes = readRegularFile( + options.bootstrapSpecPath, + "Homebrew bootstrap tree spec", + ); + // Node can afford to re-derive the complete ZIP inventory synchronously. + // Chromium binds Web-Crypto-verified bytes to this same serialized tree + // contract without importing Node's crypto implementation. + const bootstrapTree = derivePackageDeferredZipTree( + parseJson(bootstrapSpecBytes, options.bootstrapSpecPath), + bootstrapArchiveBytes, + ); + assertPackageDeferredZipTreeState( + MemoryFileSystem.fromImage(imageBytes), + bootstrapTree, + "deferred", + ); + const bootstrapArchiveSha256 = createHash("sha256") + .update(bootstrapArchiveBytes) + .digest("hex"); + const lazyUrlBase = options.transportMode === "closed" + ? "https://closed.kandelo.invalid/homebrew-guest-lifecycle/" + : pathToFileURL(`${dirname(options.bootstrapArchivePath)}/`).toString(); + return deriveHomebrewGuestLifecycleRuntimeInputs({ + imageBytes, + bootstrapSpecBytes, + bootstrapArchiveBytes, + bootstrapArchiveSha256, + bootstrapEnvironmentBytes, + coreRevision: options.coreRevision, + transportMode: options.transportMode, + lazyUrlBase, + validateEmbeddedBottlePlan: assertHomebrewBottleMirrorPlan, + ...(options.transportMode === "public" + ? { + expectedBootstrapTransportUrl: pathToFileURL( + options.bootstrapArchivePath, + ).toString(), + } + : { + loadClosedBottleAssets: (embeddedPlanBytes, pendingBottleTrees) => + loadHomebrewBottleMirrorBindings( + options.bottleMirrorPlanPath!, + embeddedPlanBytes, + pendingBottleTrees, + ), + }), + }); +} + +function createCapturedHost( + runtime: HomebrewGuestLifecycleRuntimeInputs, + options: Options, +): CapturedHost { + const lazyDownloads: LazyDownloadEvent[] = []; + const output = { + stdout: "", + stderr: "", + diagnostics: [] as string[], + limitExceeded: false, + }; + let outputBytes = 0; + const stdoutDecoder = new TextDecoder(); + const stderrDecoder = new TextDecoder(); + const tracedProcesses = new Set(); + let host: NodeKernelHost; + const capture = (bytes: Uint8Array, stream: "stdout" | "stderr") => { + outputBytes += bytes.byteLength; + if (outputBytes > MAX_CAPTURED_OUTPUT_BYTES) { + output.limitExceeded = true; + return; + } + const decoder = stream === "stdout" ? stdoutDecoder : stderrDecoder; + output[stream] += decoder.decode(bytes, { stream: true }); + }; + const traceProcess = async (event: { + kind: "spawn" | "exec" | "exit"; + pid: number; + }) => { + if ( + options.traceProcessesFromPid === undefined || + event.pid < options.traceProcessesFromPid || + event.kind === "exit" || + tracedProcesses.has(event.pid) + ) { + return; + } + for (const delayMs of [0, 10, 50]) { + if (delayMs !== 0) await delay(delayMs); + const snapshot = (await host.enumProcs()).find( + (process) => process.pid === event.pid, + ); + if (snapshot === undefined) continue; + tracedProcesses.add(event.pid); + process.stderr.write( + `homebrew-guest-lifecycle-process: pid=${snapshot.pid} ` + + `ppid=${snapshot.ppid} state=${snapshot.state} ` + + `cmdline=${JSON.stringify(snapshot.cmdline)}\n`, + ); + return; + } + }; + host = new NodeKernelHost({ + maxWorkers: 8, + rootfsImage: runtime.imageBytes, + rootfsLazyUrlBase: runtime.lazyUrlBase, + ...(runtime.lazyAssets === undefined + ? {} + : { rootfsLazyAssets: runtime.lazyAssets }), + enableTcpNetwork: true, + dataBufferSize: 1 << 20, + onStdout: (_pid, bytes) => capture(bytes, "stdout"), + onStderr: (_pid, bytes) => capture(bytes, "stderr"), + onHostDiagnostic: (diagnostic) => { + if (output.diagnostics.length < MAX_CAPTURED_DIAGNOSTICS) { + output.diagnostics.push(diagnostic.message); + } + }, + onLazyDownload: (event) => lazyDownloads.push(event), + onProcessEvent: (event) => { + void traceProcess(event).catch((error) => { + process.stderr.write( + `homebrew-guest-lifecycle-process: trace failed: ${String(error)}\n`, + ); + }); + }, + }); + return { host, lazyDownloads, output }; +} + +function createNodeLifecycleMachine( + runtime: HomebrewGuestLifecycleRuntimeInputs, + options: Options, +): HomebrewGuestLifecycleMachine { + const captured = createCapturedHost(runtime, options); + return { + lazyDownloads: captured.lazyDownloads, + diagnostics: captured.output.diagnostics, + start: () => captured.host.init(), + runShellScript: (scriptOptions) => + runGuestScript({ captured, ...scriptOptions }), + exportRootfsImage: () => captured.host.exportRootfsImage(), + destroy: () => captured.host.destroy(), + }; +} + +async function runGuestScript(options: { + captured: CapturedHost; + shellBytes: Uint8Array; + shellArgv0: string; + script: string; + marker: string; + label: string; + timeoutMs: number; +}): Promise { + const stdoutStart = options.captured.output.stdout.length; + const stderrStart = options.captured.output.stderr.length; + const diagnosticStart = options.captured.output.diagnostics.length; + let pid: number | undefined; + let timeout: ReturnType | undefined; + try { + const exit = options.captured.host.spawn( + toArrayBuffer(options.shellBytes), + [options.shellArgv0, "-c", options.script], + { + env: [...HOMEBREW_GUEST_LIFECYCLE_ENV], + cwd: "/home/user", + uid: 1000, + gid: 1000, + stdin: new Uint8Array(), + onStarted: (startedPid) => { + pid = startedPid; + }, + }, + ); + const timedOut = new Promise((_resolve, reject) => { + timeout = setTimeout( + () => reject( + new Error(`${options.label} timed out after ${options.timeoutMs}ms`), + ), + options.timeoutMs, + ); + }); + const exitCode = await Promise.race([exit, timedOut]); + const stdout = options.captured.output.stdout.slice(stdoutStart); + const stderr = options.captured.output.stderr.slice(stderrStart); + if (exitCode !== 0) { + throw new Error( + `${options.label} exited ${exitCode}; stdout=${JSON.stringify(stdout)}; ` + + `stderr=${JSON.stringify(stderr)}; diagnostics=` + + `${JSON.stringify(options.captured.output.diagnostics)}`, + ); + } + if (!stdout.split(/\r?\n/).includes(options.marker)) { + throw new Error( + `${options.label} marker is missing; stdout=${JSON.stringify(stdout)}; ` + + `stderr=${JSON.stringify(stderr)}`, + ); + } + assertNoUnexpectedHostDiagnostics( + options.captured.output.diagnostics.slice(diagnosticStart), + options.label, + ); + if (options.captured.output.limitExceeded) { + throw new Error( + `${options.label} exceeded the ${MAX_CAPTURED_OUTPUT_BYTES}-byte output limit`, + ); + } + } catch (error) { + if (pid !== undefined) { + await options.captured.host.terminateProcess(pid, 124).catch(() => {}); + } + throw error; + } finally { + if (timeout !== undefined) clearTimeout(timeout); + } +} + +function readRegularFile(path: string, label: string): Uint8Array { + const stat = lstatSync(path); + if (!stat.isFile() || stat.isSymbolicLink()) { + throw new Error(`${label} is not a regular non-symlink file: ${path}`); + } + return new Uint8Array(readFileSync(path)); +} + +function parseJson(bytes: Uint8Array, label: string): unknown { + try { + return JSON.parse( + new TextDecoder("utf-8", { fatal: true }).decode(bytes), + ); + } catch (error) { + throw new Error(`${label} is not valid UTF-8 JSON: ${String(error)}`); + } +} + +function toArrayBuffer(bytes: Uint8Array): ArrayBuffer { + const result = new ArrayBuffer(bytes.byteLength); + new Uint8Array(result).set(bytes); + return result; +} + +function delay(milliseconds: number): Promise { + return new Promise((resolveDelay) => setTimeout(resolveDelay, milliseconds)); +} + +function parseOptions(args: string[]): Options { + const values = new Map(); + const allowed = new Set([ + "--image", + "--homebrew-bootstrap-spec", + "--homebrew-bootstrap-archive", + "--homebrew-bootstrap-env", + "--transport-mode", + "--bottle-mirror-plan", + "--core-revision", + "--canary-revision", + "--timeout-ms", + "--trace-processes-from-pid", + ]); + for (let index = 0; index < args.length; index += 2) { + const option = args[index]; + const value = args[index + 1]; + if ( + option === undefined || + value === undefined || + !allowed.has(option) || + values.has(option) + ) { + return usage(); + } + values.set(option, value); + } + const image = values.get("--image"); + const bootstrapSpec = values.get("--homebrew-bootstrap-spec"); + const bootstrapArchive = values.get("--homebrew-bootstrap-archive"); + const bootstrapEnvironment = values.get("--homebrew-bootstrap-env"); + const transportMode = values.get("--transport-mode"); + const bottleMirrorPlan = values.get("--bottle-mirror-plan"); + const coreRevision = values.get("--core-revision"); + const canaryRevision = values.get("--canary-revision"); + const timeoutMs = Number(values.get("--timeout-ms") ?? "900000"); + const traceProcessesFromPid = values.has("--trace-processes-from-pid") + ? Number(values.get("--trace-processes-from-pid")) + : undefined; + if ( + !image || + !bootstrapSpec || + !bootstrapArchive || + !bootstrapEnvironment || + !coreRevision || + !canaryRevision || + (transportMode !== "closed" && transportMode !== "public") || + (transportMode === "closed" && bottleMirrorPlan === undefined) || + (transportMode === "public" && bottleMirrorPlan !== undefined) || + !Number.isSafeInteger(timeoutMs) || + timeoutMs < 1_000 || + ( + traceProcessesFromPid !== undefined && + (!Number.isSafeInteger(traceProcessesFromPid) || + traceProcessesFromPid < 1) + ) + ) { + return usage(); + } + assertHomebrewGuestLifecycleRevisions({ coreRevision, canaryRevision }); + return { + imagePath: resolve(image), + bootstrapSpecPath: resolve(bootstrapSpec), + bootstrapArchivePath: resolve(bootstrapArchive), + bootstrapEnvironmentPath: resolve(bootstrapEnvironment), + transportMode, + ...(bottleMirrorPlan === undefined + ? {} + : { bottleMirrorPlanPath: resolve(bottleMirrorPlan) }), + coreRevision, + canaryRevision, + timeoutMs, + ...(traceProcessesFromPid === undefined + ? {} + : { traceProcessesFromPid }), + }; +} + +function usage(): never { + throw new Error( + "usage: npx tsx homebrew/test/homebrew_guest_lifecycle_node.ts " + + "--image " + + "--homebrew-bootstrap-spec " + + "--homebrew-bootstrap-archive " + + "--homebrew-bootstrap-env " + + "--transport-mode " + + "[--bottle-mirror-plan ] " + + "--core-revision <40-character SHA> " + + "--canary-revision <40-character SHA> [--timeout-ms ] " + + "[--trace-processes-from-pid ]", + ); +} + +await main(); diff --git a/homebrew/test/homebrew_guest_lifecycle_runner.test.ts b/homebrew/test/homebrew_guest_lifecycle_runner.test.ts new file mode 100644 index 0000000000..9036e367ec --- /dev/null +++ b/homebrew/test/homebrew_guest_lifecycle_runner.test.ts @@ -0,0 +1,169 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + MemoryFileSystem, + type LazyDownloadEvent, +} from "../../host/src/vfs/memory-fs"; +import { + HOMEBREW_GUEST_LIFECYCLE_PHASE_ONE_MARKER, + HOMEBREW_GUEST_LIFECYCLE_PHASE_TWO_MARKER, +} from "./homebrew_guest_lifecycle_contract"; +import { + type HomebrewGuestLifecycleMachine, + runHomebrewGuestLifecycle, +} from "./homebrew_guest_lifecycle_runner"; +import type { + HomebrewGuestLifecycleRuntimeInputs, +} from "./homebrew_guest_lifecycle_runtime_inputs"; + +test("runs one shared lifecycle contract across export and reboot", async () => { + const bootstrapUrl = "https://example.test/homebrew-bootstrap.zip"; + const exportedImage = await createExportedImage(); + const scripts: Array<{ phase: string; marker: string; script: string }> = []; + const runtime: HomebrewGuestLifecycleRuntimeInputs = { + imageBytes: new Uint8Array([1]), + shellBytes: new Uint8Array([0, 97, 115, 109]), + shellArgv0: "bash", + lazyUrlBase: "https://example.test/", + bootstrapTransportUrl: bootstrapUrl, + bootstrapBytes: 7, + }; + + const result = await runHomebrewGuestLifecycle({ + runtime, + revisions: { + coreRevision: "1".repeat(40), + canaryRevision: "2".repeat(40), + }, + timeoutMs: 1_000, + createMachine: (_machineRuntime, phase) => { + const events: LazyDownloadEvent[] = []; + return { + lazyDownloads: events, + diagnostics: [], + start: async () => {}, + runShellScript: async ({ marker, script }) => { + scripts.push({ phase, marker, script }); + if (marker === HOMEBREW_GUEST_LIFECYCLE_PHASE_ONE_MARKER) { + events.push( + event(bootstrapUrl, "started", 0), + event(bootstrapUrl, "complete", 7), + ); + } + }, + exportRootfsImage: async () => exportedImage, + destroy: async () => {}, + }; + }, + }); + + assert.deepEqual([...result.phaseOneCompletedUrls], [bootstrapUrl]); + assert.deepEqual(result.exportedImage, exportedImage); + assert.equal(scripts.length, 3); + assert.deepEqual( + scripts.map(({ phase, marker }) => ({ phase, marker })), + [ + { phase: "phase-one", marker: "homebrew-lifecycle-offline-ok" }, + { + phase: "phase-one", + marker: HOMEBREW_GUEST_LIFECYCLE_PHASE_ONE_MARKER, + }, + { + phase: "phase-two", + marker: HOMEBREW_GUEST_LIFECYCLE_PHASE_TWO_MARKER, + }, + ], + ); + assert.match(scripts[1]!.script, /brew install --no-ask --force-bottle/); + assert.match(scripts[1]!.script, /brew reinstall --force-bottle/); + assert.match(scripts[2]!.script, /brew upgrade --force-bottle/); + assert.match(scripts[2]!.script, /brew uninstall /); + assert.match(scripts[2]!.script, /brew untap/); +}); + +test("rejects a preflight that materializes a supposedly image-owned shell", async () => { + const runtime: HomebrewGuestLifecycleRuntimeInputs = { + imageBytes: new Uint8Array([1]), + shellBytes: new Uint8Array([0, 97, 115, 109]), + shellArgv0: "bash", + lazyUrlBase: "https://example.test/", + bootstrapTransportUrl: "https://example.test/homebrew-bootstrap.zip", + bootstrapBytes: 1, + }; + await assert.rejects( + () => + runHomebrewGuestLifecycle({ + runtime, + revisions: { + coreRevision: "1".repeat(40), + canaryRevision: "2".repeat(40), + }, + timeoutMs: 1_000, + createMachine: () => { + const events: LazyDownloadEvent[] = []; + const machine: HomebrewGuestLifecycleMachine = { + lazyDownloads: events, + diagnostics: [], + start: async () => {}, + runShellScript: async () => { + events.push( + event("https://example.test/bash.wasm", "complete", 1), + ); + }, + exportRootfsImage: async () => new Uint8Array(), + destroy: async () => {}, + }; + return machine; + }, + }), + /image-owned shell preflight unexpectedly fetched/, + ); +}); + +async function createExportedImage(): Promise { + const fs = MemoryFileSystem.create(new SharedArrayBuffer(2 * 1024 * 1024)); + fs.mkdir("/etc", 0o755); + fs.mkdir("/etc/kandelo", 0o755); + fs.mkdir("/bin", 0o755); + writeFile( + fs, + "/etc/kandelo/shell.json", + new TextEncoder().encode(JSON.stringify({ + version: 1, + path: "/bin/bash", + argv: ["bash", "-l", "-i"], + })), + ); + writeFile(fs, "/bin/bash", new Uint8Array([0, 97, 115, 109]), 0o755); + return fs.saveImage(); +} + +function writeFile( + fs: MemoryFileSystem, + path: string, + bytes: Uint8Array, + mode = 0o644, +): void { + const fd = fs.open(path, 0o1101, mode); + try { + assert.equal(fs.write(fd, bytes, null, bytes.byteLength), bytes.byteLength); + } finally { + fs.close(fd); + } +} + +function event( + url: string, + status: "started" | "complete", + loadedBytes: number, +): LazyDownloadEvent { + return { + id: url, + kind: "tree", + status, + url, + loadedBytes, + t: 0, + }; +} diff --git a/homebrew/test/homebrew_guest_lifecycle_runner.ts b/homebrew/test/homebrew_guest_lifecycle_runner.ts new file mode 100644 index 0000000000..d3f2656044 --- /dev/null +++ b/homebrew/test/homebrew_guest_lifecycle_runner.ts @@ -0,0 +1,249 @@ +import { + MemoryFileSystem, + type LazyDownloadEvent, +} from "../../host/src/vfs/memory-fs"; +import { + createHomebrewGuestLifecyclePhaseOneScript, + createHomebrewGuestLifecyclePhaseTwoScript, + HOMEBREW_GUEST_LIFECYCLE_PHASE_ONE_MARKER, + HOMEBREW_GUEST_LIFECYCLE_PHASE_TWO_MARKER, + type HomebrewGuestLifecycleRevisions, +} from "./homebrew_guest_lifecycle_contract"; +import { + assertNoRepeatedLazyDownloads, + assertNoUnexpectedHostDiagnostics, + completedLazyDownloadUrls, + omitCompletedClosedLazyAssets, + resolveHomebrewGuestLifecycleShell, +} from "./homebrew_guest_lifecycle_runtime_contract"; +import type { + HomebrewGuestLifecycleRuntimeInputs, +} from "./homebrew_guest_lifecycle_runtime_inputs"; + +export const HOMEBREW_GUEST_LIFECYCLE_ENV = [ + "PATH=/home/linuxbrew/.linuxbrew/bin:/usr/bin:/bin", + "HOME=/home/user", + "USER=user", + "LOGNAME=user", + "SHELL=/bin/bash", + "TERM=dumb", + "TMPDIR=/tmp", + "HOMEBREW_NO_ANALYTICS=1", + "HOMEBREW_NO_AUTO_UPDATE=1", + "HOMEBREW_NO_ENV_HINTS=1", + "HOMEBREW_NO_INSTALL_FROM_API=1", + "GIT_TERMINAL_PROMPT=0", +] as const; + +export type HomebrewGuestLifecyclePhase = "phase-one" | "phase-two"; + +export interface HomebrewGuestLifecycleMachine { + readonly lazyDownloads: readonly LazyDownloadEvent[]; + readonly diagnostics: readonly string[]; + start(): Promise; + runShellScript(options: { + shellBytes: Uint8Array; + shellArgv0: string; + script: string; + marker: string; + label: string; + timeoutMs: number; + }): Promise; + exportRootfsImage(): Promise; + destroy(): Promise; +} + +export interface HomebrewGuestLifecycleRunResult { + exportedImage: Uint8Array; + phaseOneCompletedUrls: ReadonlySet; + phaseOneLazyDownloads: readonly LazyDownloadEvent[]; + phaseTwoLazyDownloads: readonly LazyDownloadEvent[]; +} + +export async function runHomebrewGuestLifecycle(options: { + runtime: HomebrewGuestLifecycleRuntimeInputs; + revisions: HomebrewGuestLifecycleRevisions; + timeoutMs: number; + createMachine: ( + runtime: HomebrewGuestLifecycleRuntimeInputs, + phase: HomebrewGuestLifecyclePhase, + ) => HomebrewGuestLifecycleMachine; +}): Promise { + const phaseOneMachine = options.createMachine( + options.runtime, + "phase-one", + ); + let exportedImage: Uint8Array | undefined; + let phaseOneCompletedUrls: ReadonlySet | undefined; + let phaseOneLazyDownloads: readonly LazyDownloadEvent[] | undefined; + try { + await phaseOneMachine.start(); + const preflightStart = phaseOneMachine.lazyDownloads.length; + await phaseOneMachine.runShellScript({ + shellBytes: options.runtime.shellBytes, + shellArgv0: options.runtime.shellArgv0, + script: + "set -eu; test -n \"$BASH_VERSION\"; " + + "printf 'homebrew-lifecycle-offline-ok\\n'", + marker: "homebrew-lifecycle-offline-ok", + label: "Homebrew lifecycle image-owned shell preflight", + timeoutMs: options.timeoutMs, + }); + assertNoLazyDownload( + phaseOneMachine.lazyDownloads.slice(preflightStart), + "image-owned shell preflight", + ); + + await phaseOneMachine.runShellScript({ + shellBytes: options.runtime.shellBytes, + shellArgv0: options.runtime.shellArgv0, + script: createHomebrewGuestLifecyclePhaseOneScript(options.revisions), + marker: HOMEBREW_GUEST_LIFECYCLE_PHASE_ONE_MARKER, + label: "stock Homebrew guest lifecycle phase one", + timeoutMs: options.timeoutMs, + }); + assertSingleCompletedLazyDownload( + phaseOneMachine.lazyDownloads, + options.runtime.bootstrapTransportUrl, + options.runtime.bootstrapBytes, + "phase-one Homebrew bootstrap", + ); + phaseOneLazyDownloads = [...phaseOneMachine.lazyDownloads]; + phaseOneCompletedUrls = completedLazyDownloadUrls( + phaseOneLazyDownloads, + ); + exportedImage = await exportRootfsAfterProcessTeardown( + phaseOneMachine, + 5_000, + ); + } finally { + await phaseOneMachine.destroy().catch(() => {}); + assertNoUnexpectedHostDiagnostics( + phaseOneMachine.diagnostics, + "stock Homebrew guest lifecycle phase one host", + ); + } + if ( + exportedImage === undefined || + phaseOneCompletedUrls === undefined || + phaseOneLazyDownloads === undefined + ) { + throw new Error("phase one did not export a durable root filesystem"); + } + + const exportedFs = MemoryFileSystem.fromImage(exportedImage); + const exportedShell = resolveHomebrewGuestLifecycleShell(exportedFs); + const phaseTwoRuntime: HomebrewGuestLifecycleRuntimeInputs = { + ...options.runtime, + imageBytes: exportedImage, + shellBytes: exportedShell.bytes, + shellArgv0: exportedShell.argv0, + // WHY: a rebooted image must own everything phase one materialized. Do not + // leave those closed bytes available to hide an export durability defect. + lazyAssets: omitCompletedClosedLazyAssets( + options.runtime.lazyAssets, + phaseOneCompletedUrls, + ), + }; + const phaseTwoMachine = options.createMachine( + phaseTwoRuntime, + "phase-two", + ); + let phaseTwoLazyDownloads: readonly LazyDownloadEvent[] | undefined; + try { + await phaseTwoMachine.start(); + await phaseTwoMachine.runShellScript({ + shellBytes: phaseTwoRuntime.shellBytes, + shellArgv0: phaseTwoRuntime.shellArgv0, + script: createHomebrewGuestLifecyclePhaseTwoScript(options.revisions), + marker: HOMEBREW_GUEST_LIFECYCLE_PHASE_TWO_MARKER, + label: "stock Homebrew guest lifecycle phase two after rootfs reboot", + timeoutMs: options.timeoutMs, + }); + phaseTwoLazyDownloads = [...phaseTwoMachine.lazyDownloads]; + assertNoRepeatedLazyDownloads( + phaseOneCompletedUrls, + phaseTwoLazyDownloads, + "rebooted lifecycle", + ); + } finally { + await phaseTwoMachine.destroy().catch(() => {}); + assertNoUnexpectedHostDiagnostics( + phaseTwoMachine.diagnostics, + "stock Homebrew guest lifecycle phase two host", + ); + } + if (phaseTwoLazyDownloads === undefined) { + throw new Error("phase two did not complete after the durable reboot"); + } + + return { + exportedImage, + phaseOneCompletedUrls, + phaseOneLazyDownloads, + phaseTwoLazyDownloads, + }; +} + +async function exportRootfsAfterProcessTeardown( + machine: HomebrewGuestLifecycleMachine, + timeoutMs: number, +): Promise { + const deadline = Date.now() + timeoutMs; + for (;;) { + try { + return await machine.exportRootfsImage(); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if ( + !message.includes("no live or tearing-down processes") || + Date.now() >= deadline + ) { + throw error; + } + // WHY: a process exit is observable before the worker has necessarily + // finished its asynchronous teardown. Retry only that exact transient + // state; the worker-owned snapshot gate remains authoritative. + await delay(20); + } + } +} + +function assertSingleCompletedLazyDownload( + events: readonly LazyDownloadEvent[], + url: string, + expectedBytes: number, + label: string, +): void { + const matches = events.filter((event) => event.url === url); + const started = matches.filter((event) => event.status === "started"); + const completed = matches.filter((event) => event.status === "complete"); + const failed = matches.filter((event) => event.status === "error"); + if ( + started.length !== 1 || + completed.length !== 1 || + failed.length !== 0 || + matches[0]?.status !== "started" || + matches.at(-1)?.status !== "complete" || + started[0]?.loadedBytes !== 0 || + completed[0]?.loadedBytes !== expectedBytes + ) { + throw new Error( + `${label} must fetch its exact lazy tree once; events=` + + `${JSON.stringify(matches)}`, + ); + } +} + +function assertNoLazyDownload( + events: readonly LazyDownloadEvent[], + label: string, +): void { + if (events.length !== 0) { + throw new Error(`${label} unexpectedly fetched ${events[0]!.url}`); + } +} + +function delay(milliseconds: number): Promise { + return new Promise((resolveDelay) => setTimeout(resolveDelay, milliseconds)); +} diff --git a/homebrew/test/homebrew_guest_lifecycle_runtime_contract.test.ts b/homebrew/test/homebrew_guest_lifecycle_runtime_contract.test.ts new file mode 100644 index 0000000000..9310909c49 --- /dev/null +++ b/homebrew/test/homebrew_guest_lifecycle_runtime_contract.test.ts @@ -0,0 +1,223 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import test from "node:test"; + +import { + snapshotClosedLazyAssets, + type ClosedLazyAsset, +} from "../../host/src/vfs/closed-lazy-assets"; +import { + MemoryFileSystem, + type LazyDownloadEvent, + type LazyDownloadStatus, +} from "../../host/src/vfs/memory-fs"; +import { + assertHomebrewGuestLifecycleCatalog, + assertNoRepeatedLazyDownloads, + assertNoUnexpectedHostDiagnostics, + completedLazyDownloadUrls, + omitCompletedClosedLazyAssets, + resolveHomebrewGuestLifecycleShell, +} from "./homebrew_guest_lifecycle_runtime_contract"; + +const textEncoder = new TextEncoder(); + +test("binds the lifecycle revision to the exact embedded core catalog", () => { + const revision = "1".repeat(40); + const manifest = { + schema: 1, + catalog: { + tap_repository: "kandelo-dev/homebrew-tap-core", + tap_name: "kandelo-dev/tap-core", + checkout_commit: revision, + }, + }; + assert.doesNotThrow(() => + assertHomebrewGuestLifecycleCatalog(manifest, revision) + ); + + for (const [key, value, message] of [ + ["tap_repository", "someone/else", "tap_repository"], + ["tap_name", "someone/else", "tap_name"], + ["checkout_commit", "2".repeat(40), "checkout_commit"], + ] as const) { + const changed = structuredClone(manifest); + changed.catalog[key] = value; + assert.throws( + () => assertHomebrewGuestLifecycleCatalog(changed, revision), + new RegExp(message), + ); + } +}); + +test("resolves the image-owned shell from the rebooted filesystem", async () => { + const source = createShellFileSystem(new Uint8Array([0, 97, 115, 109])); + const rebooted = MemoryFileSystem.fromImage(await source.saveImage()); + const resolved = resolveHomebrewGuestLifecycleShell(rebooted); + + assert.deepEqual(resolved, { + bytes: new Uint8Array([0, 97, 115, 109]), + argv0: "bash", + }); +}); + +test("rejects a shell that export left deferred or non-executable", () => { + const deferred = createBaseFileSystem(); + writeFile( + deferred, + "/etc/kandelo/shell.json", + textEncoder.encode(JSON.stringify({ + version: 1, + path: "/bin/bash", + argv: ["bash", "-l", "-i"], + })), + ); + deferred.registerLazyFile( + "/bin/bash", + "https://example.test/bash.wasm", + 123, + 0o755, + ); + assert.throws( + () => resolveHomebrewGuestLifecycleShell(deferred), + /must be image-owned.*deferred/, + ); + + const nonExecutable = createShellFileSystem(new Uint8Array([1]), 0o644); + assert.throws( + () => resolveHomebrewGuestLifecycleShell(nonExecutable), + /not an executable regular file/, + ); +}); + +test("removes phase-one materialized assets and rejects every repeated event", () => { + const firstUrl = "https://example.test/first"; + const secondUrl = "https://example.test/second"; + const phaseOne = [ + event(firstUrl, "started"), + event(firstUrl, "complete"), + event(secondUrl, "complete"), + ]; + const completed = completedLazyDownloadUrls(phaseOne); + assert.deepEqual([...completed], [firstUrl, secondUrl]); + + const assets: ClosedLazyAsset[] = [ + asset(firstUrl, 1), + asset(secondUrl, 2), + asset("https://example.test/unopened", 3), + ]; + assert.deepEqual( + omitCompletedClosedLazyAssets(assets, completed), + [assets[2]], + ); + const allCompleted = new Set(assets.map(({ url }) => url)); + const guarded = omitCompletedClosedLazyAssets(assets, allCompleted); + assert.equal(guarded?.length, 1); + assert.ok(!allCompleted.has(guarded![0]!.url)); + assert.equal( + createHash("sha256").update(guarded![0]!.bytes).digest("hex"), + guarded![0]!.sha256, + ); + assert.doesNotThrow(() => snapshotClosedLazyAssets(guarded!)); + assert.doesNotThrow(() => + assertNoRepeatedLazyDownloads( + completed, + [event("https://example.test/unopened", "complete")], + "reboot", + ) + ); + for (const status of [ + "started", + "progress", + "complete", + "error", + ] as const) { + assert.throws( + () => + assertNoRepeatedLazyDownloads( + completed, + [event(firstUrl, status)], + "reboot", + ), + new RegExp(`phase-one materialized URL ${firstUrl}.*status ${status}`), + ); + } +}); + +test("fails closed on every host diagnostic", () => { + assert.doesNotThrow(() => + assertNoUnexpectedHostDiagnostics([], "lifecycle") + ); + for (const diagnostic of [ + "ordinary protocol failure", + "bytes 65536..131072 (FORK_SAVE_BUFFER_SIZE) are reserved", + ]) { + assert.throws( + () => assertNoUnexpectedHostDiagnostics([diagnostic], "lifecycle"), + /unexpected host diagnostics/, + ); + } +}); + +function createBaseFileSystem(): MemoryFileSystem { + const fs = MemoryFileSystem.create(new SharedArrayBuffer(2 * 1024 * 1024)); + fs.mkdir("/etc", 0o755); + fs.mkdir("/etc/kandelo", 0o755); + fs.mkdir("/bin", 0o755); + return fs; +} + +function createShellFileSystem( + bytes: Uint8Array, + mode = 0o755, +): MemoryFileSystem { + const fs = createBaseFileSystem(); + writeFile( + fs, + "/etc/kandelo/shell.json", + textEncoder.encode(JSON.stringify({ + version: 1, + path: "/bin/bash", + argv: ["bash", "-l", "-i"], + })), + ); + writeFile(fs, "/bin/bash", bytes, mode); + return fs; +} + +function writeFile( + fs: MemoryFileSystem, + path: string, + bytes: Uint8Array, + mode = 0o644, +): void { + const fd = fs.open(path, 0o1101, mode); + try { + assert.equal(fs.write(fd, bytes, null, bytes.byteLength), bytes.byteLength); + } finally { + fs.close(fd); + } +} + +function event( + url: string, + status: LazyDownloadStatus, +): LazyDownloadEvent { + return { + id: url, + kind: "tree", + status, + url, + loadedBytes: status === "complete" ? 1 : 0, + t: 0, + }; +} + +function asset(url: string, byte: number): ClosedLazyAsset { + return { + url, + sha256: byte.toString(16).padStart(64, "0"), + size: 1, + bytes: new Uint8Array([byte]), + }; +} diff --git a/homebrew/test/homebrew_guest_lifecycle_runtime_contract.ts b/homebrew/test/homebrew_guest_lifecycle_runtime_contract.ts new file mode 100644 index 0000000000..71851244a6 --- /dev/null +++ b/homebrew/test/homebrew_guest_lifecycle_runtime_contract.ts @@ -0,0 +1,149 @@ +import type { ClosedLazyAsset } from "../../host/src/vfs/closed-lazy-assets"; +import { + MemoryFileSystem, + type LazyDownloadEvent, +} from "../../host/src/vfs/memory-fs"; +import { + KANDELO_SHELL_CONFIG_PATH, + parseKandeloShellConfig, +} from "../../web-libs/kandelo-session/src/shell-config"; +import { assertMainShellGuestCatalogIdentity } from + "../../scripts/homebrew-main-shell-catalog-contract"; +import { + HOMEBREW_GUEST_LIFECYCLE_CORE_REPOSITORY, + HOMEBREW_GUEST_LIFECYCLE_CORE_TAP, +} from "./homebrew_guest_lifecycle_contract"; + +export interface HomebrewGuestLifecycleShell { + bytes: Uint8Array; + argv0: string; +} + +export function assertHomebrewGuestLifecycleCatalog( + guestManifest: unknown, + coreRevision: string, +): void { + assertMainShellGuestCatalogIdentity(guestManifest, { + tapRepository: HOMEBREW_GUEST_LIFECYCLE_CORE_REPOSITORY, + tapName: HOMEBREW_GUEST_LIFECYCLE_CORE_TAP, + tapCommit: coreRevision, + }); +} + +/** + * Resolve the executable from the supplied filesystem rather than carrying + * bytes over from an earlier boot. A reboot proof must fail if export omitted + * or re-deferred the image-owned shell. + */ +export function resolveHomebrewGuestLifecycleShell( + fs: MemoryFileSystem, +): HomebrewGuestLifecycleShell { + const shellConfig = parseKandeloShellConfig( + new TextDecoder("utf-8", { fatal: true }).decode( + readVfsFile(fs, KANDELO_SHELL_CONFIG_PATH), + ), + ); + if (shellConfig === null) { + throw new Error(`${KANDELO_SHELL_CONFIG_PATH} has an unsupported schema`); + } + if (fs.isPathDeferred(shellConfig.path)) { + throw new Error( + `lifecycle shell must be image-owned, but ${shellConfig.path} is deferred`, + ); + } + const stat = fs.stat(shellConfig.path); + if ((stat.mode & 0xf000) !== 0x8000 || (stat.mode & 0o111) === 0) { + throw new Error(`${shellConfig.path} is not an executable regular file`); + } + return { + bytes: readVfsFile(fs, shellConfig.path, stat.size), + argv0: shellConfig.argv[0]!, + }; +} + +export function completedLazyDownloadUrls( + events: readonly LazyDownloadEvent[], +): ReadonlySet { + return new Set( + events + .filter((event) => event.status === "complete") + .map((event) => event.url), + ); +} + +/** + * Remove materialized phase-one payloads from the closed reboot transport. + * If export accidentally restores one as deferred, phase two must fail closed + * instead of hiding the durability regression with the original local bytes. + */ +export function omitCompletedClosedLazyAssets( + assets: readonly ClosedLazyAsset[] | undefined, + completedUrls: ReadonlySet, +): readonly ClosedLazyAsset[] | undefined { + if (assets === undefined) return undefined; + const remaining = assets.filter((asset) => !completedUrls.has(asset.url)); + if (remaining.length !== 0) return remaining; + // WHY: the host's exhaustive closed transport intentionally rejects an + // empty binding set. Keep that transport active with one unreachable guard + // identity; no VFS descriptor names it, while every phase-one URL remains + // absent and therefore fails closed if export accidentally re-defers it. + return [{ + url: "https://closed.kandelo.invalid/homebrew-guest-lifecycle/reboot-guard", + sha256: + "6e340b9cffb37a989ca544e6bb780a2c78901d3fb33738768511a30617afa01d", + size: 1, + bytes: new Uint8Array([0]), + }]; +} + +export function assertNoRepeatedLazyDownloads( + phaseOneCompletedUrls: ReadonlySet, + phaseTwoEvents: readonly LazyDownloadEvent[], + label: string, +): void { + const repeated = phaseTwoEvents.find((event) => + phaseOneCompletedUrls.has(event.url) + ); + if (repeated !== undefined) { + throw new Error( + `${label} fetched phase-one materialized URL ${repeated.url} ` + + `with status ${repeated.status}`, + ); + } +} + +export function assertNoUnexpectedHostDiagnostics( + diagnostics: readonly string[], + label: string, +): void { + if (diagnostics.length !== 0) { + throw new Error( + `${label} emitted unexpected host diagnostics: ${JSON.stringify(diagnostics)}`, + ); + } +} + +function readVfsFile( + fs: MemoryFileSystem, + path: string, + expectedSize?: number, +): Uint8Array { + const stat = fs.stat(path); + const size = expectedSize ?? stat.size; + if ((stat.mode & 0xf000) !== 0x8000 || stat.size !== size) { + throw new Error(`${path} is not the expected regular file`); + } + const bytes = new Uint8Array(size); + const fd = fs.open(path, 0, 0); + try { + let offset = 0; + while (offset < size) { + const count = fs.read(fd, bytes.subarray(offset), null, size - offset); + if (count <= 0) throw new Error(`${path} ended before ${size} bytes`); + offset += count; + } + } finally { + fs.close(fd); + } + return bytes; +} diff --git a/homebrew/test/homebrew_guest_lifecycle_runtime_inputs.test.ts b/homebrew/test/homebrew_guest_lifecycle_runtime_inputs.test.ts new file mode 100644 index 0000000000..58ed5873cb --- /dev/null +++ b/homebrew/test/homebrew_guest_lifecycle_runtime_inputs.test.ts @@ -0,0 +1,227 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import test from "node:test"; +import { zipSync, type Zippable } from "fflate"; + +import { + encodeHomebrewBottleMirrorCollectionIdentity, + encodeHomebrewBottleMirrorPlan, + HOMEBREW_BOTTLE_MIRROR_PLAN_ASSET, + HOMEBREW_BOTTLE_MIRROR_PLAN_KIND, + HOMEBREW_BOTTLE_MIRROR_PLAN_VFS_PATH, + type HomebrewBottleMirrorPlan, +} from "../../host/src/homebrew-bottle-mirror-plan"; +import { homebrewRuntimeLayerPayloadAsset } from + "../../host/src/homebrew-runtime-layer-limits"; +import { MemoryFileSystem } from "../../host/src/vfs/memory-fs"; +import { + derivePackageDeferredZipTree, + registerPackageDeferredZipTree, +} from "../../host/src/vfs/package-deferred-tree"; +import { + deriveHomebrewGuestLifecycleRuntimeInputs, +} from "./homebrew_guest_lifecycle_runtime_inputs"; + +const encoder = new TextEncoder(); + +test("binds verified bootstrap bytes and bottle payloads to one exact image", async () => { + const coreRevision = "1".repeat(40); + const bootstrapArchive = zipSync({ + "bin/": zipEntry(new Uint8Array(), 0o040755), + "bin/brew": zipEntry(encoder.encode("#!/bin/sh\n"), 0o100755), + }, { level: 9 }); + const bootstrapSpec = { + schema: 1, + kind: "kandelo-package-deferred-zip-tree", + id: "shell/homebrew-bootstrap", + content_role: "source-tree", + package: { + name: "shell", + output: "homebrew-bootstrap.zip", + }, + archive: { + url: "homebrew-bootstrap.zip", + mode_policy: "portable-posix-v1", + }, + mount_prefix: "/home/linuxbrew/.linuxbrew", + owner: { uid: 1000, gid: 1000 }, + activation: { + mode: "first-use", + capabilities: ["homebrew:bootstrap"], + roots: ["/home/linuxbrew/.linuxbrew/bin/brew"], + }, + } as const; + const bootstrapTree = derivePackageDeferredZipTree( + bootstrapSpec, + bootstrapArchive, + ); + const bottleBytes = new Uint8Array([42]); + const mirror = createMirrorPlan(bottleBytes); + const mirrorBytes = encodeHomebrewBottleMirrorPlan(mirror); + const environmentBytes = encoder.encode( + "HOMEBREW_SYSTEM=Kandelo\nHOMEBREW_PROCESSOR=wasm32\n", + ); + + const fs = MemoryFileSystem.create( + new SharedArrayBuffer(32 * 1024 * 1024), + ); + for (const path of [ + "/etc", + "/etc/kandelo", + "/etc/homebrew", + "/bin", + "/home", + "/home/linuxbrew", + "/home/linuxbrew/.linuxbrew", + "/bottle", + ]) { + fs.mkdir(path, 0o755); + } + fs.chown("/home", 1000, 1000); + fs.chown("/home/linuxbrew", 1000, 1000); + fs.chown("/home/linuxbrew/.linuxbrew", 1000, 1000); + writeFile(fs, "/bin/bash", new Uint8Array([0, 97, 115, 109]), 0o755); + writeFile( + fs, + "/etc/kandelo/shell.json", + encoder.encode(JSON.stringify({ + version: 1, + path: "/bin/bash", + argv: ["bash", "-l", "-i"], + })), + ); + writeFile(fs, "/etc/homebrew/brew.env", environmentBytes); + writeFile( + fs, + "/etc/kandelo/homebrew-vfs.json", + encoder.encode(JSON.stringify({ + schema: 1, + catalog: { + tap_repository: "kandelo-dev/homebrew-tap-core", + tap_name: "kandelo-dev/tap-core", + checkout_commit: coreRevision, + }, + })), + ); + writeFile(fs, HOMEBREW_BOTTLE_MIRROR_PLAN_VFS_PATH, mirrorBytes); + registerPackageDeferredZipTree(fs, bootstrapTree); + fs.registerLazyTree( + { + decoder: "zip-v1", + mediaType: "application/zip", + sha256: sha256(bottleBytes), + bytes: bottleBytes.byteLength, + expandedBytes: 1, + sourceEntryCount: 1, + transports: [mirror.assets[0]!.url], + modePolicy: "portable-posix-v1", + }, + [{ + vfsPath: "/bottle/tool", + sourcePath: "tool", + type: "file", + mode: 0o755, + size: 1, + inodeGroup: "bottle:tool", + }], + "/bottle", + { + mode: "first-use", + capabilities: ["homebrew-bottle:bottle-test"], + roots: ["/bottle/tool"], + }, + { uid: 1000, gid: 1000 }, + ); + + const imageBytes = await fs.saveImage(); + let validatedMirror: HomebrewBottleMirrorPlan | undefined; + const runtime = deriveHomebrewGuestLifecycleRuntimeInputs({ + imageBytes, + bootstrapSpecBytes: encoder.encode(JSON.stringify(bootstrapSpec)), + bootstrapArchiveBytes: bootstrapArchive, + bootstrapArchiveSha256: sha256(bootstrapArchive), + bootstrapEnvironmentBytes: environmentBytes, + coreRevision, + transportMode: "closed", + lazyUrlBase: "https://closed.kandelo.invalid/lifecycle/", + expectedEmbeddedBottlePlanBytes: mirrorBytes, + validateEmbeddedBottlePlan: (plan) => { + validatedMirror = plan; + }, + closedBottleAssets: [{ + url: mirror.assets[0]!.url, + sha256: sha256(bottleBytes), + size: bottleBytes.byteLength, + bytes: bottleBytes, + }], + }); + + assert.deepEqual(runtime.shellBytes, new Uint8Array([0, 97, 115, 109])); + assert.equal(runtime.shellArgv0, "bash"); + assert.equal( + runtime.bootstrapTransportUrl, + "https://closed.kandelo.invalid/lifecycle/homebrew-bootstrap.zip", + ); + assert.equal(runtime.bootstrapBytes, bootstrapArchive.byteLength); + assert.equal(runtime.lazyAssets?.length, 2); + assert.equal( + runtime.lazyAssets?.[1]?.sha256, + bootstrapTree.content.sha256, + ); + assert.deepEqual(validatedMirror, mirror); +}); + +function createMirrorPlan(payload: Uint8Array): HomebrewBottleMirrorPlan { + const repository = "example/project"; + const identity = { + id: "bottle-test", + package: "example/tap/test", + asset: homebrewRuntimeLayerPayloadAsset("bottle-test"), + sha256: sha256(payload), + bytes: payload.byteLength, + }; + const collection = sha256( + encodeHomebrewBottleMirrorCollectionIdentity(repository, [identity]), + ); + const tag = `homebrew-shell-bottles-sha256-${collection}`; + const releaseRoot = + `https://github.com/${repository}/releases/download/${tag}`; + return { + schema: 1, + kind: HOMEBREW_BOTTLE_MIRROR_PLAN_KIND, + repository, + collection_sha256: collection, + tag, + release_root: releaseRoot, + manifest_asset: HOMEBREW_BOTTLE_MIRROR_PLAN_ASSET, + assets: [{ + ...identity, + url: `${releaseRoot}/${identity.asset}`, + }], + }; +} + +function zipEntry( + bytes: Uint8Array, + mode: number, +): Zippable[string] { + return [bytes, { os: 3, attrs: ((mode << 16) >>> 0) }]; +} + +function writeFile( + fs: MemoryFileSystem, + path: string, + bytes: Uint8Array, + mode = 0o644, +): void { + const fd = fs.open(path, 0o1101, mode); + try { + assert.equal(fs.write(fd, bytes, null, bytes.byteLength), bytes.byteLength); + } finally { + fs.close(fd); + } +} + +function sha256(bytes: Uint8Array): string { + return createHash("sha256").update(bytes).digest("hex"); +} diff --git a/homebrew/test/homebrew_guest_lifecycle_runtime_inputs.ts b/homebrew/test/homebrew_guest_lifecycle_runtime_inputs.ts new file mode 100644 index 0000000000..2523cca3a1 --- /dev/null +++ b/homebrew/test/homebrew_guest_lifecycle_runtime_inputs.ts @@ -0,0 +1,417 @@ +import type { ClosedLazyAsset } from "../../host/src/vfs/closed-lazy-assets"; +import { + MemoryFileSystem, + type SerializedLazyArchiveEntry, +} from "../../host/src/vfs/memory-fs"; +import { + parsePackageDeferredZipTreeSpec, + type PackageDeferredZipTreeSpec, +} from "../../host/src/vfs/package-deferred-tree-contract"; +import { + HOMEBREW_BOTTLE_MIRROR_PLAN_VFS_PATH, + type HomebrewBottleMirrorPlan, +} from "../../host/src/homebrew-bottle-mirror-plan"; +import { + assertPendingTreeHomebrewBottleMirrorBinding, + bytesEqual, + decodeHomebrewBottleMirrorPlan, +} from "../../scripts/homebrew-closed-lazy-assets-contract"; +import { + assertHomebrewGuestLifecycleCatalog, + resolveHomebrewGuestLifecycleShell, +} from "./homebrew_guest_lifecycle_runtime_contract"; + +export type HomebrewGuestLifecycleTransportMode = "closed" | "public"; + +export interface HomebrewGuestLifecycleRuntimeInputs { + imageBytes: Uint8Array; + shellBytes: Uint8Array; + shellArgv0: string; + lazyUrlBase: string; + lazyAssets?: readonly ClosedLazyAsset[]; + bootstrapTransportUrl: string; + bootstrapBytes: number; +} + +export interface DeriveHomebrewGuestLifecycleRuntimeInputs { + imageBytes: Uint8Array; + bootstrapSpecBytes: Uint8Array; + bootstrapArchiveBytes: Uint8Array; + bootstrapArchiveSha256: string; + bootstrapEnvironmentBytes: Uint8Array; + coreRevision: string; + transportMode: HomebrewGuestLifecycleTransportMode; + lazyUrlBase: string; + expectedBootstrapTransportUrl?: string; + expectedEmbeddedBottlePlanBytes?: Uint8Array; + validateEmbeddedBottlePlan?: (plan: HomebrewBottleMirrorPlan) => void; + closedBottleAssets?: readonly ClosedLazyAsset[]; + loadClosedBottleAssets?: ( + embeddedPlanBytes: Uint8Array, + pendingBottleTrees: readonly SerializedLazyArchiveEntry[], + ) => readonly ClosedLazyAsset[]; +} + +const HOMEBREW_COMPOSITION_PATH = "/etc/kandelo/homebrew-vfs.json"; +const SHA256_RE = /^[0-9a-f]{64}$/; +const S_IFMT = 0xf000; +const S_IFREG = 0x8000; +const S_IFDIR = 0x4000; +const S_IFLNK = 0xa000; + +/** + * Bind one lifecycle run to the same image, package tree, catalog, shell, and + * bottle plan on Node and browser. Transport acquisition is host-specific; + * acceptance of the acquired bytes is deliberately shared. + */ +export function deriveHomebrewGuestLifecycleRuntimeInputs( + input: DeriveHomebrewGuestLifecycleRuntimeInputs, +): HomebrewGuestLifecycleRuntimeInputs { + const bootstrapSpec = parsePackageDeferredZipTreeSpec( + parseJson( + input.bootstrapSpecBytes, + "Homebrew bootstrap tree spec", + ), + ); + if ( + !SHA256_RE.test(input.bootstrapArchiveSha256) || + input.bootstrapArchiveBytes.byteLength === 0 + ) { + throw new Error("Homebrew bootstrap archive identity is invalid"); + } + const fs = MemoryFileSystem.fromImage(input.imageBytes); + assertExactBytes( + readVfsFile(fs, "/etc/homebrew/brew.env"), + input.bootstrapEnvironmentBytes, + "main-shell Homebrew environment", + ); + const guestManifest = parseJson( + readVfsFile(fs, HOMEBREW_COMPOSITION_PATH), + HOMEBREW_COMPOSITION_PATH, + ); + assertHomebrewGuestLifecycleCatalog(guestManifest, input.coreRevision); + const shell = resolveHomebrewGuestLifecycleShell(fs); + + const pendingTrees = classifyPendingTrees(fs.exportLazyArchiveEntries()); + if ( + pendingTrees.bootstrap.length !== 1 || + pendingTrees.unclassified.length !== 0 + ) { + throw new Error( + `lifecycle image has ${pendingTrees.bootstrap.length} pending Homebrew ` + + `source trees and ${pendingTrees.unclassified.length} unclassified ` + + `package trees`, + ); + } + assertBootstrapTreeBinding( + fs, + pendingTrees.bootstrap[0]!, + bootstrapSpec, + input.bootstrapArchiveSha256, + input.bootstrapArchiveBytes.byteLength, + ); + + const embeddedPlanBytes = readVfsFile( + fs, + HOMEBREW_BOTTLE_MIRROR_PLAN_VFS_PATH, + ); + if ( + input.expectedEmbeddedBottlePlanBytes !== undefined && + !bytesEqual(embeddedPlanBytes, input.expectedEmbeddedBottlePlanBytes) + ) { + throw new Error( + "live bottle mirror plan differs from the exact VFS-embedded plan", + ); + } + const embeddedPlan = decodeHomebrewBottleMirrorPlan( + embeddedPlanBytes, + HOMEBREW_BOTTLE_MIRROR_PLAN_VFS_PATH, + ); + input.validateEmbeddedBottlePlan?.(embeddedPlan); + if (pendingTrees.bottles.length !== embeddedPlan.assets.length) { + throw new Error( + `lifecycle image has ${pendingTrees.bottles.length} pending bottle ` + + `trees, while its mirror plan declares ${embeddedPlan.assets.length}`, + ); + } + assertPendingTreeHomebrewBottleMirrorBinding( + pendingTrees.bottles, + embeddedPlan, + ); + + const bootstrapTransportUrl = new URL( + bootstrapSpec.archive.url, + input.lazyUrlBase, + ).toString(); + if ( + input.expectedBootstrapTransportUrl !== undefined && + bootstrapTransportUrl !== input.expectedBootstrapTransportUrl + ) { + throw new Error( + `Homebrew bootstrap transport resolves to ${bootstrapTransportUrl}, ` + + `expected ${input.expectedBootstrapTransportUrl}`, + ); + } + + const lazyAssets = bindClosedLifecycleAssets( + input, + embeddedPlan, + embeddedPlanBytes, + pendingTrees.bottles, + bootstrapTransportUrl, + input.bootstrapArchiveSha256, + input.bootstrapArchiveBytes.byteLength, + ); + + return { + imageBytes: input.imageBytes, + shellBytes: shell.bytes, + shellArgv0: shell.argv0, + lazyUrlBase: input.lazyUrlBase, + ...(lazyAssets === undefined ? {} : { lazyAssets }), + bootstrapTransportUrl, + bootstrapBytes: input.bootstrapArchiveBytes.byteLength, + }; +} + +function assertBootstrapTreeBinding( + fs: MemoryFileSystem, + tree: SerializedLazyArchiveEntry, + spec: PackageDeferredZipTreeSpec, + archiveSha256: string, + archiveBytes: number, +): void { + const content = tree.content; + const inventory = tree.inventory; + const inventoryBytes = inventory?.reduce( + (total, entry) => total + entry.size, + 0, + ); + if ( + spec.content_role !== "source-tree" || + ( + tree.kind !== "kandelo-deferred-tree-v1" && + tree.kind !== "kandelo-deferred-tree-v2" + ) || + tree.materialized || + content === undefined || + inventory === undefined || + inventory.length === 0 || + tree.mountPrefix !== spec.mount_prefix || + tree.url !== spec.archive.url || + content.decoder !== "zip-v1" || + content.mediaType !== "application/zip" || + content.sha256 !== archiveSha256 || + content.bytes !== archiveBytes || + content.expandedBytes !== inventoryBytes || + content.sourceEntryCount !== inventory.length || + content.transports.length !== 1 || + content.transports[0] !== spec.archive.url || + content.modePolicy !== spec.archive.mode_policy || + content.source !== undefined || + JSON.stringify(tree.activation) !== JSON.stringify(spec.activation) + ) { + throw new Error( + `Homebrew bootstrap deferred tree ${spec.id} changed descriptor: ` + + `${JSON.stringify({ + kind: tree.kind, + materialized: tree.materialized, + mountPrefix: tree.mountPrefix, + url: tree.url, + content, + inventoryBytes, + inventoryLength: inventory?.length, + activation: tree.activation, + })}`, + ); + } + + for (const entry of inventory) { + if (entry.type === "hardlink") { + throw new Error( + `Homebrew bootstrap ZIP tree ${spec.id} contains a hardlink inventory entry`, + ); + } + const stat = fs.lstat(entry.vfsPath); + const expectedType = entry.type === "directory" + ? S_IFDIR + : entry.type === "symlink" + ? S_IFLNK + : S_IFREG; + if ( + (stat.mode & S_IFMT) !== expectedType || + (stat.mode & 0o7777) !== entry.mode || + stat.uid !== spec.owner.uid || + stat.gid !== spec.owner.gid || + (entry.type !== "directory" && stat.size !== entry.size) || + (entry.type === "file" && !fs.isPathDeferred(entry.vfsPath)) || + ( + entry.type === "symlink" && + fs.readlink(entry.vfsPath) !== entry.target + ) + ) { + throw new Error( + `Homebrew bootstrap deferred tree ${spec.id} changed ${entry.vfsPath}`, + ); + } + } +} + +function bindClosedLifecycleAssets( + input: DeriveHomebrewGuestLifecycleRuntimeInputs, + plan: HomebrewBottleMirrorPlan, + embeddedPlanBytes: Uint8Array, + pendingBottleTrees: readonly SerializedLazyArchiveEntry[], + bootstrapTransportUrl: string, + bootstrapSha256: string, + bootstrapBytes: number, +): readonly ClosedLazyAsset[] | undefined { + if (input.transportMode === "public") { + if ( + input.closedBottleAssets !== undefined || + input.loadClosedBottleAssets !== undefined + ) { + throw new Error("public lifecycle transport cannot carry closed bottle bytes"); + } + return undefined; + } + if ( + input.closedBottleAssets !== undefined && + input.loadClosedBottleAssets !== undefined + ) { + throw new Error( + "closed lifecycle transport must have exactly one bottle-byte source", + ); + } + const closedBottleAssets = input.closedBottleAssets ?? + input.loadClosedBottleAssets?.(embeddedPlanBytes, pendingBottleTrees); + if (closedBottleAssets === undefined) { + throw new Error("closed lifecycle transport requires exact bottle bytes"); + } + assertClosedBottleAssets(closedBottleAssets, plan); + return [ + ...closedBottleAssets, + { + url: bootstrapTransportUrl, + sha256: bootstrapSha256, + size: bootstrapBytes, + bytes: input.bootstrapArchiveBytes, + }, + ]; +} + +function assertClosedBottleAssets( + assets: readonly ClosedLazyAsset[], + plan: HomebrewBottleMirrorPlan, +): void { + if (assets.length !== plan.assets.length) { + throw new Error( + `closed bottle binding count ${assets.length} differs from mirror ` + + `asset count ${plan.assets.length}`, + ); + } + const byUrl = new Map(assets.map((asset) => [asset.url, asset])); + if (byUrl.size !== assets.length) { + throw new Error("closed bottle bindings duplicate a release URL"); + } + for (const expected of plan.assets) { + const actual = byUrl.get(expected.url); + if ( + actual === undefined || + actual.sha256 !== expected.sha256 || + actual.size !== expected.bytes || + actual.bytes.byteLength !== expected.bytes + ) { + throw new Error( + `closed bottle binding does not match mirror asset ${expected.package}`, + ); + } + } +} + +function classifyPendingTrees(entries: readonly SerializedLazyArchiveEntry[]): { + bottles: SerializedLazyArchiveEntry[]; + bootstrap: SerializedLazyArchiveEntry[]; + unclassified: SerializedLazyArchiveEntry[]; +} { + const pending = entries.filter((tree) => tree.content !== undefined); + for (const tree of pending) { + const capabilities = tree.activation?.capabilities ?? []; + const bottleCapabilities = capabilities.filter((capability) => + capability.startsWith("homebrew-bottle:") + ); + if ( + bottleCapabilities.length > 1 || + ( + bottleCapabilities.length === 1 && + capabilities.includes("homebrew:bootstrap") + ) + ) { + throw new Error( + `pending tree ${tree.mountPrefix} has ambiguous Homebrew ownership`, + ); + } + } + const bottles = pending.filter((tree) => + tree.activation?.capabilities.some((capability) => + capability.startsWith("homebrew-bottle:") + ) + ); + const bootstrap = pending.filter((tree) => + tree.activation?.capabilities.includes("homebrew:bootstrap") + ); + const unclassified = pending.filter( + (tree) => !bottles.includes(tree) && !bootstrap.includes(tree), + ); + return { bottles, bootstrap, unclassified }; +} + +function readVfsFile(fs: MemoryFileSystem, path: string): Uint8Array { + const stat = fs.stat(path); + if ((stat.mode & 0xf000) !== 0x8000) { + throw new Error(`${path} is not a regular file`); + } + const bytes = new Uint8Array(stat.size); + const fd = fs.open(path, 0, 0); + try { + let offset = 0; + while (offset < bytes.byteLength) { + const count = fs.read( + fd, + bytes.subarray(offset), + null, + bytes.byteLength - offset, + ); + if (count <= 0) { + throw new Error(`${path} ended after ${offset}/${bytes.byteLength} bytes`); + } + offset += count; + } + } finally { + fs.close(fd); + } + return bytes; +} + +function parseJson(bytes: Uint8Array, label: string): unknown { + try { + return JSON.parse( + new TextDecoder("utf-8", { fatal: true }).decode(bytes), + ); + } catch (error) { + throw new Error(`${label} is not valid UTF-8 JSON: ${String(error)}`); + } +} + +function assertExactBytes( + actual: Uint8Array, + expected: Uint8Array, + label: string, +): void { + if ( + actual.byteLength !== expected.byteLength || + !actual.every((byte, index) => byte === expected[index]) + ) { + throw new Error(`${label} differs from the resolved package output`); + } +} diff --git a/host/src/browser-kernel-host.ts b/host/src/browser-kernel-host.ts index 5edb12b9ef..ca012b4a4f 100644 --- a/host/src/browser-kernel-host.ts +++ b/host/src/browser-kernel-host.ts @@ -126,6 +126,8 @@ export interface BrowserKernelBootOptions { export class BrowserKernel { private kernelWorkerHandle!: Worker; + private workerStarted = false; + private initialized = false; /** POSIX shared-memory / semaphore SAB shared with the kernel worker. Small * and fixed (1 MiB); the live VFS is owned by the worker, not here. */ private shmSab: SharedArrayBuffer; @@ -243,6 +245,7 @@ export class BrowserKernel { : snapshotClosedLazyAssets(opts.closedLazyAssets); // Create the kernel worker this.kernelWorkerHandle = new Worker(kernelWorkerEntryUrl, { type: "module" }); + this.workerStarted = true; this.kernelWorkerHandle.onmessage = (e: MessageEvent) => { this.handleWorkerMessage(e.data as KernelToMainMessage); @@ -335,6 +338,7 @@ export class BrowserKernel { } this.kernelWorkerHandle.postMessage(initMsg, transfer); }); + this.initialized = true; } /** @@ -949,14 +953,39 @@ export class BrowserKernel { return result === true; } - /** Destroy the kernel and release all resources. */ - async destroy(): Promise { + /** + * Serialize the quiescent worker-owned root filesystem for a later boot. + * The root image is durable; boot-scoped scratch and device mounts are not. + * Callers must wait for every guest process to exit before invoking this. + */ + async exportRootfsImage(): Promise { + if (!this.initialized) { + throw new Error("rootfs export requires an initialized kernel"); + } const requestId = this.nextRequestId++; - await this.request(requestId, { - type: "destroy", + const result = await this.request(requestId, { + type: "export_rootfs_image", requestId, }); + if (!(result instanceof Uint8Array)) { + throw new Error("kernel worker returned an invalid rootfs image"); + } + return result; + } + + /** Destroy the kernel and release all resources. */ + async destroy(): Promise { + if (!this.workerStarted) return; + if (this.initialized) { + const requestId = this.nextRequestId++; + await this.request(requestId, { + type: "destroy", + requestId, + }); + } + this.initialized = false; this.kernelWorkerHandle.terminate(); + this.workerStarted = false; this.exitResolvers.clear(); this.unclaimedExitStatuses.clear(); this.pendingRequests.clear(); diff --git a/host/src/browser-kernel-protocol.ts b/host/src/browser-kernel-protocol.ts index 1b7c26ab72..39a813febf 100644 --- a/host/src/browser-kernel-protocol.ts +++ b/host/src/browser-kernel-protocol.ts @@ -116,6 +116,17 @@ export interface UnlinkVfsFileMessage { path: string; } +/** + * Serialize the quiescent worker-owned root filesystem. + * + * This deliberately captures only the `/` image backend. Scratch and device + * mounts are boot-scoped and are recreated by the host on the next boot. + */ +export interface ExportRootfsImageMessage { + type: "export_rootfs_image"; + requestId: number; +} + export interface AppendStdinDataMessage { type: "append_stdin_data"; pid: number; @@ -345,6 +356,7 @@ export type MainToKernelMessage = | ReadVfsFileMessage | WriteVfsFileMessage | UnlinkVfsFileMessage + | ExportRootfsImageMessage | AppendStdinDataMessage | SetStdinDataMessage | PtyWriteMessage diff --git a/host/src/browser-kernel-worker-entry.ts b/host/src/browser-kernel-worker-entry.ts index 96f268ba7e..7b9746b089 100644 --- a/host/src/browser-kernel-worker-entry.ts +++ b/host/src/browser-kernel-worker-entry.ts @@ -55,6 +55,7 @@ import { threadWorkerFailureDisposition, } from "./thread-worker-disposition"; import { VmInterruptTimerManager } from "./vm-interrupt-timer"; +import { RootfsSnapshotGate } from "./rootfs-snapshot-gate"; import type { CentralizedWorkerInitMessage, CentralizedThreadInitMessage, @@ -96,6 +97,7 @@ type LazyRegistrationMessage = Extract< let initReady = false; let initFailure: string | null = null; const pendingLazyRegistrationMessages: LazyRegistrationMessage[] = []; +const rootfsSnapshotGate = new RootfsSnapshotGate(); // Process tracking interface ForkReplayContext { @@ -358,6 +360,13 @@ function respond(requestId: number, result: unknown) { post({ type: "response", requestId, result }); } +function respondTransferredBytes(requestId: number, result: Uint8Array) { + post( + { type: "response", requestId, result }, + [result.buffer as ArrayBuffer], + ); +} + function respondError(requestId: number, error: string) { post({ type: "response", requestId, result: null, error }); } @@ -423,12 +432,18 @@ function handleLazyRegistration(msg: LazyRegistrationMessage): void { pendingLazyRegistrationMessages.push(msg); return; } + let releaseMutation: (() => void) | undefined; try { + releaseMutation = rootfsSnapshotGate.beginMutation( + "register lazy rootfs entries", + ); applyLazyRegistration(msg); } catch (err) { const error = formatError(err); respondErrorIfRequested(msg, error); reportWorkerProtocolError(`${msg.type} failed: ${error}`); + } finally { + releaseMutation?.(); } } @@ -778,8 +793,10 @@ async function handleInit(msg: Extract) { // ── Spawn ── async function handleSpawn(msg: Extract) { + let releaseMutation: (() => void) | undefined; let createdPid: number | undefined; try { + releaseMutation = rootfsSnapshotGate.beginMutation("spawn a process"); await waitForProcessTeardowns(); let programBytes: ArrayBuffer; @@ -888,6 +905,8 @@ async function handleSpawn(msg: Extract) kernelWorker.removeProcessFromKernelTable(createdPid); } respondError(msg.requestId, String(e)); + } finally { + releaseMutation?.(); } } @@ -1716,7 +1735,13 @@ async function handleReadVfsFile( msg: Extract, ) { if (!io) { respond(msg.requestId, null); return; } + let releaseMutation: (() => void) | undefined; try { + // A read can materialize a lazy file/tree and is therefore serialized + // with snapshots even though an already-materialized read is non-mutating. + releaseMutation = rootfsSnapshotGate.beginMutation( + "read or materialize a rootfs file", + ); const { data, stat } = await readPreparedPlatformFile(io, msg.path); // Copy into a plain (non-shared) ArrayBuffer so it structured-clones back. const result = data.slice(); @@ -1727,6 +1752,8 @@ async function handleReadVfsFile( } catch (error) { if (isMissingPathError(error)) respond(msg.requestId, null); else respondError(msg.requestId, formatError(error)); + } finally { + releaseMutation?.(); } } @@ -1735,8 +1762,10 @@ async function handleReadVfsFile( // stage transient files between process spawns. function handleWriteVfsFile(msg: Extract) { if (!io) { respondError(msg.requestId, "VFS is not initialized"); return; } + let releaseMutation: (() => void) | undefined; let fd: number | null = null; try { + releaseMutation = rootfsSnapshotGate.beginMutation("write a rootfs file"); fd = io.open(msg.path, 0o1101 /* O_WRONLY|O_CREAT|O_TRUNC */, msg.mode & 0o7777); let offset = 0; while (offset < msg.data.byteLength) { @@ -1762,12 +1791,16 @@ function handleWriteVfsFile(msg: Extract) { if (!io) { respondError(msg.requestId, "VFS is not initialized"); return; } + let releaseMutation: (() => void) | undefined; try { + releaseMutation = rootfsSnapshotGate.beginMutation("unlink a rootfs file"); try { io.lstat(msg.path); } catch { @@ -1778,6 +1811,38 @@ function handleUnlinkVfsFile(msg: Extract, +) { + if (!memfs) { + respondError(msg.requestId, "VFS is not initialized"); + return; + } + if (!initReady) { + respondError(msg.requestId, "rootfs export requires an initialized kernel"); + return; + } + try { + const image = await rootfsSnapshotGate.runSnapshot(async () => { + if ( + processes.size !== 0 || + processTeardowns.size !== 0 || + workerTeardowns.size !== 0 + ) { + throw new Error( + "rootfs export requires a quiescent kernel with no live or tearing-down processes", + ); + } + return memfs.saveImage(); + }); + respondTransferredBytes(msg.requestId, image); + } catch (error) { + respondError(msg.requestId, formatError(error)); } } @@ -2213,6 +2278,7 @@ sw.onmessage = (e: MessageEvent) => { case "read_vfs_file": void handleReadVfsFile(msg); break; case "write_vfs_file": handleWriteVfsFile(msg); break; case "unlink_vfs_file": handleUnlinkVfsFile(msg); break; + case "export_rootfs_image": void handleExportRootfsImage(msg); break; case "append_stdin_data": kernelWorker.appendStdinData(msg.pid, msg.data); break; case "set_stdin_data": kernelWorker.setStdinData(msg.pid, msg.data); break; case "pty_write": handlePtyWrite(msg); break; diff --git a/host/src/node-kernel-host.ts b/host/src/node-kernel-host.ts index 11eb216872..f09a3eca43 100644 --- a/host/src/node-kernel-host.ts +++ b/host/src/node-kernel-host.ts @@ -147,6 +147,8 @@ export interface SpawnOptions { export class NodeKernelHost { private worker!: NodeThreadWorker; + private workerStarted = false; + private initialized = false; private pendingRequests = new Map void; reject: (err: Error) => void }>(); private exitResolvers = new Map void>(); private unclaimedExitStatuses = new Map(); @@ -177,6 +179,7 @@ export class NodeKernelHost { : snapshotClosedLazyAssets(this.options.rootfsLazyAssets); this.worker = spawnKernelWorkerThread(); + this.workerStarted = true; this.worker.on("message", (msg: KernelToMainMessage) => { this.handleWorkerMessage(msg); @@ -254,6 +257,7 @@ export class NodeKernelHost { ); this.worker.postMessage(initMsg, transfer); }); + this.initialized = true; } /** @@ -530,23 +534,48 @@ export class NodeKernelHost { }; } + /** + * Serialize the quiescent worker-owned root filesystem for a later boot. + * The root image is durable; boot-scoped scratch and device mounts are not. + * Callers must wait for every guest process to exit before invoking this. + */ + async exportRootfsImage(): Promise { + if (!this.initialized) { + throw new Error("rootfs export requires an initialized kernel"); + } + const requestId = this._nextRequestId++; + const result = await this.request(requestId, { + type: "export_rootfs_image", + requestId, + }); + if (!(result instanceof Uint8Array)) { + throw new Error("kernel worker returned an invalid rootfs image"); + } + return result; + } + /** Destroy the kernel and release all resources */ async destroy(): Promise { - const requestId = this._nextRequestId++; - let timeoutId: ReturnType | undefined; - try { - await Promise.race([ - this.request(requestId, { type: "destroy", requestId }), - new Promise((resolve) => { - timeoutId = setTimeout(resolve, DESTROY_REQUEST_TIMEOUT_MS); - }), - ]); - } catch { - // Worker may have already exited - } finally { - if (timeoutId !== undefined) clearTimeout(timeoutId); + if (!this.workerStarted) return; + if (this.initialized) { + const requestId = this._nextRequestId++; + let timeoutId: ReturnType | undefined; + try { + await Promise.race([ + this.request(requestId, { type: "destroy", requestId }), + new Promise((resolve) => { + timeoutId = setTimeout(resolve, DESTROY_REQUEST_TIMEOUT_MS); + }), + ]); + } catch { + // Worker may have already exited + } finally { + if (timeoutId !== undefined) clearTimeout(timeoutId); + } } - await this.worker.terminate(); + await this.worker.terminate().catch(() => {}); + this.workerStarted = false; + this.initialized = false; this.exitResolvers.clear(); this.pendingRequests.clear(); this.lazyDownloadListeners.clear(); diff --git a/host/src/node-kernel-protocol.ts b/host/src/node-kernel-protocol.ts index c23c5896ef..ac17bac221 100644 --- a/host/src/node-kernel-protocol.ts +++ b/host/src/node-kernel-protocol.ts @@ -118,6 +118,15 @@ export interface DestroyMessage { requestId: number; } +/** + * Serialize the quiescent worker-owned root filesystem. Boot-scoped scratch + * and device mounts are intentionally outside this root-image snapshot. + */ +export interface ExportRootfsImageMessage { + type: "export_rootfs_image"; + requestId: number; +} + /** Request the kernel's per-process fork counter. The kernel-worker entry * forwards this to `kernel_get_fork_count` and posts a `response` message * with `result` set to a `bigint` (u64 as BigInt). Used by the spawn @@ -209,6 +218,7 @@ export type MainToKernelMessage = | PtyResizeMessage | TerminateProcessMessage | DestroyMessage + | ExportRootfsImageMessage | GetForkCountRequestMessage | GetKernelMemoryPagesRequestMessage | ResolveExecResponseMessage diff --git a/host/src/node-kernel-worker-entry.ts b/host/src/node-kernel-worker-entry.ts index 45adced395..ba822b8f36 100644 --- a/host/src/node-kernel-worker-entry.ts +++ b/host/src/node-kernel-worker-entry.ts @@ -66,6 +66,7 @@ import { threadWorkerFailureDisposition, } from "./thread-worker-disposition"; import { VmInterruptTimerManager } from "./vm-interrupt-timer"; +import { RootfsSnapshotGate } from "./rootfs-snapshot-gate"; import { computeProcessMemoryLayout, createProcessMemory, @@ -104,6 +105,7 @@ let defaultThreadSlots: number = DEFAULT_PROCESS_THREAD_SLOTS; let execPrograms: Record = {}; let vfsExecIO: PlatformIO | null = null; let rootfsMemfs: MemoryFileSystem | null = null; +let initReady = false; /** Per-boot scratch directory; cleaned up on `destroy`. Only set when the * worker constructs a `VirtualPlatformIO` from the default mount spec. */ let sessionDir: string | null = null; @@ -137,6 +139,7 @@ const vmInterruptTimers = new VmInterruptTimerManager( (pid) => processes.get(pid), ); const reportedExits = new Set(); +const rootfsSnapshotGate = new RootfsSnapshotGate(); // Workers terminated by the kernel-worker entry itself (handleExit / // handleExec / handleTerminate). The crash safety-net listener checks @@ -375,6 +378,13 @@ function respond(requestId: number, result: unknown) { post({ type: "response", requestId, result }); } +function respondTransferredBytes(requestId: number, result: Uint8Array) { + port.postMessage( + { type: "response", requestId, result } satisfies KernelToMainMessage, + [result.buffer as ArrayBuffer], + ); +} + function respondError(requestId: number, error: string) { post({ type: "response", requestId, result: null, error }); } @@ -627,6 +637,7 @@ function cleanupSessionDir(): void { } async function handleInit(msg: InitMessage) { + initReady = false; maxPages = msg.config.maxPages ?? DEFAULT_MAX_PAGES; defaultThreadSlots = msg.config.defaultThreadSlots ?? DEFAULT_PROCESS_THREAD_SLOTS; execPrograms = msg.execPrograms ?? {}; @@ -699,14 +710,17 @@ async function handleInit(msg: InitMessage) { await kernelWorker.init(msg.kernelWasmBytes); + initReady = true; post({ type: "ready" }); } // --- Spawn --- function handleSpawn(msg: SpawnMessage) { + let releaseMutation: (() => void) | undefined; let createdPid: number | undefined; try { + releaseMutation = rootfsSnapshotGate.beginMutation("spawn a process"); if (!isWasmModuleBytes(msg.programBytes)) { respondError(msg.requestId, "ENOEXEC: program is not a WebAssembly module"); return; @@ -820,6 +834,8 @@ function handleSpawn(msg: SpawnMessage) { kernelWorker.removeProcessFromKernelTable(createdPid); } respondError(msg.requestId, String(e)); + } finally { + releaseMutation?.(); } } @@ -1647,6 +1663,35 @@ async function handleHttpRequest(msg: HttpRequestMessage) { } } +async function handleExportRootfsImage( + msg: Extract, +) { + if (!rootfsMemfs) { + respondError(msg.requestId, "rootfs export requires a VFS-backed kernel"); + return; + } + if (!initReady) { + respondError(msg.requestId, "rootfs export requires an initialized kernel"); + return; + } + try { + const image = await rootfsSnapshotGate.runSnapshot(async () => { + if (processes.size !== 0 || processTeardowns.size !== 0) { + throw new Error( + "rootfs export requires a quiescent kernel with no live or tearing-down processes", + ); + } + return rootfsMemfs!.saveImage(); + }); + respondTransferredBytes(msg.requestId, image); + } catch (error) { + respondError( + msg.requestId, + error instanceof Error ? error.message : String(error), + ); + } +} + // --- Message dispatch --- port.on("message", (msg: MainToKernelMessage) => { @@ -1675,6 +1720,9 @@ port.on("message", (msg: MainToKernelMessage) => { case "destroy": void handleDestroy(msg); break; + case "export_rootfs_image": + void handleExportRootfsImage(msg); + break; case "get_fork_count": { // Round-trip access to the kernel's per-process fork counter for // tests asserting SYS_SPAWN didn't fall back to fork. Result is a diff --git a/host/src/rootfs-snapshot-gate.ts b/host/src/rootfs-snapshot-gate.ts new file mode 100644 index 0000000000..8d819feae5 --- /dev/null +++ b/host/src/rootfs-snapshot-gate.ts @@ -0,0 +1,60 @@ +/** + * Worker-owned serialization for rootfs snapshots. + * + * Worker message handlers may overlap whenever an async handler yields. A + * rootfs snapshot must therefore close the gate synchronously, wait for + * already-started mutations, and keep later mutations out until saveImage() + * has either completed or failed. + */ +export class RootfsSnapshotGate { + private snapshotActive = false; + private activeMutations = 0; + private mutationDrainWaiters: Array<() => void> = []; + + /** + * Enter a rootfs-affecting operation. The returned release callback must be + * called exactly once, normally from a finally block. + */ + beginMutation(operation: string): () => void { + if (this.snapshotActive) { + throw new Error(`rootfs export is in progress; cannot ${operation}`); + } + this.activeMutations += 1; + let released = false; + return () => { + if (released) { + throw new Error(`rootfs snapshot mutation released twice: ${operation}`); + } + released = true; + this.activeMutations -= 1; + if (this.activeMutations === 0) { + const waiters = this.mutationDrainWaiters.splice(0); + for (const resolve of waiters) resolve(); + } + }; + } + + /** + * Run one atomic rootfs snapshot. + * + * The gate closes before the first await, so messages delivered after this + * call cannot start a process or mutate/materialize rootfs state. Mutations + * that entered first are allowed to finish and are included in the image. + */ + async runSnapshot(snapshot: () => Promise): Promise { + if (this.snapshotActive) { + throw new Error("rootfs export is already in progress"); + } + this.snapshotActive = true; + try { + if (this.activeMutations !== 0) { + await new Promise((resolve) => { + this.mutationDrainWaiters.push(resolve); + }); + } + return await snapshot(); + } finally { + this.snapshotActive = false; + } + } +} diff --git a/host/src/vfs/package-deferred-tree-contract.ts b/host/src/vfs/package-deferred-tree-contract.ts new file mode 100644 index 0000000000..4d38f56a58 --- /dev/null +++ b/host/src/vfs/package-deferred-tree-contract.ts @@ -0,0 +1,215 @@ +import type { LazyTreeActivation } from "./memory-fs"; +import { VFS_DEFERRED_TREE_LIMITS } from "./deferred-tree-limits"; + +const MAX_OWNER_ID = 0xffff_fffe; +const PACKAGE_NAME_RE = /^[a-z0-9][a-z0-9+._-]*$/; +const OUTPUT_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9+._-]*$/; +const TREE_ID_RE = /^[a-z0-9][a-z0-9+._/-]*$/; +const textEncoder = new TextEncoder(); + +export interface PackageDeferredZipTreeSpec { + schema: 1; + kind: "kandelo-package-deferred-zip-tree"; + id: string; + /** Distribution meaning; source trees are not package/bottle payloads. */ + content_role: "source-tree" | "runtime-tree"; + package: { + name: string; + output: string; + }; + archive: { + url: string; + mode_policy: "portable-posix-v1"; + }; + mount_prefix: string; + owner: { + uid: number; + gid: number; + }; + activation: LazyTreeActivation; +} + +/** Parse the reviewable recipe without trusting unknown fields or host paths. */ +export function parsePackageDeferredZipTreeSpec( + value: unknown, +): PackageDeferredZipTreeSpec { + const record = exactRecord(value, [ + "schema", + "kind", + "id", + "content_role", + "package", + "archive", + "mount_prefix", + "owner", + "activation", + ], "package deferred ZIP tree spec"); + const packageRecord = exactRecord( + record.package, + ["name", "output"], + "package deferred ZIP tree package", + ); + const archive = exactRecord( + record.archive, + ["url", "mode_policy"], + "package deferred ZIP tree archive", + ); + const owner = exactRecord( + record.owner, + ["uid", "gid"], + "package deferred ZIP tree owner", + ); + const activation = exactRecord( + record.activation, + ["mode", "capabilities", "roots"], + "package deferred ZIP tree activation", + ); + if ( + record.schema !== 1 || + record.kind !== "kandelo-package-deferred-zip-tree" || + typeof record.id !== "string" || !TREE_ID_RE.test(record.id) || + utf8Length(record.id) > 255 || + record.id.includes("//") || record.id.endsWith("/") || + ( + record.content_role !== "source-tree" && + record.content_role !== "runtime-tree" + ) || + typeof packageRecord.name !== "string" || + !PACKAGE_NAME_RE.test(packageRecord.name) || + utf8Length(packageRecord.name) > 255 || + typeof packageRecord.output !== "string" || + !OUTPUT_NAME_RE.test(packageRecord.output) || + utf8Length(packageRecord.output) > 255 || + typeof archive.url !== "string" || + !isRelativeAssetUrl(archive.url) || + archive.url !== packageRecord.output || + archive.mode_policy !== "portable-posix-v1" || + typeof record.mount_prefix !== "string" || + utf8Length(record.mount_prefix) > + VFS_DEFERRED_TREE_LIMITS.maxPathBytes || + canonicalAbsolutePath(record.mount_prefix, true) !== record.mount_prefix || + !isOwnerId(owner.uid) || + !isOwnerId(owner.gid) || + ( + activation.mode !== "first-use" && + activation.mode !== "boot-prefetch" + ) || + !Array.isArray(activation.capabilities) || + activation.capabilities.length === 0 || + activation.capabilities.length > + VFS_DEFERRED_TREE_LIMITS.maxActivationCapabilities || + !activation.capabilities.every((capability) => + typeof capability === "string" && + utf8Length(capability) <= + VFS_DEFERRED_TREE_LIMITS.maxActivationCapabilityBytes && + /^[a-z0-9][a-z0-9:._-]*$/.test(capability) + ) || + new Set(activation.capabilities).size !== + activation.capabilities.length || + !Array.isArray(activation.roots) || + activation.roots.length === 0 || + activation.roots.length > + VFS_DEFERRED_TREE_LIMITS.maxActivationRoots || + !activation.roots.every((root) => + typeof root === "string" && + utf8Length(root) <= VFS_DEFERRED_TREE_LIMITS.maxPathBytes && + canonicalAbsolutePath(root, true) === root + ) || + new Set(activation.roots).size !== activation.roots.length + ) { + throw new Error("package deferred ZIP tree spec is invalid"); + } + for (const root of activation.roots as string[]) { + if ( + record.mount_prefix !== "/" && + root !== record.mount_prefix && + !root.startsWith(`${record.mount_prefix}/`) + ) { + throw new Error( + `package deferred ZIP tree activation root escapes its mount: ${root}`, + ); + } + } + return { + schema: 1, + kind: "kandelo-package-deferred-zip-tree", + id: record.id, + content_role: record.content_role, + package: { + name: packageRecord.name, + output: packageRecord.output, + }, + archive: { + url: archive.url, + mode_policy: "portable-posix-v1", + }, + mount_prefix: record.mount_prefix, + owner: { uid: owner.uid, gid: owner.gid }, + activation: { + mode: activation.mode, + capabilities: [...activation.capabilities] as string[], + roots: [...activation.roots] as string[], + }, + }; +} + +function exactRecord( + value: unknown, + fields: readonly string[], + label: string, +): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error(`${label} must be an object`); + } + const record = value as Record; + if ( + JSON.stringify(Object.keys(record).sort()) !== + JSON.stringify([...fields].sort()) + ) { + throw new Error(`${label} has unsupported fields`); + } + return record; +} + +function canonicalAbsolutePath( + value: string, + allowRoot: boolean, +): string | null { + if (allowRoot && value === "/") return value; + if ( + !value.startsWith("/") || + value.endsWith("/") || + value.includes("\\") || + value.includes("\0") || + value.slice(1).split("/").some((segment) => + segment === "" || segment === "." || segment === ".." + ) + ) { + return null; + } + return value; +} + +function isRelativeAssetUrl(value: string): boolean { + return ( + value.length > 0 && + value.length <= 255 && + !value.startsWith("/") && + !value.includes("\\") && + !value.includes("\0") && + !/^[a-z][a-z0-9+.-]*:/i.test(value) && + value.split("/").every((segment) => + segment !== "" && segment !== "." && segment !== ".." + ) + ); +} + +function isOwnerId(value: unknown): value is number { + return Number.isSafeInteger(value) && + (value as number) >= 0 && + (value as number) <= MAX_OWNER_ID; +} + +function utf8Length(value: string): number { + return textEncoder.encode(value).byteLength; +} diff --git a/host/src/vfs/package-deferred-tree.ts b/host/src/vfs/package-deferred-tree.ts index 45a546d124..49794a47c9 100644 --- a/host/src/vfs/package-deferred-tree.ts +++ b/host/src/vfs/package-deferred-tree.ts @@ -13,42 +13,22 @@ import { type ZipEntry, } from "./zip"; import { VFS_DEFERRED_TREE_LIMITS } from "./deferred-tree-limits"; +import { + parsePackageDeferredZipTreeSpec, + type PackageDeferredZipTreeSpec, +} from "./package-deferred-tree-contract"; import { ENOENT, SFSError } from "./sharedfs-vendor"; +export { + parsePackageDeferredZipTreeSpec, + type PackageDeferredZipTreeSpec, +} from "./package-deferred-tree-contract"; const S_IFMT = 0xf000; const S_IFREG = 0x8000; const S_IFDIR = 0x4000; const S_IFLNK = 0xa000; -// uid_t/gid_t -1 is the POSIX chown "leave unchanged" sentinel and therefore -// cannot truthfully identify the owner declared by a package descriptor. -const MAX_OWNER_ID = 0xffff_fffe; -const PACKAGE_NAME_RE = /^[a-z0-9][a-z0-9+._-]*$/; -const OUTPUT_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9+._-]*$/; -const TREE_ID_RE = /^[a-z0-9][a-z0-9+._/-]*$/; const textEncoder = new TextEncoder(); -export interface PackageDeferredZipTreeSpec { - schema: 1; - kind: "kandelo-package-deferred-zip-tree"; - id: string; - /** Distribution meaning; source trees are not package/bottle payloads. */ - content_role: "source-tree" | "runtime-tree"; - package: { - name: string; - output: string; - }; - archive: { - url: string; - mode_policy: "portable-posix-v1"; - }; - mount_prefix: string; - owner: { - uid: number; - gid: number; - }; - activation: LazyTreeActivation; -} - export interface PackageDeferredZipTreeDescriptor { schema: 1; kind: "kandelo-package-deferred-zip-tree"; @@ -90,112 +70,6 @@ export interface RegisteredPackageDeferredZipTree extends materialization: DeferredTreeMaterializationHandle; } -/** Parse the reviewable recipe without trusting unknown fields or host paths. */ -export function parsePackageDeferredZipTreeSpec( - value: unknown, -): PackageDeferredZipTreeSpec { - const record = exactRecord(value, [ - "schema", - "kind", - "id", - "content_role", - "package", - "archive", - "mount_prefix", - "owner", - "activation", - ], "package deferred ZIP tree spec"); - const packageRecord = exactRecord( - record.package, - ["name", "output"], - "package deferred ZIP tree package", - ); - const archive = exactRecord( - record.archive, - ["url", "mode_policy"], - "package deferred ZIP tree archive", - ); - const owner = exactRecord( - record.owner, - ["uid", "gid"], - "package deferred ZIP tree owner", - ); - const activation = exactRecord( - record.activation, - ["mode", "capabilities", "roots"], - "package deferred ZIP tree activation", - ); - if ( - record.schema !== 1 || - record.kind !== "kandelo-package-deferred-zip-tree" || - typeof record.id !== "string" || !TREE_ID_RE.test(record.id) || - utf8Length(record.id) > 255 || - record.id.includes("//") || record.id.endsWith("/") || - (record.content_role !== "source-tree" && record.content_role !== "runtime-tree") || - typeof packageRecord.name !== "string" || - !PACKAGE_NAME_RE.test(packageRecord.name) || utf8Length(packageRecord.name) > 255 || - typeof packageRecord.output !== "string" || - !OUTPUT_NAME_RE.test(packageRecord.output) || utf8Length(packageRecord.output) > 255 || - typeof archive.url !== "string" || !isRelativeAssetUrl(archive.url) || - archive.url !== packageRecord.output || - archive.mode_policy !== "portable-posix-v1" || - typeof record.mount_prefix !== "string" || - utf8Length(record.mount_prefix) > VFS_DEFERRED_TREE_LIMITS.maxPathBytes || - canonicalAbsolutePath(record.mount_prefix, true) !== record.mount_prefix || - !isOwnerId(owner.uid) || !isOwnerId(owner.gid) || - (activation.mode !== "first-use" && activation.mode !== "boot-prefetch") || - !Array.isArray(activation.capabilities) || activation.capabilities.length === 0 || - activation.capabilities.length > - VFS_DEFERRED_TREE_LIMITS.maxActivationCapabilities || - !activation.capabilities.every((capability) => - typeof capability === "string" && - utf8Length(capability) <= VFS_DEFERRED_TREE_LIMITS.maxActivationCapabilityBytes && - /^[a-z0-9][a-z0-9:._-]*$/.test(capability) - ) || - new Set(activation.capabilities).size !== activation.capabilities.length || - !Array.isArray(activation.roots) || activation.roots.length === 0 || - activation.roots.length > VFS_DEFERRED_TREE_LIMITS.maxActivationRoots || - !activation.roots.every((root) => - typeof root === "string" && - utf8Length(root) <= VFS_DEFERRED_TREE_LIMITS.maxPathBytes && - canonicalAbsolutePath(root, true) === root - ) || - new Set(activation.roots).size !== activation.roots.length - ) { - throw new Error("package deferred ZIP tree spec is invalid"); - } - for (const root of activation.roots as string[]) { - if ( - record.mount_prefix !== "/" && - root !== record.mount_prefix && - !root.startsWith(`${record.mount_prefix}/`) - ) { - throw new Error(`package deferred ZIP tree activation root escapes its mount: ${root}`); - } - } - return { - schema: 1, - kind: "kandelo-package-deferred-zip-tree", - id: record.id, - content_role: record.content_role, - package: { - name: packageRecord.name, - output: packageRecord.output, - }, - archive: { - url: archive.url, - mode_policy: "portable-posix-v1", - }, - mount_prefix: record.mount_prefix, - owner: { uid: owner.uid, gid: owner.gid }, - activation: { - mode: activation.mode, - capabilities: [...activation.capabilities] as string[], - roots: [...activation.roots] as string[], - }, - }; -} - /** * Derive the complete typed-tree contract from one exact package output. * The returned descriptor is the only recipe used by lazy registration and @@ -558,24 +432,6 @@ function preflightNamespace( } } -function exactRecord( - value: unknown, - fields: readonly string[], - label: string, -): Record { - if (typeof value !== "object" || value === null || Array.isArray(value)) { - throw new Error(`${label} must be an object`); - } - const record = value as Record; - if ( - JSON.stringify(Object.keys(record).sort()) !== - JSON.stringify([...fields].sort()) - ) { - throw new Error(`${label} has unsupported fields`); - } - return record; -} - function canonicalRelativePath(value: string): string { if ( value.length === 0 || value.startsWith("/") || value.includes("\\") || @@ -588,34 +444,6 @@ function canonicalRelativePath(value: string): string { return value; } -function canonicalAbsolutePath(value: string, allowRoot: boolean): string | null { - if (allowRoot && value === "/") return value; - if ( - !value.startsWith("/") || value.endsWith("/") || value.includes("\\") || - value.includes("\0") || - value.slice(1).split("/").some((segment) => - segment === "" || segment === "." || segment === ".." - ) - ) return null; - return value; -} - -function isRelativeAssetUrl(value: string): boolean { - return ( - value.length > 0 && value.length <= 255 && !value.startsWith("/") && - !value.includes("\\") && !value.includes("\0") && - !/^[a-z][a-z0-9+.-]*:/i.test(value) && - value.split("/").every((segment) => - segment !== "" && segment !== "." && segment !== ".." - ) - ); -} - -function isOwnerId(value: unknown): value is number { - return Number.isSafeInteger(value) && (value as number) >= 0 && - (value as number) <= MAX_OWNER_ID; -} - function canonicalJsonBytes(value: unknown): Uint8Array { return new TextEncoder().encode(`${JSON.stringify(sortJson(value))}\n`); } diff --git a/host/test/browser-kernel.test.ts b/host/test/browser-kernel.test.ts index 2bc345461c..0fc2d96056 100644 --- a/host/test/browser-kernel.test.ts +++ b/host/test/browser-kernel.test.ts @@ -469,6 +469,121 @@ describe("BrowserKernel", () => { expect(await unlinkPromise).toBe(true); }); + it("rejects rootfs export before the kernel worker is initialized", async () => { + const BrowserKernel = await loadBrowserKernel(); + const kernel = new BrowserKernel({ kernelOwnedFs: true }); + + await expect(kernel.exportRootfsImage()).rejects.toThrow( + "rootfs export requires an initialized kernel", + ); + expect(MockWorker.instances).toHaveLength(0); + await expect(kernel.destroy()).resolves.toBeUndefined(); + }); + + it("returns the exact rootfs bytes supplied by the worker", async () => { + const BrowserKernel = await loadBrowserKernel(); + const kernel = new BrowserKernel({ kernelOwnedFs: true }); + const initPromise = kernel.initFromImage({ + kernelWasm: new ArrayBuffer(8), + vfsImage: new Uint8Array(0), + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + const worker = MockWorker.instances[0]!; + worker.simulateMessage({ type: "ready" }); + await initPromise; + + const exportPromise = kernel.exportRootfsImage(); + await new Promise((resolve) => setTimeout(resolve, 0)); + const request = worker.lastMessage("export_rootfs_image"); + expect(request).toMatchObject({ type: "export_rootfs_image" }); + const expected = new Uint8Array([0, 255, 7, 91]); + worker.simulateMessage({ + type: "response", + requestId: request.requestId, + result: expected, + }); + + const actual = await exportPromise; + expect(actual).toBe(expected); + expect(actual).toEqual(new Uint8Array([0, 255, 7, 91])); + }); + + it("fails closed for malformed and rejected rootfs export responses", async () => { + const BrowserKernel = await loadBrowserKernel(); + const kernel = new BrowserKernel({ kernelOwnedFs: true }); + const initPromise = kernel.initFromImage({ + kernelWasm: new ArrayBuffer(8), + vfsImage: new Uint8Array(0), + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + const worker = MockWorker.instances[0]!; + worker.simulateMessage({ type: "ready" }); + await initPromise; + + const malformedPromise = kernel.exportRootfsImage(); + await new Promise((resolve) => setTimeout(resolve, 0)); + const malformed = worker.lastMessage("export_rootfs_image"); + worker.simulateMessage({ + type: "response", + requestId: malformed.requestId, + result: [1, 2, 3], + }); + await expect(malformedPromise).rejects.toThrow( + "kernel worker returned an invalid rootfs image", + ); + + const rejectedPromise = kernel.exportRootfsImage(); + await new Promise((resolve) => setTimeout(resolve, 0)); + const rejected = worker.lastMessage("export_rootfs_image"); + worker.simulateMessage({ + type: "response", + requestId: rejected.requestId, + result: null, + error: "rootfs export is already in progress", + }); + await expect(rejectedPromise).rejects.toThrow( + "rootfs export is already in progress", + ); + }); + + it("keeps concurrent rootfs export responses paired to their requests", async () => { + const BrowserKernel = await loadBrowserKernel(); + const kernel = new BrowserKernel({ kernelOwnedFs: true }); + const initPromise = kernel.initFromImage({ + kernelWasm: new ArrayBuffer(8), + vfsImage: new Uint8Array(0), + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + const worker = MockWorker.instances[0]!; + worker.simulateMessage({ type: "ready" }); + await initPromise; + + const first = kernel.exportRootfsImage(); + const second = kernel.exportRootfsImage(); + await new Promise((resolve) => setTimeout(resolve, 0)); + const requests = worker.sent + .map(({ data }) => data) + .filter((message) => message?.type === "export_rootfs_image"); + expect(requests).toHaveLength(2); + + worker.simulateMessage({ + type: "response", + requestId: requests[1].requestId, + result: null, + error: "rootfs export is already in progress", + }); + worker.simulateMessage({ + type: "response", + requestId: requests[0].requestId, + result: new Uint8Array([4, 2]), + }); + + await expect(first).resolves.toEqual(new Uint8Array([4, 2])); + await expect(second).rejects.toThrow( + "rootfs export is already in progress", + ); + }); + it("reads and validates kernel allocator page telemetry", async () => { const BrowserKernel = await loadBrowserKernel(); const kernel = new BrowserKernel({ kernelOwnedFs: true }); diff --git a/host/test/global-setup.ts b/host/test/global-setup.ts index 7add51c87d..c0a256a3b8 100644 --- a/host/test/global-setup.ts +++ b/host/test/global-setup.ts @@ -63,6 +63,7 @@ const TEST_PROGRAMS = [ "spawn-smoke.c", "spawn-coverage.c", "spawn-pause.c", + "block-forever.c", "mount_probe_test.c", "getpwent_smoke.c", "initial-credentials-test.c", diff --git a/host/test/homebrew-vfs-image-save.test.ts b/host/test/homebrew-vfs-image-save.test.ts index e159a2701c..9d02369b15 100644 --- a/host/test/homebrew-vfs-image-save.test.ts +++ b/host/test/homebrew-vfs-image-save.test.ts @@ -1,19 +1,253 @@ -import { - existsSync, - mkdtempSync, - rmSync, -} from "node:fs"; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; +import { zipSync, type Zippable } from "fflate"; import { + assertHomebrewBootstrapConsumerState, + installHomebrewBootstrapConsumerState, + prepareHomebrewBootstrapConsumerNamespace, + readHomebrewBootstrapEnvironment, saveVerifiedHomebrewVfsImage, } from "../../images/vfs/scripts/build-homebrew-vfs-image"; +import { + assertPackageDeferredZipTreeState, + derivePackageDeferredZipTree, + materializePackageDeferredZipTree, + registerPackageDeferredZipTree, + type PackageDeferredZipTreeSpec, +} from "../src/vfs/package-deferred-tree"; import { MemoryFileSystem } from "../src/vfs/memory-fs"; +import { writeVfsBinary } from "../src/vfs/image-helpers"; const MiB = 1024 * 1024; +const encoder = new TextEncoder(); +const bootstrapEnvironment = encoder.encode( + "HOMEBREW_NO_ANALYTICS=1\n" + + "HOMEBREW_NO_AUTO_UPDATE=1\n" + + "HOMEBREW_SYSTEM_ENV_TAKES_PRIORITY=1\n" + + "HOMEBREW_KANDELO_BOTTLE_TAG=wasm32_kandelo\n", +); +const bootstrapSpec = { + schema: 1, + kind: "kandelo-package-deferred-zip-tree", + id: "homebrew-bootstrap/source-tree", + content_role: "source-tree", + package: { name: "homebrew-bootstrap", output: "homebrew-bootstrap.zip" }, + archive: { + url: "homebrew-bootstrap.zip", + mode_policy: "portable-posix-v1", + }, + mount_prefix: "/home/linuxbrew/.linuxbrew", + owner: { uid: 1000, gid: 1000 }, + activation: { + mode: "first-use", + capabilities: ["homebrew:bootstrap"], + roots: ["/home/linuxbrew/.linuxbrew/bin/brew"], + }, +} as const satisfies PackageDeferredZipTreeSpec; describe("Homebrew VFS image publication boundary", () => { + it.each(["deferred", "materialized"] as const)( + "adopts a real bottle prefix for a %s package source tree and survives serialization", + async (state) => { + const archive = bootstrapArchive(true); + const derived = derivePackageDeferredZipTree(bootstrapSpec, archive); + const fs = bootstrapConsumerFs(); + + expect(() => registerPackageDeferredZipTree(fs, derived)).toThrow( + "collides with the base", + ); + prepareHomebrewBootstrapConsumerNamespace(fs, derived); + const registered = registerPackageDeferredZipTree(fs, derived); + if (state === "materialized") { + await materializePackageDeferredZipTree(fs, registered, archive); + } + const consumer = installHomebrewBootstrapConsumerState( + fs, + derived, + bootstrapEnvironment, + ); + assertPackageDeferredZipTreeState(fs, derived, state); + assertHomebrewBootstrapConsumerState(fs, consumer); + + const restored = MemoryFileSystem.fromImagePreservingCapacity( + await fs.saveImage(), + ); + assertPackageDeferredZipTreeState(restored, derived, state); + assertHomebrewBootstrapConsumerState(restored, consumer); + expect( + restored.lstat("/home/linuxbrew/.linuxbrew/Cellar/existing/1/bin/tool"), + ).toMatchObject({ uid: 1000, gid: 1000 }); + expect(restored.lstat("/etc/homebrew/brew.env")).toMatchObject({ + mode: expect.any(Number), + uid: 0, + gid: 0, + }); + expect(restored.readlink("/usr/bin/brew")).toBe( + "/home/linuxbrew/.linuxbrew/bin/brew", + ); + }, + ); + + it("materializes the source tree through the public /usr/bin/brew alias", async () => { + const archive = bootstrapArchive(true); + const derived = derivePackageDeferredZipTree(bootstrapSpec, archive); + const fs = bootstrapConsumerFs(); + prepareHomebrewBootstrapConsumerNamespace(fs, derived); + registerPackageDeferredZipTree(fs, derived); + installHomebrewBootstrapConsumerState(fs, derived, bootstrapEnvironment); + let fetchCount = 0; + fs.setLazyFetcher(async (url) => { + fetchCount += 1; + expect(url).toBe("homebrew-bootstrap.zip"); + return new Response(archive, { + headers: { "content-length": String(archive.byteLength) }, + }); + }); + + expect(fs.isPathDeferred("/usr/bin/brew")).toBe(true); + await expect(fs.preparePath("/usr/bin/brew")).resolves.toBe(true); + expect(fetchCount).toBe(1); + expect(fs.isPathDeferred("/usr/bin/brew")).toBe(false); + assertPackageDeferredZipTreeState(fs, derived, "materialized"); + + await expect(fs.preparePath("/usr/bin/brew")).resolves.toBe(false); + expect(fetchCount).toBe(1); + }); + + it("rejects a missing or dangling Homebrew entrypoint and a changed launcher policy", () => { + const incomplete = derivePackageDeferredZipTree( + bootstrapSpec, + bootstrapArchive(false), + ); + const incompleteFs = bootstrapConsumerFs(); + prepareHomebrewBootstrapConsumerNamespace(incompleteFs, incomplete); + expect(() => + registerPackageDeferredZipTree(incompleteFs, incomplete), + ).toThrow("activation root"); + + const valid = derivePackageDeferredZipTree( + bootstrapSpec, + bootstrapArchive(true), + ); + const danglingFs = bootstrapConsumerFs(); + prepareHomebrewBootstrapConsumerNamespace(danglingFs, valid); + registerPackageDeferredZipTree(danglingFs, valid); + danglingFs.unlink("/home/linuxbrew/.linuxbrew/bin/brew"); + expect(() => + installHomebrewBootstrapConsumerState( + danglingFs, + valid, + bootstrapEnvironment, + ), + ).toThrow("canonical deferred source tree"); + + const directory = mkdtempSync(join(tmpdir(), "homebrew-bootstrap-env-")); + try { + const valid = join(directory, "brew.env"); + const changed = join(directory, "changed.env"); + writeFileSync(valid, bootstrapEnvironment); + writeFileSync( + changed, + new TextEncoder().encode( + "HOMEBREW_NO_ANALYTICS=1\n" + + "HOMEBREW_NO_AUTO_UPDATE=1\n" + + "HOMEBREW_SYSTEM_ENV_TAKES_PRIORITY=1\n" + + "HOMEBREW_KANDELO_BOTTLE_TAG=wasm64_kandelo\n", + ), + ); + expect(readHomebrewBootstrapEnvironment(valid, "wasm32")).toEqual( + bootstrapEnvironment, + ); + expect(() => readHomebrewBootstrapEnvironment(changed, "wasm32")).toThrow( + "does not select wasm32_kandelo", + ); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); + + it.each(["environment", "entrypoint"] as const)( + "does not replace pre-existing Homebrew %s state", + (kind) => { + const tree = derivePackageDeferredZipTree( + bootstrapSpec, + bootstrapArchive(true), + ); + const fs = bootstrapConsumerFs(); + prepareHomebrewBootstrapConsumerNamespace(fs, tree); + registerPackageDeferredZipTree(fs, tree); + if (kind === "environment") { + fs.mkdir("/etc/homebrew", 0o755); + writeVfsBinary( + fs, + "/etc/homebrew/brew.env", + encoder.encode("existing\n"), + 0o644, + ); + } else { + writeVfsBinary( + fs, + "/usr/bin/brew", + encoder.encode("existing\n"), + 0o755, + ); + } + expect(() => + installHomebrewBootstrapConsumerState(fs, tree, bootstrapEnvironment), + ).toThrow("refusing to replace Homebrew bootstrap consumer state"); + }, + ); + + it.each([ + "environment", + "entrypoint", + "target", + "prefix-owner", + "cache-owner", + ] as const)("detects %s drift after installation", (kind) => { + const tree = derivePackageDeferredZipTree( + bootstrapSpec, + bootstrapArchive(true), + ); + const fs = bootstrapConsumerFs(); + prepareHomebrewBootstrapConsumerNamespace(fs, tree); + registerPackageDeferredZipTree(fs, tree); + const consumer = installHomebrewBootstrapConsumerState( + fs, + tree, + bootstrapEnvironment, + ); + + switch (kind) { + case "environment": + fs.unlink("/etc/homebrew/brew.env"); + writeVfsBinary( + fs, + "/etc/homebrew/brew.env", + encoder.encode("changed\n"), + 0o644, + ); + break; + case "entrypoint": + fs.unlink("/usr/bin/brew"); + fs.symlink("/wrong/brew", "/usr/bin/brew"); + break; + case "target": + fs.unlink("/home/linuxbrew/.linuxbrew/bin/brew"); + break; + case "prefix-owner": + fs.chown("/home/linuxbrew/.linuxbrew", 0, 0); + break; + case "cache-owner": + fs.chown("/home/user/.cache/Homebrew", 0, 0); + break; + } + + expect(() => assertHomebrewBootstrapConsumerState(fs, consumer)).toThrow(); + }); + it("writes an image whose encoded ceiling matches its consumer contract", async () => { const maxByteLength = 8 * MiB; const fs = MemoryFileSystem.create( @@ -30,9 +264,9 @@ describe("Homebrew VFS image publication boundary", () => { maxByteLength, ); - expect( - MemoryFileSystem.readImageCapacity(image).maxByteLength, - ).toBe(maxByteLength); + expect(MemoryFileSystem.readImageCapacity(image).maxByteLength).toBe( + maxByteLength, + ); expect(existsSync(outFile)).toBe(true); } finally { rmSync(dir, { recursive: true, force: true }); @@ -74,3 +308,53 @@ describe("Homebrew VFS image publication boundary", () => { } }); }); + +function bootstrapArchive(includeBrew: boolean): Uint8Array { + const entries: Zippable = { + "bin/": zipEntry(new Uint8Array(), 0o040755), + "Library/": zipEntry(new Uint8Array(), 0o040755), + "Library/Homebrew/": zipEntry(new Uint8Array(), 0o040755), + "Library/Homebrew/global.rb": zipEntry( + encoder.encode("GLOBAL = true\n"), + 0o100644, + ), + }; + if (includeBrew) { + entries["bin/brew"] = zipEntry(encoder.encode("#!/bin/bash\n"), 0o100755); + } + return zipSync(entries, { level: 9 }); +} + +function zipEntry(bytes: Uint8Array, mode: number): Zippable[string] { + return [bytes, { os: 3, attrs: (mode << 16) >>> 0 }]; +} + +function bootstrapConsumerFs(): MemoryFileSystem { + const fs = MemoryFileSystem.create( + new SharedArrayBuffer(8 * MiB, { maxByteLength: 32 * MiB }), + 32 * MiB, + ); + for (const path of [ + "/home", + "/home/linuxbrew", + "/home/linuxbrew/.linuxbrew", + "/home/linuxbrew/.linuxbrew/bin", + "/home/linuxbrew/.linuxbrew/Cellar", + "/home/linuxbrew/.linuxbrew/Cellar/existing", + "/home/linuxbrew/.linuxbrew/Cellar/existing/1", + "/home/linuxbrew/.linuxbrew/Cellar/existing/1/bin", + "/home/user", + "/usr", + "/usr/bin", + "/etc", + ]) { + fs.mkdir(path, 0o755); + } + writeVfsBinary( + fs, + "/home/linuxbrew/.linuxbrew/Cellar/existing/1/bin/tool", + encoder.encode("tool\n"), + 0o755, + ); + return fs; +} diff --git a/host/test/node-rootfs-export.test.ts b/host/test/node-rootfs-export.test.ts new file mode 100644 index 0000000000..26827bbe5b --- /dev/null +++ b/host/test/node-rootfs-export.test.ts @@ -0,0 +1,176 @@ +import { existsSync, readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { describe, expect, it } from "vitest"; + +import { tryResolveBinary } from "../src/binary-resolver"; +import { NodeKernelHost } from "../src/node-kernel-host"; +import { MemoryFileSystem } from "../src/vfs/memory-fs"; + +const here = dirname(fileURLToPath(import.meta.url)); +const repoRoot = join(here, "../.."); +const kernelPath = tryResolveBinary("kernel.wasm"); +const blockForeverPath = join(repoRoot, "examples/block-forever.wasm"); +const haveKernel = kernelPath !== null; +const haveBlockForever = existsSync(blockForeverPath); + +function asArrayBuffer(bytes: Uint8Array): ArrayBuffer { + return bytes.buffer.slice( + bytes.byteOffset, + bytes.byteOffset + bytes.byteLength, + ) as ArrayBuffer; +} + +function writeFile( + fs: MemoryFileSystem, + path: string, + bytes: Uint8Array, + mode = 0o644, +): void { + const fd = fs.open(path, 0o1101 /* O_WRONLY|O_CREAT|O_TRUNC */, mode); + try { + expect(fs.write(fd, bytes, null, bytes.byteLength)).toBe(bytes.byteLength); + } finally { + fs.close(fd); + } +} + +function readFile(fs: MemoryFileSystem, path: string): Uint8Array { + const stat = fs.stat(path); + const bytes = new Uint8Array(stat.size); + const fd = fs.open(path, 0, 0); + try { + expect(fs.read(fd, bytes, null, bytes.byteLength)).toBe(bytes.byteLength); + } finally { + fs.close(fd); + } + return bytes; +} + +async function createRootfs(): Promise { + const fs = MemoryFileSystem.create(new SharedArrayBuffer(8 * 1024 * 1024)); + fs.mkdir("/var", 0o755); + fs.mkdir("/var/lib", 0o755); + writeFile( + fs, + "/var/lib/persisted-state", + new TextEncoder().encode("survives reboot\n"), + 0o640, + ); + fs.registerLazyFile( + "/opt/lazy-tool", + "https://packages.example.test/lazy-tool.wasm", + 123_456, + 0o755, + ); + return fs.saveImage(); +} + +describe("NodeKernelHost rootfs export contract", () => { + it("rejects export before initialization without starting a worker", async () => { + const host = new NodeKernelHost({ rootfsImage: new Uint8Array() }); + await expect(host.exportRootfsImage()).rejects.toThrow( + "rootfs export requires an initialized kernel", + ); + await expect(host.destroy()).resolves.toBeUndefined(); + }); + + it.skipIf(!haveKernel)( + "rejects a host-filesystem kernel because it has no VFS image", + async () => { + const host = new NodeKernelHost(); + try { + await host.init(asArrayBuffer(new Uint8Array(readFileSync(kernelPath!)))); + await expect(host.exportRootfsImage()).rejects.toThrow( + "rootfs export requires a VFS-backed kernel", + ); + } finally { + await host.destroy(); + } + }, + ); + + it.skipIf(!haveKernel)( + "transfers exact bytes, preserves lazy descriptors, and reboots from the export", + async () => { + const kernel = new Uint8Array(readFileSync(kernelPath!)); + const initialImage = await createRootfs(); + const first = new NodeKernelHost({ rootfsImage: initialImage }); + let exported: Uint8Array; + try { + await first.init(asArrayBuffer(kernel)); + exported = await first.exportRootfsImage(); + } finally { + await first.destroy(); + } + + expect(exported).toBeInstanceOf(Uint8Array); + const restored = MemoryFileSystem.fromImage(exported); + expect(new TextDecoder().decode( + readFile(restored, "/var/lib/persisted-state"), + )).toBe("survives reboot\n"); + expect(restored.stat("/var/lib/persisted-state").mode & 0o7777).toBe(0o640); + expect(restored.exportLazyEntries()).toEqual([expect.objectContaining({ + path: "/opt/lazy-tool", + url: "https://packages.example.test/lazy-tool.wasm", + size: 123_456, + })]); + + const rebooted = new NodeKernelHost({ rootfsImage: exported }); + try { + await rebooted.init(asArrayBuffer(kernel)); + const afterReboot = await rebooted.exportRootfsImage(); + const afterRebootFs = MemoryFileSystem.fromImage(afterReboot); + expect(new TextDecoder().decode( + readFile(afterRebootFs, "/var/lib/persisted-state"), + )).toBe("survives reboot\n"); + expect(afterRebootFs.exportLazyEntries()).toEqual([ + expect.objectContaining({ + path: "/opt/lazy-tool", + url: "https://packages.example.test/lazy-tool.wasm", + size: 123_456, + }), + ]); + } finally { + await rebooted.destroy(); + } + }, + ); + + it.skipIf(!haveKernel || !haveBlockForever)( + "rejects live and tearing-down processes without racing a snapshot", + async () => { + const kernel = new Uint8Array(readFileSync(kernelPath!)); + const program = new Uint8Array(readFileSync(blockForeverPath)); + const host = new NodeKernelHost({ rootfsImage: await createRootfs() }); + try { + await host.init(asArrayBuffer(kernel)); + let startedPid = -1; + const exit = host.spawn(asArrayBuffer(program), ["block-forever"], { + onStarted: (pid) => { + startedPid = pid; + }, + }); + for (let tries = 0; startedPid < 0 && tries < 100; tries += 1) { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(startedPid).toBeGreaterThan(0); + + await expect(host.exportRootfsImage()).rejects.toThrow( + "no live or tearing-down processes", + ); + + const terminating = host.terminateProcess(startedPid, 143); + await expect(host.exportRootfsImage()).rejects.toThrow( + "no live or tearing-down processes", + ); + await terminating; + await expect(exit).resolves.toBe(143); + await expect(host.exportRootfsImage()).resolves.toBeInstanceOf(Uint8Array); + } finally { + await host.destroy(); + } + }, + ); +}); diff --git a/host/test/rootfs-snapshot-gate.test.ts b/host/test/rootfs-snapshot-gate.test.ts new file mode 100644 index 0000000000..f6e0246b47 --- /dev/null +++ b/host/test/rootfs-snapshot-gate.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it, vi } from "vitest"; + +import { RootfsSnapshotGate } from "../src/rootfs-snapshot-gate"; + +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise((settle) => { + resolve = settle; + }); + return { promise, resolve }; +} + +describe("RootfsSnapshotGate", () => { + it("waits for an earlier mutation and excludes every later mutation", async () => { + const gate = new RootfsSnapshotGate(); + const releaseMutation = gate.beginMutation("finish an earlier write"); + const snapshotEntered = vi.fn(); + + const snapshot = gate.runSnapshot(async () => { + snapshotEntered(); + return new Uint8Array([1, 2, 3]); + }); + await Promise.resolve(); + + expect(snapshotEntered).not.toHaveBeenCalled(); + expect(() => gate.beginMutation("spawn a process")).toThrow( + "rootfs export is in progress; cannot spawn a process", + ); + await expect(gate.runSnapshot(async () => new Uint8Array())).rejects.toThrow( + "rootfs export is already in progress", + ); + + releaseMutation(); + await expect(snapshot).resolves.toEqual(new Uint8Array([1, 2, 3])); + expect(snapshotEntered).toHaveBeenCalledOnce(); + + const releaseAfterSnapshot = gate.beginMutation("write after export"); + expect(() => releaseAfterSnapshot()).not.toThrow(); + }); + + it("holds the gate until an asynchronous snapshot settles", async () => { + const gate = new RootfsSnapshotGate(); + const save = deferred(); + const snapshot = gate.runSnapshot(() => save.promise); + + expect(() => gate.beginMutation("unlink a rootfs file")).toThrow( + "rootfs export is in progress; cannot unlink a rootfs file", + ); + save.resolve(new Uint8Array([9])); + await expect(snapshot).resolves.toEqual(new Uint8Array([9])); + }); + + it("reopens after a failed snapshot without swallowing the failure", async () => { + const gate = new RootfsSnapshotGate(); + await expect( + gate.runSnapshot(async () => { + throw new Error("saveImage failed"); + }), + ).rejects.toThrow("saveImage failed"); + + const release = gate.beginMutation("retry a write"); + release(); + await expect( + gate.runSnapshot(async () => "retry succeeded"), + ).resolves.toBe("retry succeeded"); + }); + + it("detects a duplicate mutation release", () => { + const gate = new RootfsSnapshotGate(); + const release = gate.beginMutation("one write"); + release(); + expect(() => release()).toThrow( + "rootfs snapshot mutation released twice: one write", + ); + }); +}); diff --git a/host/test/shell-lazy-archive-inputs.test.ts b/host/test/shell-lazy-archive-inputs.test.ts index b9a7e0b125..a68fbf99e6 100644 --- a/host/test/shell-lazy-archive-inputs.test.ts +++ b/host/test/shell-lazy-archive-inputs.test.ts @@ -538,12 +538,15 @@ describe("declared shell lazy-archive inputs", () => { }; // The canonical shell no longer resolves the old registry ZIP packages. - // Its reviewed lock maps those historical identities to direct Formula - // roots, and the exact immutable tap is part of the package cache key. - expect(packageToml).toMatch(/^depends_on\s*=\s*\[\]$/m); + // Its only registry dependency is the atomic Homebrew source/launcher + // package needed to register `brew` lazily; the reviewed lock maps the + // historical program identities to direct Formula roots. + expect(packageToml).toMatch( + /^depends_on\s*=\s*\["homebrew-bootstrap@6\.0\.3-4-g4ead861"\]$/m, + ); expect(packageToml).not.toContain("vim-browser-bundle@"); expect(packageToml).not.toContain("nethack-browser-bundle@"); - expect(buildToml).toMatch(/^revision\s*=\s*18$/m); + expect(buildToml).toMatch(/^revision\s*=\s*19$/m); for (const input of [ "scripts/build-homebrew-main-shell-closure.sh", "scripts/check-homebrew-main-shell-brewfile.mjs", diff --git a/images/vfs/scripts/build-homebrew-vfs-image.ts b/images/vfs/scripts/build-homebrew-vfs-image.ts index b99f526c64..25aedcf943 100644 --- a/images/vfs/scripts/build-homebrew-vfs-image.ts +++ b/images/vfs/scripts/build-homebrew-vfs-image.ts @@ -110,6 +110,7 @@ interface CliOptions { bottleMirrorOut?: string; packageTreeSpec?: string; packageTreeArchive?: string; + homebrewBootstrapEnv?: string; materializePackageTree: boolean; } @@ -119,8 +120,7 @@ interface CliOptions { * candidate composer imports while both paths reuse the same CLI, planning, * image metadata, and serialization implementation. */ -export interface HomebrewVfsImageMaterializationOptions - extends HomebrewVfsBuildOptions { +export interface HomebrewVfsImageMaterializationOptions extends HomebrewVfsBuildOptions { fs: MemoryFileSystem; collectionFs: MemoryFileSystem; policy: unknown; @@ -158,6 +158,18 @@ export async function saveVerifiedHomebrewVfsImage( const DEFAULT_MAX_BYTES = 128 * 1024 * 1024; const SHARED_FS_BLOCK_BYTES = 4096; const HOMEBREW_COMPOSITION_PATH = "/etc/kandelo/homebrew-vfs.json"; +const HOMEBREW_BOOTSTRAP_ENV_PATH = "/etc/homebrew/brew.env"; +const HOMEBREW_BOOTSTRAP_ENTRYPOINT = "/usr/bin/brew"; +const HOMEBREW_BOOTSTRAP_PREFIX = "/home/linuxbrew/.linuxbrew"; +const HOMEBREW_BOOTSTRAP_TARGET = "/home/linuxbrew/.linuxbrew/bin/brew"; +const HOMEBREW_BOOTSTRAP_MUTABLE_PATHS = [ + "/home/linuxbrew/.linuxbrew/Cellar", + "/home/linuxbrew/.linuxbrew/Library/Taps", + "/home/linuxbrew/.linuxbrew/var/homebrew/linked", + "/home/linuxbrew/.linuxbrew/var/homebrew/locks", + "/home/user/.cache/Homebrew", +] as const; +const MAX_HOMEBREW_BOOTSTRAP_ENV_BYTES = 1024; const MAX_SIDECAR_JSON_BYTES = 16_777_216; const MAX_BREWFILE_BYTES = 65_536; const MAX_BREWFILE_PACKAGES = 128; @@ -213,6 +225,24 @@ interface LoadedMigrationLock { bytes: number; } +export interface HomebrewBootstrapConsumerState { + environment: { + path: typeof HOMEBREW_BOOTSTRAP_ENV_PATH; + sha256: string; + bytes: number; + }; + entrypoint: { + path: typeof HOMEBREW_BOOTSTRAP_ENTRYPOINT; + target: typeof HOMEBREW_BOOTSTRAP_TARGET; + }; + ownership: { + prefix: typeof HOMEBREW_BOOTSTRAP_PREFIX; + uid: 1000; + gid: 1000; + mutable_paths: string[]; + }; +} + export async function runHomebrewVfsImageBuilder( args: string[], materialize?: HomebrewVfsImageMaterializer, @@ -229,9 +259,20 @@ export async function runHomebrewVfsImageBuilder( const migrationLock = options.migrationLock ? readMigrationLock(options.migrationLock) : undefined; - const compatibilityPolicy = migrationLock === undefined - ? undefined - : migrationLockCompatibilityPolicy(migrationLock.value, options.migrationLock!); + const homebrewBootstrapEnv = + options.homebrewBootstrapEnv === undefined + ? undefined + : readHomebrewBootstrapEnvironment( + options.homebrewBootstrapEnv, + options.arch, + ); + const compatibilityPolicy = + migrationLock === undefined + ? undefined + : migrationLockCompatibilityPolicy( + migrationLock.value, + options.migrationLock!, + ); const brewfileSelection = options.brewfile ? readBrewfileSelection(options.brewfile) : undefined; @@ -246,7 +287,9 @@ export async function runHomebrewVfsImageBuilder( ); const tapRoots = new Map([ [primaryTapName, options.tapRoot], - ...dependencyMetadata.map(({ tapName, tapRoot }) => [tapName, tapRoot] as const), + ...dependencyMetadata.map( + ({ tapName, tapRoot }) => [tapName, tapRoot] as const, + ), ]); const commonPlanOptions = { packages: requestedPackages, @@ -255,26 +298,30 @@ export async function runHomebrewVfsImageBuilder( expectedCacheKeys: options.expectedCacheKeys, allowFallback: options.allowFallback, }; - const plan = dependencyMetadata.length === 0 - ? await planHomebrewVfs(metadata, { - ...commonPlanOptions, - expectedTapName: brewfileSelection?.tap_name, - loadLinkManifest: (relPath: string) => readJsonFile(join(options.tapRoot, relPath)), - }) - : await planFederatedHomebrewVfs( - [metadata, ...dependencyMetadata.map(({ metadata: value }) => value)], - { - ...commonPlanOptions, - rootTapName: brewfileSelection?.tap_name ?? primaryTapName, - loadLinkManifest: (tap, relPath) => { - const root = tapRoots.get(tap.tapName); - if (root === undefined) { - throw new Error(`no immutable checkout is available for tap ${tap.tapName}`); - } - return readJsonFile(join(root, relPath)); - }, - }, - ); + const plan = + dependencyMetadata.length === 0 + ? await planHomebrewVfs(metadata, { + ...commonPlanOptions, + expectedTapName: brewfileSelection?.tap_name, + loadLinkManifest: (relPath: string) => + readJsonFile(join(options.tapRoot, relPath)), + }) + : await planFederatedHomebrewVfs( + [metadata, ...dependencyMetadata.map(({ metadata: value }) => value)], + { + ...commonPlanOptions, + rootTapName: brewfileSelection?.tap_name ?? primaryTapName, + loadLinkManifest: (tap, relPath) => { + const root = tapRoots.get(tap.tapName); + if (root === undefined) { + throw new Error( + `no immutable checkout is available for tap ${tap.tapName}`, + ); + } + return readJsonFile(join(root, relPath)); + }, + }, + ); const { fs, baseImage, maxByteLength } = createFs( options.baseImage, @@ -289,22 +336,30 @@ export async function runHomebrewVfsImageBuilder( loadedBottleBytes.set(pkg.fullName, bytes); return bytes; }; - const selectionSource = brewfileSelection ? { - kind: "brewfile" as const, - parser: brewfileSelection.kind, - sha256: brewfileSelection.sha256, - bytes: brewfileSelection.bytes, - requestedPackages: brewfileSelection.packages, - } : undefined; - const catalogCheckout = options.catalogCommit === undefined ? undefined : { - tapRepository: plan.tapRepository, - tapName: plan.tapName, - checkoutCommit: options.catalogCommit, - }; - const migrationLockBinding = migrationLock === undefined ? undefined : { - sha256: migrationLock.sha256, - bytes: migrationLock.bytes, - }; + const selectionSource = brewfileSelection + ? { + kind: "brewfile" as const, + parser: brewfileSelection.kind, + sha256: brewfileSelection.sha256, + bytes: brewfileSelection.bytes, + requestedPackages: brewfileSelection.packages, + } + : undefined; + const catalogCheckout = + options.catalogCommit === undefined + ? undefined + : { + tapRepository: plan.tapRepository, + tapName: plan.tapName, + checkoutCommit: options.catalogCommit, + }; + const migrationLockBinding = + migrationLock === undefined + ? undefined + : { + sha256: migrationLock.sha256, + bytes: migrationLock.bytes, + }; let materializedBuild: HomebrewVfsImageMaterializedBuild | undefined; const commonBuildOptions = { fs, @@ -328,32 +383,38 @@ export async function runHomebrewVfsImageBuilder( materializedBuild = await materialize(plan, { ...commonBuildOptions, fs, - collectionFs: createFs( - undefined, - maxByteLength, - plan.kandeloAbi, - ).fs, + collectionFs: createFs(undefined, maxByteLength, plan.kandeloAbi).fs, policy: readJsonFile(options.materializationPolicy), mirrorRepository: options.bottleMirrorRepository!, }); result = materializedBuild.result; materializedBuild.assert(fs); } - let packageTree: { - derived: DerivedPackageDeferredZipTree; - state: "deferred" | "materialized"; - } | undefined; + let packageTree: + | { + derived: DerivedPackageDeferredZipTree; + state: "deferred" | "materialized"; + } + | undefined; + let homebrewBootstrapConsumerState: + HomebrewBootstrapConsumerState | undefined; if (options.packageTreeSpec !== undefined) { const archiveBytes = readPackageTreeArchive(options.packageTreeArchive!); const derived = derivePackageDeferredZipTree( readJsonFile(options.packageTreeSpec), archiveBytes, ); - if (basename(options.packageTreeArchive!) !== derived.descriptor.package.output) { + if ( + basename(options.packageTreeArchive!) !== + derived.descriptor.package.output + ) { throw new Error( `package tree archive must be named ${derived.descriptor.package.output}`, ); } + if (homebrewBootstrapEnv !== undefined) { + prepareHomebrewBootstrapConsumerNamespace(fs, derived); + } const registered = registerPackageDeferredZipTree(fs, derived); if (options.materializePackageTree) { await materializePackageDeferredZipTree(fs, registered, archiveBytes); @@ -361,6 +422,13 @@ export async function runHomebrewVfsImageBuilder( const state = options.materializePackageTree ? "materialized" : "deferred"; assertPackageDeferredZipTreeState(fs, derived, state); packageTree = { derived, state }; + if (homebrewBootstrapEnv !== undefined) { + homebrewBootstrapConsumerState = installHomebrewBootstrapConsumerState( + fs, + derived, + homebrewBootstrapEnv, + ); + } } if (shellConfig) { assertShellExecutable(fs, shellConfig.config.path); @@ -391,107 +459,135 @@ export async function runHomebrewVfsImageBuilder( writeVfsBinary(fs, KANDELO_DEMO_CONFIG_PATH, demoConfig.source, 0o644); } - const imageBytes = await saveVerifiedHomebrewVfsImage(fs, options.out, { - normalizeTimestampsMs: sourceDateEpochMilliseconds( - process.env.SOURCE_DATE_EPOCH, - ), - metadata: { - version: 1, - kernelAbi: plan.kandeloAbi, - createdBy: "images/vfs/scripts/build-homebrew-vfs-image.ts", - capacity: { maxByteLength }, - ...(baseImage ? { baseImage: baseImage.binding } : {}), - ...(packageTree === undefined ? {} : { - packageDeferredTrees: [packageTreeBinding(packageTree)], - }), - homebrew: { - tapRepository: plan.tapRepository, - tapName: plan.tapName, - tapCommit: plan.tapCommit, - releaseTag: plan.releaseTag, - ...(result.report.catalog === undefined ? {} : { - catalog: { - tapRepository: result.report.catalog.tap_repository, - tapName: result.report.catalog.tap_name, - checkoutCommit: result.report.catalog.checkout_commit, - }, - }), - ...(result.report.migration_lock === undefined ? {} : { - migrationLock: { - sha256: result.report.migration_lock.sha256, - bytes: result.report.migration_lock.bytes, - }, - }), - ...(result.report.runtime_state === undefined ? {} : { - runtimeState: result.report.runtime_state.map((entry) => ({ - requiresPackage: entry.requires_package, - path: entry.path, - kind: entry.kind, - mode: entry.mode, - uid: entry.uid, - gid: entry.gid, - reason: entry.reason, - ...(entry.content_sha256 === undefined ? {} : { - contentSha256: entry.content_sha256, - contentBytes: entry.content_bytes, + const imageBytes = await saveVerifiedHomebrewVfsImage( + fs, + options.out, + { + normalizeTimestampsMs: sourceDateEpochMilliseconds( + process.env.SOURCE_DATE_EPOCH, + ), + metadata: { + version: 1, + kernelAbi: plan.kandeloAbi, + createdBy: "images/vfs/scripts/build-homebrew-vfs-image.ts", + capacity: { maxByteLength }, + ...(baseImage ? { baseImage: baseImage.binding } : {}), + ...(packageTree === undefined + ? {} + : { + packageDeferredTrees: [packageTreeBinding(packageTree)], }), - })), - }), - selection: { - kind: result.report.selection.kind, - requestedPackageCount: - result.report.selection.requested_packages.length, - requestedPackagesSha256: - result.report.selection.requested_packages_sha256, - ...(result.report.selection.brewfile - ? { brewfile: result.report.selection.brewfile } + ...(homebrewBootstrapConsumerState === undefined + ? {} + : { + homebrewBootstrap: homebrewBootstrapConsumerState, + }), + homebrew: { + tapRepository: plan.tapRepository, + tapName: plan.tapName, + tapCommit: plan.tapCommit, + releaseTag: plan.releaseTag, + ...(result.report.catalog === undefined + ? {} + : { + catalog: { + tapRepository: result.report.catalog.tap_repository, + tapName: result.report.catalog.tap_name, + checkoutCommit: result.report.catalog.checkout_commit, + }, + }), + ...(result.report.migration_lock === undefined + ? {} + : { + migrationLock: { + sha256: result.report.migration_lock.sha256, + bytes: result.report.migration_lock.bytes, + }, + }), + ...(result.report.runtime_state === undefined + ? {} + : { + runtimeState: result.report.runtime_state.map((entry) => ({ + requiresPackage: entry.requires_package, + path: entry.path, + kind: entry.kind, + mode: entry.mode, + uid: entry.uid, + gid: entry.gid, + reason: entry.reason, + ...(entry.content_sha256 === undefined + ? {} + : { + contentSha256: entry.content_sha256, + contentBytes: entry.content_bytes, + }), + })), + }), + selection: { + kind: result.report.selection.kind, + requestedPackageCount: + result.report.selection.requested_packages.length, + requestedPackagesSha256: + result.report.selection.requested_packages_sha256, + ...(result.report.selection.brewfile + ? { brewfile: result.report.selection.brewfile } + : {}), + }, + ...(result.report.materialization === undefined + ? {} + : { + materialization: result.report.materialization, + }), + ...(shellConfig + ? { + defaultShell: { + path: shellConfig.config.path, + argv: shellConfig.config.argv, + configSha256: shellConfig.sha256, + }, + } : {}), + ...(demoConfig + ? { + demoConfig: { + path: KANDELO_DEMO_CONFIG_PATH, + sha256: demoConfig.sha256, + bytes: demoConfig.bytes, + }, + } + : {}), + packages: plan.packages.map((pkg) => ({ + name: pkg.name, + fullName: pkg.fullName, + tapRepository: pkg.tapRepository, + tapName: pkg.tapName, + tapCommit: pkg.tapCommit, + version: pkg.version, + arch: pkg.arch, + sourceStatus: pkg.sourceStatus, + cacheKeySha: pkg.cacheKeySha, + ...(pkg.builtFrom === undefined + ? {} + : { + builtFrom: { + tapRepository: pkg.builtFrom.tapRepository, + tapCommit: pkg.builtFrom.tapCommit, + kandeloRepository: pkg.builtFrom.kandeloRepository, + kandeloCommit: pkg.builtFrom.kandeloCommit, + formulaSha256: pkg.builtFrom.formulaSha256, + }, + }), + })), }, - ...(result.report.materialization === undefined ? {} : { - materialization: result.report.materialization, - }), - ...(shellConfig ? { - defaultShell: { - path: shellConfig.config.path, - argv: shellConfig.config.argv, - configSha256: shellConfig.sha256, - }, - } : {}), - ...(demoConfig ? { - demoConfig: { - path: KANDELO_DEMO_CONFIG_PATH, - sha256: demoConfig.sha256, - bytes: demoConfig.bytes, - }, - } : {}), - packages: plan.packages.map((pkg) => ({ - name: pkg.name, - fullName: pkg.fullName, - tapRepository: pkg.tapRepository, - tapName: pkg.tapName, - tapCommit: pkg.tapCommit, - version: pkg.version, - arch: pkg.arch, - sourceStatus: pkg.sourceStatus, - cacheKeySha: pkg.cacheKeySha, - ...(pkg.builtFrom === undefined ? {} : { - builtFrom: { - tapRepository: pkg.builtFrom.tapRepository, - tapCommit: pkg.builtFrom.tapCommit, - kandeloRepository: pkg.builtFrom.kandeloRepository, - kandeloCommit: pkg.builtFrom.kandeloCommit, - formulaSha256: pkg.builtFrom.formulaSha256, - }, - }), - })), }, }, - }, maxByteLength); + maxByteLength, + ); const imageCapacity = MemoryFileSystem.readImageCapacity(imageBytes); if (imageCapacity.maxByteLength !== maxByteLength) { throw new Error( `saved VFS capacity ${imageCapacity.maxByteLength} does not match ` + - `the declared consumer contract ${maxByteLength}`, + `the declared consumer contract ${maxByteLength}`, ); } let bottleMirrorOutput: unknown; @@ -505,6 +601,12 @@ export async function runHomebrewVfsImageBuilder( packageTree.state, ); } + if (homebrewBootstrapConsumerState !== undefined) { + assertHomebrewBootstrapConsumerState( + restored, + homebrewBootstrapConsumerState, + ); + } if (shellConfig !== undefined) { assertShellExecutable(restored, shellConfig.config.path); if (restored.isPathDeferred(shellConfig.config.path)) { @@ -545,18 +647,30 @@ export async function runHomebrewVfsImageBuilder( }); mkdirSync(dirname(options.lazyLayerOut), { recursive: true }); mkdirSync(dirname(options.lazyLayerDescriptor), { recursive: true }); - const rootPayload = layer.payloads.find((payload) => payload.id === options.runtimeLayerId); - if (rootPayload === undefined || basename(options.lazyLayerOut) !== rootPayload.asset) { - throw new Error("Homebrew runtime root payload does not match --lazy-layer-out"); + const rootPayload = layer.payloads.find( + (payload) => payload.id === options.runtimeLayerId, + ); + if ( + rootPayload === undefined || + basename(options.lazyLayerOut) !== rootPayload.asset + ) { + throw new Error( + "Homebrew runtime root payload does not match --lazy-layer-out", + ); } for (const payload of layer.payloads) { - writeFileSync(join(dirname(options.lazyLayerOut), payload.asset), payload.bytes); + writeFileSync( + join(dirname(options.lazyLayerOut), payload.asset), + payload.bytes, + ); } writeFileSync( options.lazyLayerDescriptor, encodeHomebrewLazyLayerDescriptor(layer.descriptor), ); - console.log(`Homebrew ${options.runtimeLayerId} runtime layer: ${options.lazyLayerOut}`); + console.log( + `Homebrew ${options.runtimeLayerId} runtime layer: ${options.lazyLayerOut}`, + ); console.log( `Homebrew ${options.runtimeLayerId} runtime layer descriptor: ` + options.lazyLayerDescriptor, @@ -565,37 +679,52 @@ export async function runHomebrewVfsImageBuilder( const report = { ...result.report, - ...(shellConfig ? { - default_shell: { - path: shellConfig.config.path, - argv: shellConfig.config.argv, - config_sha256: shellConfig.sha256, - config_bytes: shellConfig.bytes, - }, - } : {}), - ...(demoConfig ? { - demo_config: { - path: KANDELO_DEMO_CONFIG_PATH, - sha256: demoConfig.sha256, - bytes: demoConfig.bytes, - }, - } : {}), - ...(baseImage ? { - base_image: { - ...baseImage.binding, - metadata: baseImage.metadata, - }, - } : {}), + ...(shellConfig + ? { + default_shell: { + path: shellConfig.config.path, + argv: shellConfig.config.argv, + config_sha256: shellConfig.sha256, + config_bytes: shellConfig.bytes, + }, + } + : {}), + ...(demoConfig + ? { + demo_config: { + path: KANDELO_DEMO_CONFIG_PATH, + sha256: demoConfig.sha256, + bytes: demoConfig.bytes, + }, + } + : {}), + ...(baseImage + ? { + base_image: { + ...baseImage.binding, + metadata: baseImage.metadata, + }, + } + : {}), image_capacity: { byte_length: imageCapacity.byteLength, max_byte_length: imageCapacity.maxByteLength, }, - ...(bottleMirrorOutput === undefined ? {} : { - bottle_mirror: bottleMirrorOutput, - }), - ...(packageTree === undefined ? {} : { - package_deferred_trees: [packageTreeBinding(packageTree)], - }), + ...(bottleMirrorOutput === undefined + ? {} + : { + bottle_mirror: bottleMirrorOutput, + }), + ...(packageTree === undefined + ? {} + : { + package_deferred_trees: [packageTreeBinding(packageTree)], + }), + ...(homebrewBootstrapConsumerState === undefined + ? {} + : { + homebrew_bootstrap: homebrewBootstrapConsumerState, + }), // Report a reproducible artifact identity, not a runner/worktree path. image: basename(options.out), }; @@ -665,7 +794,8 @@ function parseArgs(args: string[]): CliOptions { break; case "--expected-cache-key": { const [name, sha] = requireValue(args, ++i, arg).split("=", 2); - if (!name || !sha) usage(`--expected-cache-key must be =`); + if (!name || !sha) + usage(`--expected-cache-key must be =`); options.expectedCacheKeys[name] = sha; break; } @@ -776,6 +906,12 @@ function parseArgs(args: string[]): CliOptions { } options.packageTreeArchive = requireValue(args, ++i, arg); break; + case "--homebrew-bootstrap-env": + if (options.homebrewBootstrapEnv !== undefined) { + usage("--homebrew-bootstrap-env may be provided only once"); + } + options.homebrewBootstrapEnv = requireValue(args, ++i, arg); + break; case "--materialize-package-tree": if (options.materializePackageTree) { usage("--materialize-package-tree may be provided only once"); @@ -792,25 +928,40 @@ function parseArgs(args: string[]): CliOptions { } for (const required of ["metadata", "tapRoot", "out", "report"] as const) { - if (!options[required]) usage(`missing --${required.replace(/[A-Z]/g, (c) => `-${c.toLowerCase()}`)}`); + if (!options[required]) + usage( + `missing --${required.replace(/[A-Z]/g, (c) => `-${c.toLowerCase()}`)}`, + ); } if (options.brewfile && options.packages.length > 0) { usage("--brewfile cannot be combined with --package"); } if (!options.brewfile && options.packages.length === 0) { - usage("exactly one package selection mode is required: --brewfile or --package"); + usage( + "exactly one package selection mode is required: --brewfile or --package", + ); } if (options.baseImage && !existsSync(options.baseImage)) { usage(`base image does not exist: ${options.baseImage}`); } if (options.shellConfig && !options.writeProfile) { - usage("--shell-config requires --write-profile so the Homebrew environment is initialized"); + usage( + "--shell-config requires --write-profile so the Homebrew environment is initialized", + ); } - if (options.catalogCommit !== undefined && !GIT_SHA_RE.test(options.catalogCommit)) { + if ( + options.catalogCommit !== undefined && + !GIT_SHA_RE.test(options.catalogCommit) + ) { usage("--catalog-commit must be a lowercase 40-character git SHA"); } - if (options.catalogCommit !== undefined && Object.keys(options.dependencyTapRoots).length > 0) { - usage("--catalog-commit currently supports only a single-tap catalog checkout"); + if ( + options.catalogCommit !== undefined && + Object.keys(options.dependencyTapRoots).length > 0 + ) { + usage( + "--catalog-commit currently supports only a single-tap catalog checkout", + ); } if (options.migrationLock && !existsSync(options.migrationLock)) { usage(`migration lock does not exist: ${options.migrationLock}`); @@ -833,28 +984,64 @@ function parseArgs(args: string[]): CliOptions { options.materializationPolicy !== undefined && !existsSync(options.materializationPolicy) ) { - usage(`materialization policy does not exist: ${options.materializationPolicy}`); + usage( + `materialization policy does not exist: ${options.materializationPolicy}`, + ); } - if (options.bottleMirrorOut !== undefined && existsSync(options.bottleMirrorOut)) { - usage(`bottle mirror output must not already exist: ${options.bottleMirrorOut}`); + if ( + options.bottleMirrorOut !== undefined && + existsSync(options.bottleMirrorOut) + ) { + usage( + `bottle mirror output must not already exist: ${options.bottleMirrorOut}`, + ); } - if (Boolean(options.packageTreeSpec) !== Boolean(options.packageTreeArchive)) { - usage("--package-tree-spec and --package-tree-archive must be provided together"); + if ( + Boolean(options.packageTreeSpec) !== Boolean(options.packageTreeArchive) + ) { + usage( + "--package-tree-spec and --package-tree-archive must be provided together", + ); } if (options.materializePackageTree && options.packageTreeSpec === undefined) { usage("--materialize-package-tree requires a package tree"); } - if (options.packageTreeSpec !== undefined && !existsSync(options.packageTreeSpec)) { + if ( + options.homebrewBootstrapEnv !== undefined && + options.packageTreeSpec === undefined + ) { + usage("--homebrew-bootstrap-env requires a package tree"); + } + if ( + options.packageTreeSpec !== undefined && + !existsSync(options.packageTreeSpec) + ) { usage(`package tree spec does not exist: ${options.packageTreeSpec}`); } - if (options.packageTreeArchive !== undefined && !existsSync(options.packageTreeArchive)) { + if ( + options.packageTreeArchive !== undefined && + !existsSync(options.packageTreeArchive) + ) { usage(`package tree archive does not exist: ${options.packageTreeArchive}`); } - if (options.materializationPolicy !== undefined && options.lazyLayerOut !== undefined) { + if ( + options.homebrewBootstrapEnv !== undefined && + !existsSync(options.homebrewBootstrapEnv) + ) { + usage( + `Homebrew bootstrap environment does not exist: ${options.homebrewBootstrapEnv}`, + ); + } + if ( + options.materializationPolicy !== undefined && + options.lazyLayerOut !== undefined + ) { usage("materialized shell composition cannot also emit a runtime layer"); } if (Boolean(options.lazyLayerOut) !== Boolean(options.lazyLayerDescriptor)) { - usage("--lazy-layer-out and --lazy-layer-descriptor must be provided together"); + usage( + "--lazy-layer-out and --lazy-layer-descriptor must be provided together", + ); } if ( options.lazyLayerOut && @@ -882,7 +1069,9 @@ function parseArgs(args: string[]): CliOptions { usage("--runtime-layer-policy requires lazy layer outputs"); } if (options.lazyLayerBaseImage && !existsSync(options.lazyLayerBaseImage)) { - usage(`lazy layer base image does not exist: ${options.lazyLayerBaseImage}`); + usage( + `lazy layer base image does not exist: ${options.lazyLayerBaseImage}`, + ); } if ( options.lazyLayerBasePackageSource && @@ -897,7 +1086,9 @@ function parseArgs(args: string[]): CliOptions { usage(`runtime layer policy does not exist: ${options.runtimeLayerPolicy}`); } if (options.lazyLayerOut && options.runtimeLayerId) { - const payloadAsset = homebrewRuntimeLayerPayloadAsset(options.runtimeLayerId); + const payloadAsset = homebrewRuntimeLayerPayloadAsset( + options.runtimeLayerId, + ); const descriptorAsset = homebrewRuntimeLayerDescriptorAsset( options.runtimeLayerId, ); @@ -944,10 +1135,13 @@ function parseByteSize(value: string): number { if (!match) usage(`--max-bytes must be a positive byte size, got ${value}`); const amount = Number(match[1]); const suffix = (match[2] ?? "b").toLowerCase(); - const multiplier = suffix.startsWith("g") ? 1024 ** 3 - : suffix.startsWith("m") ? 1024 ** 2 - : suffix.startsWith("k") ? 1024 - : 1; + const multiplier = suffix.startsWith("g") + ? 1024 ** 3 + : suffix.startsWith("m") + ? 1024 ** 2 + : suffix.startsWith("k") + ? 1024 + : 1; const bytes = amount * multiplier; if (!Number.isSafeInteger(bytes) || bytes <= 0) { usage(`--max-bytes is too large: ${value}`); @@ -968,11 +1162,263 @@ function parseBaseImagePath(value: string): string { function readPackageTreeArchive(path: string): Uint8Array { const stat = lstatSync(path); if (!stat.isFile() || stat.isSymbolicLink() || stat.size <= 0) { - throw new Error(`package tree archive is not a nonempty regular file: ${path}`); + throw new Error( + `package tree archive is not a nonempty regular file: ${path}`, + ); } return new Uint8Array(readFileSync(path)); } +export function readHomebrewBootstrapEnvironment( + path: string, + arch: HomebrewBottleArch, +): Uint8Array { + const bytes = readBoundedRegularFile( + path, + MAX_HOMEBREW_BOOTSTRAP_ENV_BYTES, + "Homebrew bootstrap environment", + ); + let source: string; + try { + source = new TextDecoder("utf-8", { fatal: true }).decode(bytes); + } catch (error) { + throw new Error("Homebrew bootstrap environment is not valid UTF-8", { + cause: error, + }); + } + const expected = [ + "HOMEBREW_NO_ANALYTICS=1", + "HOMEBREW_NO_AUTO_UPDATE=1", + "HOMEBREW_SYSTEM_ENV_TAKES_PRIORITY=1", + `HOMEBREW_KANDELO_BOTTLE_TAG=${arch}_kandelo`, + "", + ].join("\n"); + if (source !== expected) { + throw new Error( + `Homebrew bootstrap environment does not select ${arch}_kandelo with system precedence`, + ); + } + return bytes; +} + +export function installHomebrewBootstrapConsumerState( + fs: MemoryFileSystem, + tree: DerivedPackageDeferredZipTree, + environment: Uint8Array, +): HomebrewBootstrapConsumerState { + const descriptor = tree.descriptor; + if ( + descriptor.content_role !== "source-tree" || + descriptor.package.name !== "homebrew-bootstrap" || + descriptor.mount_prefix !== "/home/linuxbrew/.linuxbrew" || + !descriptor.activation.roots.includes(HOMEBREW_BOOTSTRAP_TARGET) || + !descriptor.inventory.some( + (entry) => + entry.vfs_path === HOMEBREW_BOOTSTRAP_TARGET && + entry.type === "file" && + (entry.mode & 0o111) !== 0, + ) + ) { + throw new Error( + "Homebrew bootstrap environment requires the canonical deferred source tree", + ); + } + // WHY: descriptor ownership alone does not prove the registered VFS still + // contains its activation root. Check before writing consumer state so a + // deleted tree member cannot leave a new dangling /usr/bin/brew alias. + assertHomebrewBootstrapTarget(fs, HOMEBREW_BOOTSTRAP_TARGET); + for (const path of [ + HOMEBREW_BOOTSTRAP_ENV_PATH, + HOMEBREW_BOOTSTRAP_ENTRYPOINT, + ]) { + if (vfsPathExists(fs, path)) { + throw new Error( + `refusing to replace Homebrew bootstrap consumer state: ${path}`, + ); + } + } + ensureDirRecursive(fs, dirname(HOMEBREW_BOOTSTRAP_ENV_PATH)); + writeVfsBinary(fs, HOMEBREW_BOOTSTRAP_ENV_PATH, environment, 0o644); + // WHY: Homebrew derives its canonical Kandelo prefix from this public alias. + // Pointing PATH straight at bin/brew appears to work but bypasses that + // launcher contract and can select the wrong prefix or bottle tag. + fs.symlink(HOMEBREW_BOOTSTRAP_TARGET, HOMEBREW_BOOTSTRAP_ENTRYPOINT); + const state: HomebrewBootstrapConsumerState = { + environment: { + path: HOMEBREW_BOOTSTRAP_ENV_PATH, + sha256: createHash("sha256").update(environment).digest("hex"), + bytes: environment.byteLength, + }, + entrypoint: { + path: HOMEBREW_BOOTSTRAP_ENTRYPOINT, + target: HOMEBREW_BOOTSTRAP_TARGET, + }, + ownership: { + prefix: HOMEBREW_BOOTSTRAP_PREFIX, + uid: 1000, + gid: 1000, + mutable_paths: [...HOMEBREW_BOOTSTRAP_MUTABLE_PATHS], + }, + }; + assertHomebrewBootstrapConsumerState(fs, state); + return state; +} + +export function assertHomebrewBootstrapConsumerState( + fs: MemoryFileSystem, + expected: HomebrewBootstrapConsumerState, +): void { + const environment = readVfsBinary(fs, expected.environment.path); + const environmentStat = fs.lstat(expected.environment.path); + if ( + environment.byteLength !== expected.environment.bytes || + createHash("sha256").update(environment).digest("hex") !== + expected.environment.sha256 || + (environmentStat.mode & 0xf000) !== 0x8000 || + (environmentStat.mode & 0o7777) !== 0o644 || + environmentStat.uid !== 0 || + environmentStat.gid !== 0 + ) { + throw new Error("Homebrew bootstrap system environment changed in the VFS"); + } + const stat = fs.lstat(expected.entrypoint.path); + if ( + (stat.mode & 0xf000) !== 0xa000 || + (stat.mode & 0o7777) !== 0o777 || + stat.uid !== 0 || + stat.gid !== 0 || + fs.readlink(expected.entrypoint.path) !== expected.entrypoint.target + ) { + throw new Error("Homebrew bootstrap entrypoint changed in the VFS"); + } + assertHomebrewBootstrapTarget(fs, expected.entrypoint.target); + assertVfsTreeOwner( + fs, + expected.ownership.prefix, + expected.ownership.uid, + expected.ownership.gid, + ); + for (const path of expected.ownership.mutable_paths) { + const mutable = fs.lstat(path); + if ( + (mutable.mode & 0xf000) !== 0x4000 || + mutable.uid !== expected.ownership.uid || + mutable.gid !== expected.ownership.gid + ) { + throw new Error(`Homebrew mutable path has the wrong owner: ${path}`); + } + } +} + +function assertHomebrewBootstrapTarget( + fs: MemoryFileSystem, + path: string, +): void { + try { + const target = fs.stat(path); + if ((target.mode & 0xf000) !== 0x8000 || (target.mode & 0o111) === 0) { + throw new Error("target is not an executable regular file"); + } + } catch (error) { + throw new Error( + "Homebrew bootstrap environment requires the canonical deferred source tree", + { cause: error }, + ); + } +} + +export function prepareHomebrewBootstrapConsumerNamespace( + fs: MemoryFileSystem, + tree: DerivedPackageDeferredZipTree, +): void { + if ( + tree.descriptor.package.name !== "homebrew-bootstrap" || + tree.descriptor.mount_prefix !== HOMEBREW_BOOTSTRAP_PREFIX + ) { + throw new Error( + "Homebrew bootstrap ownership requires the canonical deferred source tree", + ); + } + for (const path of HOMEBREW_BOOTSTRAP_MUTABLE_PATHS) { + ensureDirRecursive(fs, path); + } + // WHY: a real Linuxbrew installation belongs to the unprivileged brew user. + // Bottle composition initially creates structural prefix directories as + // root; adopting the complete prefix here both avoids a false lazy-tree + // collision and lets in-guest brew update Cellar, taps, links, and locks. + chownVfsTree(fs, HOMEBREW_BOOTSTRAP_PREFIX, 1000, 1000); + chownVfsTree(fs, "/home/user/.cache", 1000, 1000); +} + +function chownVfsTree( + fs: MemoryFileSystem, + root: string, + uid: number, + gid: number, +): void { + fs.lchown(root, uid, gid); + if ((fs.lstat(root).mode & 0xf000) !== 0x4000) return; + const handle = fs.opendir(root); + try { + for (;;) { + const entry = fs.readdir(handle); + if (entry === null) break; + if (entry.name === "." || entry.name === "..") continue; + const path = root === "/" ? `/${entry.name}` : `${root}/${entry.name}`; + chownVfsTree(fs, path, uid, gid); + } + } finally { + fs.closedir(handle); + } +} + +function assertVfsTreeOwner( + fs: MemoryFileSystem, + root: string, + uid: number, + gid: number, +): void { + const stat = fs.lstat(root); + if (stat.uid !== uid || stat.gid !== gid) { + throw new Error(`Homebrew prefix entry has the wrong owner: ${root}`); + } + if ((stat.mode & 0xf000) !== 0x4000) return; + const handle = fs.opendir(root); + try { + for (;;) { + const entry = fs.readdir(handle); + if (entry === null) break; + if (entry.name === "." || entry.name === "..") continue; + const path = root === "/" ? `/${entry.name}` : `${root}/${entry.name}`; + assertVfsTreeOwner(fs, path, uid, gid); + } + } finally { + fs.closedir(handle); + } +} + +function readVfsBinary(fs: MemoryFileSystem, path: string): Uint8Array { + const stat = fs.stat(path); + const fd = fs.open(path, 0, 0); + try { + const bytes = new Uint8Array(stat.size); + let offset = 0; + while (offset < bytes.byteLength) { + const count = fs.read( + fd, + bytes.subarray(offset), + null, + bytes.byteLength - offset, + ); + if (count <= 0) throw new Error(`short read from VFS file: ${path}`); + offset += count; + } + return bytes; + } finally { + fs.close(fd); + } +} + function packageTreeBinding(tree: { derived: DerivedPackageDeferredZipTree; state: "deferred" | "materialized"; @@ -1007,7 +1453,11 @@ function createFs( baseImage: string | undefined, maxBytes: number | undefined, expectedAbi: number, -): { fs: MemoryFileSystem; baseImage?: LoadedBaseImage; maxByteLength: number } { +): { + fs: MemoryFileSystem; + baseImage?: LoadedBaseImage; + maxByteLength: number; +} { if (baseImage) { const image = new Uint8Array(readFileSync(baseImage)); const restored = MemoryFileSystem.fromImagePreservingCapacity(image); @@ -1020,7 +1470,7 @@ function createFs( if (metadata.kernelAbi !== expectedAbi) { throw new Error( `base image ${baseImage} declares kernel ABI ${metadata.kernelAbi}, ` + - `but bottle metadata requires ABI ${expectedAbi}`, + `but bottle metadata requires ABI ${expectedAbi}`, ); } if ( @@ -1029,7 +1479,7 @@ function createFs( ) { throw new Error( `base image ${baseImage} already contains a Homebrew composition; ` + - "use a platform-only base image", + "use a platform-only base image", ); } @@ -1053,7 +1503,7 @@ function createFs( if (targetMaxBytes !== recordedMaxBytes) { console.log( `Rebasing base VFS capacity from ${formatMib(recordedMaxBytes)} ` + - `to ${formatMib(targetMaxBytes)}...`, + `to ${formatMib(targetMaxBytes)}...`, ); return { fs: restored.rebaseToNewFileSystem(targetMaxBytes), @@ -1181,7 +1631,11 @@ function migrationLockCompatibilityPolicy( value: unknown, path: string, ): HomebrewVfsCompatibilityPolicy { - if (!isRecord(value) || value.schema !== 1 || !isRecord(value.compatibility)) { + if ( + !isRecord(value) || + value.schema !== 1 || + !isRecord(value.compatibility) + ) { throw new Error(`Homebrew migration lock has an invalid schema: ${path}`); } const compatibility = value.compatibility; @@ -1196,25 +1650,29 @@ function migrationLockCompatibilityPolicy( (compatibility.runtime_state !== undefined && !Array.isArray(compatibility.runtime_state)) ) { - throw new Error(`Homebrew migration lock has an invalid compatibility policy: ${path}`); + throw new Error( + `Homebrew migration lock has an invalid compatibility policy: ${path}`, + ); } - const linkConflictOwners = compatibility.link_conflict_owners.map((value, index) => { - if ( - !isRecord(value) || - typeof value.target !== "string" || - typeof value.package !== "string" || - typeof value.reason !== "string" - ) { - throw new Error( - `Homebrew migration lock compatibility.link_conflict_owners[${index}] is invalid: ${path}`, - ); - } - return { - target: value.target, - package: value.package, - reason: value.reason, - }; - }); + const linkConflictOwners = compatibility.link_conflict_owners.map( + (value, index) => { + if ( + !isRecord(value) || + typeof value.target !== "string" || + typeof value.package !== "string" || + typeof value.reason !== "string" + ) { + throw new Error( + `Homebrew migration lock compatibility.link_conflict_owners[${index}] is invalid: ${path}`, + ); + } + return { + target: value.target, + package: value.package, + reason: value.reason, + }; + }, + ); const aliases = compatibility.aliases.map< HomebrewVfsCompatibilityPolicy["aliases"][number] >((value, index) => { @@ -1237,9 +1695,9 @@ function migrationLockCompatibilityPolicy( targets: [...value.targets], }; }); - const runtimeState = (compatibility.runtime_state ?? []).map< - HomebrewVfsRuntimeStateDeclaration - >((value, index) => { + const runtimeState = ( + compatibility.runtime_state ?? [] + ).map((value, index) => { if (!isRecord(value)) { throw new Error( `Homebrew migration lock compatibility.runtime_state[${index}] is invalid: ${path}`, @@ -1280,7 +1738,9 @@ function migrationLockCompatibilityPolicy( uid: value.uid, gid: value.gid, reason: value.reason, - ...(value.kind === "text_file" ? { contents: value.contents as string } : {}), + ...(value.kind === "text_file" + ? { contents: value.contents as string } + : {}), }; }); return { @@ -1302,7 +1762,9 @@ function readShellConfig(path: string): LoadedShellConfig { const source = new TextDecoder("utf-8", { fatal: true }).decode(bytes); const config = parseKandeloShellConfig(source); if (!config) { - throw new Error(`Kandelo default shell config has an unsupported version: ${path}`); + throw new Error( + `Kandelo default shell config has an unsupported version: ${path}`, + ); } return { config, @@ -1352,13 +1814,19 @@ function assertShellExecutable(fs: MemoryFileSystem, path: string): void { try { stat = fs.stat(path); } catch { - throw new Error(`default shell executable is missing from the composed VFS: ${path}`); + throw new Error( + `default shell executable is missing from the composed VFS: ${path}`, + ); } if ((stat.mode & 0xf000) !== 0x8000) { - throw new Error(`default shell path is not a regular file in the composed VFS: ${path}`); + throw new Error( + `default shell path is not a regular file in the composed VFS: ${path}`, + ); } if ((stat.mode & 0o111) === 0) { - throw new Error(`default shell is not executable in the composed VFS: ${path}`); + throw new Error( + `default shell is not executable in the composed VFS: ${path}`, + ); } if (stat.size > MAX_KANDELO_SHELL_EXECUTABLE_BYTES) { throw new Error( @@ -1395,7 +1863,8 @@ function readBrewfileSelection(path: string): BrewfileSelection { throw new Error(`cannot parse Brewfile ${path}: ${parsed.error.message}`); } if (parsed.status !== 0) { - const detail = parsed.stderr.trim() || + const detail = + parsed.stderr.trim() || `parser exited with status ${String(parsed.status)}`; throw new Error(`cannot parse Brewfile ${path}: ${detail}`); } @@ -1409,12 +1878,23 @@ function readBrewfileSelection(path: string): BrewfileSelection { if (!isRecord(value)) { throw new Error(`Brewfile parser returned a non-object for ${path}`); } - const expectedKeys = ["bytes", "kind", "packages", "schema", "sha256", "tap_name"]; + const expectedKeys = [ + "bytes", + "kind", + "packages", + "schema", + "sha256", + "tap_name", + ]; if (Object.keys(value).sort().join("\0") !== expectedKeys.join("\0")) { - throw new Error(`Brewfile parser returned an unsupported result shape for ${path}`); + throw new Error( + `Brewfile parser returned an unsupported result shape for ${path}`, + ); } if (value.schema !== 1 || value.kind !== "kandelo-static-brewfile-v1") { - throw new Error(`Brewfile parser returned an unsupported schema for ${path}`); + throw new Error( + `Brewfile parser returned an unsupported schema for ${path}`, + ); } if (typeof value.tap_name !== "string" || !TAP_NAME_RE.test(value.tap_name)) { throw new Error(`Brewfile parser returned an invalid tap name for ${path}`); @@ -1428,18 +1908,22 @@ function readBrewfileSelection(path: string): BrewfileSelection { value.bytes <= 0 || value.bytes > MAX_BREWFILE_BYTES ) { - throw new Error(`Brewfile parser returned an invalid byte count for ${path}`); + throw new Error( + `Brewfile parser returned an invalid byte count for ${path}`, + ); } if ( !Array.isArray(value.packages) || value.packages.length === 0 || value.packages.length > MAX_BREWFILE_PACKAGES || - value.packages.some((pkg) => - typeof pkg !== "string" || !PACKAGE_NAME_RE.test(pkg) + value.packages.some( + (pkg) => typeof pkg !== "string" || !PACKAGE_NAME_RE.test(pkg), ) || new Set(value.packages).size !== value.packages.length ) { - throw new Error(`Brewfile parser returned invalid requested packages for ${path}`); + throw new Error( + `Brewfile parser returned invalid requested packages for ${path}`, + ); } return value as unknown as BrewfileSelection; } @@ -1498,6 +1982,7 @@ function usage(message?: string, code = 2): never { --bottle-mirror-out ] \\ [--package-tree-spec \\ --package-tree-archive \\ + [--homebrew-bootstrap-env ] \\ [--materialize-package-tree]] \\ [--lazy-layer-out \\ --lazy-layer-descriptor \\ diff --git a/packages/registry/homebrew-bootstrap/build-homebrew-bootstrap.sh b/packages/registry/homebrew-bootstrap/build-homebrew-bootstrap.sh index 4a59f2f5ee..28a53b8b74 100755 --- a/packages/registry/homebrew-bootstrap/build-homebrew-bootstrap.sh +++ b/packages/registry/homebrew-bootstrap/build-homebrew-bootstrap.sh @@ -124,11 +124,14 @@ PROVENANCE="$BUILD_DIR/homebrew-source.json" --provenance "$PROVENANCE" OUTPUT="$KANDELO_PACKAGE_OUT_DIR/homebrew-bootstrap.zip" -if [ -e "$OUTPUT" ] || [ -L "$OUTPUT" ]; then - echo "ERROR: homebrew-bootstrap output already exists: $OUTPUT" >&2 +ENV_OUTPUT="$KANDELO_PACKAGE_OUT_DIR/homebrew-brew.env" +if [ -e "$OUTPUT" ] || [ -L "$OUTPUT" ] || + [ -e "$ENV_OUTPUT" ] || [ -L "$ENV_OUTPUT" ]; then + echo "ERROR: homebrew-bootstrap output already exists" >&2 exit 1 fi cp "$ARCHIVE" "$OUTPUT" +cp "$ENV_FILE" "$ENV_OUTPUT" node "$VERIFY" \ --lock "$LOCK" \ --package-name "$PACKAGE_NAME" \ @@ -144,4 +147,4 @@ node "$VERIFY" \ --provenance "$PROVENANCE" \ --archive "$OUTPUT" -echo "==> Built provenance-locked Homebrew bootstrap: $OUTPUT" +echo "==> Built provenance-locked Homebrew bootstrap: $OUTPUT + $ENV_OUTPUT" diff --git a/packages/registry/homebrew-bootstrap/build.toml b/packages/registry/homebrew-bootstrap/build.toml index 15a37e81fb..93fed058d3 100644 --- a/packages/registry/homebrew-bootstrap/build.toml +++ b/packages/registry/homebrew-bootstrap/build.toml @@ -10,7 +10,7 @@ inputs = [ ] repo_url = "https://github.com/Automattic/kandelo.git" commit = "UNPUBLISHED" -revision = 1 +revision = 2 [[git_inputs]] name = "homebrew_brew" diff --git a/packages/registry/homebrew-bootstrap/package.toml b/packages/registry/homebrew-bootstrap/package.toml index 3e1524fc4b..776f0f8f99 100644 --- a/packages/registry/homebrew-bootstrap/package.toml +++ b/packages/registry/homebrew-bootstrap/package.toml @@ -25,6 +25,13 @@ name = "homebrew-bootstrap" wasm = "homebrew-bootstrap.zip" fork_instrumentation = "disabled" +# Keep the launcher policy in the same atomic package publication as the +# source tree. Consumers must not recreate the bottle tag or precedence rules. +[[outputs]] +name = "homebrew-brew" +wasm = "homebrew-brew.env" +fork_instrumentation = "disabled" + [[host_tools]] name = "git" version_constraint = ">=2.30" diff --git a/packages/registry/program-packages.json b/packages/registry/program-packages.json index 16cb27b045..8d4501cacb 100644 --- a/packages/registry/program-packages.json +++ b/packages/registry/program-packages.json @@ -128,10 +128,10 @@ } }, "homebrew-bootstrap": { - "manifestSha256": "9446c30113764de79abe29df79586849ab4f296d723fcf9614f45d885d114388", + "manifestSha256": "b171060c86cb6642e8cadef9d2698e671c6bf188868c7d835b3e8ecf3a3b054a", "cacheKeys": { - "wasm32": "3f44ee7f53ebf6d26e30f1dbb332484753df28728e341ba4473d73f16e6a6ebf", - "wasm64": "32c128e0b63f160a4d37e0b1cb025a1a28c6c926a5556758cf936e1edee722c6" + "wasm32": "8230aaca4bfc1f7bff80d2830534151afff6edb2c03f640af4afa385d145426d", + "wasm64": "fa191c56e67b1e59d9f9b0b56ff1914c2dc584a1a39a8763cc4cb82bd890a038" } }, "icu": { @@ -158,8 +158,8 @@ "lamp": { "manifestSha256": "250b64635d64f8537178188fb5488dbfd2980d79a1c2ffbf346af67008088ba0", "cacheKeys": { - "wasm32": "52b8525aaf27598d1d89e1b3ed0f84f9f7a415052c246d632eb8fda2b4a1fa99", - "wasm64": "2b72b49cf85f25af65fa88a65c552b2aa86bb0953f3f284f551a97b6720673cf" + "wasm32": "89b0d8b7a8c6ffdefa8d3134015524384f6176203e18428c865f7ebee95eb026", + "wasm64": "e8f79f3bae1ea103840a83501e6de1dc4fd52daf1295181aa4eea4762e7d4d0c" } }, "less": { @@ -312,15 +312,15 @@ "nginx-php-vfs": { "manifestSha256": "97976410cb02f8ba710d856b4ac904bcf976d677b50eefdee39fb64176070d4b", "cacheKeys": { - "wasm32": "a87b91753dc5369b7a4ac5f8e57aa0018ac99d8ad2fcd3156656aa152c868718", - "wasm64": "8c05a763de5f2b0f3c76b9fc11a89ca99ada21398fdf8255f7c928e9af45b395" + "wasm32": "404deaf3939a3c04b102117c5c5df277f2e0107ca8e6c66d034ee45a2da533e8", + "wasm64": "c1a0c78c0d23014fa04d93e2366b8da5fa4fd9df91a8f4aeb72563a6a881833b" } }, "nginx-vfs": { "manifestSha256": "46aa2d3250ac5cd0f102a85c3a36a086c7a50ba1f9e05fa1c2aae3d07ad16f10", "cacheKeys": { - "wasm32": "ca664401a0ead71d0761b6f3e15126285d239344ff918cbc28c3d54d0809f01b", - "wasm64": "38db505d84f3bb337960da2f6e8e5e65c80874d8a9f80f992ad9549cdb6c1a47" + "wasm32": "6730d4fb915fc43aac3dd123329d115e70bce84fb6380ae28ad4b50d2aa9704e", + "wasm64": "ac8b6023043edaaa45456ce93c4126586f75ac5110e639190b66a487fea80824" } }, "node": { @@ -333,8 +333,8 @@ "node-vfs": { "manifestSha256": "33315fb1b3030a4c187ae075eac08f717de8d7ab017b86c6458778ac9070eece", "cacheKeys": { - "wasm32": "3c15243220cac0639aed4b8952f9a09a39036ea4d01b2e75949ebdbacf796d02", - "wasm64": "6be87229496865f993a2fbd8d27de139f35ce63375c16cc0fcfe5ecdc804b3aa" + "wasm32": "b069a0aa069fa91a9896cdfc4df82cfa653f2859befde886e4a22377a2326893", + "wasm64": "44b56a6e095ef4abc3e63a939abee6610367fd2cc3b17e2cd47ca1903e9e89df" } }, "openssl": { @@ -396,8 +396,8 @@ "redis-vfs": { "manifestSha256": "234c45c40f94e2a89f98295313b82cb9fce15a412ac829d9e87394e0dc731813", "cacheKeys": { - "wasm32": "dfbdea8ae43c488ebafdbb4e83cb5f4dd6bf0810fadd95ede4c1ced77174feb1", - "wasm64": "a257d8a55786e4faef259a3320f5177aed8307172d52ad2f5b07a4f0ef598146" + "wasm32": "bc95db71f36cf184d9b7f254e567616c0d6f41b554e4bd2930b45acb6740f1bb", + "wasm64": "2e8840dde56f036c4472004ec5da886637507847aef0500690c1f8c62f466950" } }, "rootfs": { @@ -422,10 +422,10 @@ } }, "shell": { - "manifestSha256": "5d02cd8fee4e0d46c7552a3046c3898f9e3bd5667094e02cab01851eed97a321", + "manifestSha256": "775d46b5252f6d94fb1ce04c97d63088df7574c54fdb3cbbb5135326352aa069", "cacheKeys": { - "wasm32": "308558a09c4798e2c09083da90be88ce752307cde833d2a0f036b6a61bd28e18", - "wasm64": "e568a6050479b0ab7eea3a0c6faa38c25660ab247b764e5b5df9f66d18b526e9" + "wasm32": "a2a011cb27f2b3e351c6d48c19308ff849a52a5b982b778bf1a898410b03d03d", + "wasm64": "8bb4a493d5eeaf1fc58576079a6d1bc655826f55e0010551fd311a25dfae97ab" } }, "spidermonkey": { @@ -515,8 +515,8 @@ "wordpress": { "manifestSha256": "36465e0596a06e855a8524eecfd87da98643bc13b37ee12939f5aab44bf33418", "cacheKeys": { - "wasm32": "225610679e2e521c0d861e2c8a3c36bb1a24c682e97ce909b1b0e786a4147aa0", - "wasm64": "b811c0fe3bb7743061ffc5f3504665845034191104c0bd16fadb8742fbc3425b" + "wasm32": "f4fd748f475cab4727977a718271952fa9accecdc3b05f71081303b6cd956481", + "wasm64": "8bcffbf89f747f0fe7335d0ac1028c860862fec7ad23b70270c2e09aca8dce35" } }, "xz": { @@ -1033,12 +1033,12 @@ ] }, "homebrew-bootstrap": { - "manifestSha256": "9446c30113764de79abe29df79586849ab4f296d723fcf9614f45d885d114388", + "manifestSha256": "b171060c86cb6642e8cadef9d2698e671c6bf188868c7d835b3e8ecf3a3b054a", "arches": [ "wasm32" ], "cacheKeys": { - "wasm32": "3f44ee7f53ebf6d26e30f1dbb332484753df28728e341ba4473d73f16e6a6ebf" + "wasm32": "8230aaca4bfc1f7bff80d2830534151afff6edb2c03f640af4afa385d145426d" }, "dependencyClosures": { "wasm32": [] @@ -1047,9 +1047,16 @@ { "kind": "output", "sourceArtifact": "homebrew-bootstrap.zip", - "mirrorPath": "homebrew-bootstrap.zip", + "mirrorPath": "homebrew-bootstrap/homebrew-bootstrap.zip", "outputName": "homebrew-bootstrap", "forkInstrumentation": "disabled" + }, + { + "kind": "output", + "sourceArtifact": "homebrew-brew.env", + "mirrorPath": "homebrew-bootstrap/homebrew-brew.env", + "outputName": "homebrew-brew", + "forkInstrumentation": "disabled" } ] }, @@ -1086,7 +1093,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "52b8525aaf27598d1d89e1b3ed0f84f9f7a415052c246d632eb8fda2b4a1fa99" + "wasm32": "89b0d8b7a8c6ffdefa8d3134015524384f6176203e18428c865f7ebee95eb026" }, "dependencyClosures": { "wasm32": [ @@ -1095,6 +1102,11 @@ "manifestSha256": "44e125e1337503e27f8cff531cdbd79c4f406fd2d282b916a7746cf3a14057bf", "cacheKey": "28167e431807693bad67358e88d98cfb0603fd2d6efd9808bb051544846c5364" }, + { + "packageName": "homebrew-bootstrap", + "manifestSha256": "b171060c86cb6642e8cadef9d2698e671c6bf188868c7d835b3e8ecf3a3b054a", + "cacheKey": "8230aaca4bfc1f7bff80d2830534151afff6edb2c03f640af4afa385d145426d" + }, { "packageName": "icu", "manifestSha256": "f5b1f02d169ec1108fc32d9da55be53c3efb96e3a7aeba6a03aec62e0175f0dc", @@ -1157,8 +1169,8 @@ }, { "packageName": "shell", - "manifestSha256": "5d02cd8fee4e0d46c7552a3046c3898f9e3bd5667094e02cab01851eed97a321", - "cacheKey": "308558a09c4798e2c09083da90be88ce752307cde833d2a0f036b6a61bd28e18" + "manifestSha256": "775d46b5252f6d94fb1ce04c97d63088df7574c54fdb3cbbb5135326352aa069", + "cacheKey": "a2a011cb27f2b3e351c6d48c19308ff849a52a5b982b778bf1a898410b03d03d" }, { "packageName": "sqlite", @@ -1711,7 +1723,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "a87b91753dc5369b7a4ac5f8e57aa0018ac99d8ad2fcd3156656aa152c868718" + "wasm32": "404deaf3939a3c04b102117c5c5df277f2e0107ca8e6c66d034ee45a2da533e8" }, "dependencyClosures": { "wasm32": [ @@ -1720,6 +1732,11 @@ "manifestSha256": "44e125e1337503e27f8cff531cdbd79c4f406fd2d282b916a7746cf3a14057bf", "cacheKey": "28167e431807693bad67358e88d98cfb0603fd2d6efd9808bb051544846c5364" }, + { + "packageName": "homebrew-bootstrap", + "manifestSha256": "b171060c86cb6642e8cadef9d2698e671c6bf188868c7d835b3e8ecf3a3b054a", + "cacheKey": "8230aaca4bfc1f7bff80d2830534151afff6edb2c03f640af4afa385d145426d" + }, { "packageName": "icu", "manifestSha256": "f5b1f02d169ec1108fc32d9da55be53c3efb96e3a7aeba6a03aec62e0175f0dc", @@ -1772,8 +1789,8 @@ }, { "packageName": "shell", - "manifestSha256": "5d02cd8fee4e0d46c7552a3046c3898f9e3bd5667094e02cab01851eed97a321", - "cacheKey": "308558a09c4798e2c09083da90be88ce752307cde833d2a0f036b6a61bd28e18" + "manifestSha256": "775d46b5252f6d94fb1ce04c97d63088df7574c54fdb3cbbb5135326352aa069", + "cacheKey": "a2a011cb27f2b3e351c6d48c19308ff849a52a5b982b778bf1a898410b03d03d" }, { "packageName": "sqlite", @@ -1803,7 +1820,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "ca664401a0ead71d0761b6f3e15126285d239344ff918cbc28c3d54d0809f01b" + "wasm32": "6730d4fb915fc43aac3dd123329d115e70bce84fb6380ae28ad4b50d2aa9704e" }, "dependencyClosures": { "wasm32": [ @@ -1812,6 +1829,11 @@ "manifestSha256": "44e125e1337503e27f8cff531cdbd79c4f406fd2d282b916a7746cf3a14057bf", "cacheKey": "28167e431807693bad67358e88d98cfb0603fd2d6efd9808bb051544846c5364" }, + { + "packageName": "homebrew-bootstrap", + "manifestSha256": "b171060c86cb6642e8cadef9d2698e671c6bf188868c7d835b3e8ecf3a3b054a", + "cacheKey": "8230aaca4bfc1f7bff80d2830534151afff6edb2c03f640af4afa385d145426d" + }, { "packageName": "libcxx", "manifestSha256": "6e1ebfa6914043770eae2a6c156b915acf6a56fe763f07d215ffe6f6f2305007", @@ -1824,8 +1846,8 @@ }, { "packageName": "shell", - "manifestSha256": "5d02cd8fee4e0d46c7552a3046c3898f9e3bd5667094e02cab01851eed97a321", - "cacheKey": "308558a09c4798e2c09083da90be88ce752307cde833d2a0f036b6a61bd28e18" + "manifestSha256": "775d46b5252f6d94fb1ce04c97d63088df7574c54fdb3cbbb5135326352aa069", + "cacheKey": "a2a011cb27f2b3e351c6d48c19308ff849a52a5b982b778bf1a898410b03d03d" } ] }, @@ -1887,10 +1909,15 @@ "wasm32" ], "cacheKeys": { - "wasm32": "3c15243220cac0639aed4b8952f9a09a39036ea4d01b2e75949ebdbacf796d02" + "wasm32": "b069a0aa069fa91a9896cdfc4df82cfa653f2859befde886e4a22377a2326893" }, "dependencyClosures": { "wasm32": [ + { + "packageName": "homebrew-bootstrap", + "manifestSha256": "b171060c86cb6642e8cadef9d2698e671c6bf188868c7d835b3e8ecf3a3b054a", + "cacheKey": "8230aaca4bfc1f7bff80d2830534151afff6edb2c03f640af4afa385d145426d" + }, { "packageName": "libcxx", "manifestSha256": "6e1ebfa6914043770eae2a6c156b915acf6a56fe763f07d215ffe6f6f2305007", @@ -1908,8 +1935,8 @@ }, { "packageName": "shell", - "manifestSha256": "5d02cd8fee4e0d46c7552a3046c3898f9e3bd5667094e02cab01851eed97a321", - "cacheKey": "308558a09c4798e2c09083da90be88ce752307cde833d2a0f036b6a61bd28e18" + "manifestSha256": "775d46b5252f6d94fb1ce04c97d63088df7574c54fdb3cbbb5135326352aa069", + "cacheKey": "a2a011cb27f2b3e351c6d48c19308ff849a52a5b982b778bf1a898410b03d03d" }, { "packageName": "spidermonkey", @@ -2443,7 +2470,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "dfbdea8ae43c488ebafdbb4e83cb5f4dd6bf0810fadd95ede4c1ced77174feb1" + "wasm32": "bc95db71f36cf184d9b7f254e567616c0d6f41b554e4bd2930b45acb6740f1bb" }, "dependencyClosures": { "wasm32": [ @@ -2622,15 +2649,21 @@ ] }, "shell": { - "manifestSha256": "5d02cd8fee4e0d46c7552a3046c3898f9e3bd5667094e02cab01851eed97a321", + "manifestSha256": "775d46b5252f6d94fb1ce04c97d63088df7574c54fdb3cbbb5135326352aa069", "arches": [ "wasm32" ], "cacheKeys": { - "wasm32": "308558a09c4798e2c09083da90be88ce752307cde833d2a0f036b6a61bd28e18" + "wasm32": "a2a011cb27f2b3e351c6d48c19308ff849a52a5b982b778bf1a898410b03d03d" }, "dependencyClosures": { - "wasm32": [] + "wasm32": [ + { + "packageName": "homebrew-bootstrap", + "manifestSha256": "b171060c86cb6642e8cadef9d2698e671c6bf188868c7d835b3e8ecf3a3b054a", + "cacheKey": "8230aaca4bfc1f7bff80d2830534151afff6edb2c03f640af4afa385d145426d" + } + ] }, "members": [ { @@ -2919,7 +2952,7 @@ "wasm32" ], "cacheKeys": { - "wasm32": "225610679e2e521c0d861e2c8a3c36bb1a24c682e97ce909b1b0e786a4147aa0" + "wasm32": "f4fd748f475cab4727977a718271952fa9accecdc3b05f71081303b6cd956481" }, "dependencyClosures": { "wasm32": [ @@ -2928,6 +2961,11 @@ "manifestSha256": "44e125e1337503e27f8cff531cdbd79c4f406fd2d282b916a7746cf3a14057bf", "cacheKey": "28167e431807693bad67358e88d98cfb0603fd2d6efd9808bb051544846c5364" }, + { + "packageName": "homebrew-bootstrap", + "manifestSha256": "b171060c86cb6642e8cadef9d2698e671c6bf188868c7d835b3e8ecf3a3b054a", + "cacheKey": "8230aaca4bfc1f7bff80d2830534151afff6edb2c03f640af4afa385d145426d" + }, { "packageName": "icu", "manifestSha256": "f5b1f02d169ec1108fc32d9da55be53c3efb96e3a7aeba6a03aec62e0175f0dc", @@ -2980,8 +3018,8 @@ }, { "packageName": "shell", - "manifestSha256": "5d02cd8fee4e0d46c7552a3046c3898f9e3bd5667094e02cab01851eed97a321", - "cacheKey": "308558a09c4798e2c09083da90be88ce752307cde833d2a0f036b6a61bd28e18" + "manifestSha256": "775d46b5252f6d94fb1ce04c97d63088df7574c54fdb3cbbb5135326352aa069", + "cacheKey": "a2a011cb27f2b3e351c6d48c19308ff849a52a5b982b778bf1a898410b03d03d" }, { "packageName": "sqlite", diff --git a/packages/registry/shell/build-shell.sh b/packages/registry/shell/build-shell.sh index 0641f2ffdf..96f1fafc7f 100755 --- a/packages/registry/shell/build-shell.sh +++ b/packages/registry/shell/build-shell.sh @@ -9,6 +9,7 @@ REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" OUT_DIR="${WASM_POSIX_DEP_OUT_DIR:-}" HOMEBREW_TAP_ROOT="${WASM_POSIX_BUILD_GIT_HOMEBREW_TAP_CORE_DIR:-}" HOMEBREW_TAP_SHA="${WASM_POSIX_BUILD_GIT_HOMEBREW_TAP_CORE_COMMIT:-}" +HOMEBREW_BOOTSTRAP_DIR="${WASM_POSIX_DEP_HOMEBREW_BOOTSTRAP_DIR:-}" if [ -z "$OUT_DIR" ]; then echo "ERROR: shell is a resolver-owned package build; WASM_POSIX_DEP_OUT_DIR is required" >&2 @@ -18,6 +19,10 @@ if [ -z "$HOMEBREW_TAP_ROOT" ] || [ -z "$HOMEBREW_TAP_SHA" ]; then echo "ERROR: shell requires build.toml git input homebrew_tap_core (DIR and COMMIT)" >&2 exit 2 fi +if [ -z "$HOMEBREW_BOOTSTRAP_DIR" ]; then + echo "ERROR: shell requires its declared homebrew-bootstrap dependency" >&2 + exit 2 +fi if [ "${WASM_POSIX_DEP_TARGET_ARCH:-}" != "wasm32" ]; then echo "ERROR: shell Homebrew closure currently supports only wasm32" >&2 exit 2 @@ -36,6 +41,8 @@ export LANG=C BUILD_DIR="$OUT_DIR/.homebrew-shell-build" WORK_DIR="$BUILD_DIR/work" VFS="$BUILD_DIR/shell.vfs.zst" +HOMEBREW_BOOTSTRAP="$HOMEBREW_BOOTSTRAP_DIR/homebrew-bootstrap.zip" +HOMEBREW_BREW_ENV="$HOMEBREW_BOOTSTRAP_DIR/homebrew-brew.env" REPORT="$BUILD_DIR/main-shell-report.json" BOTTLE_CACHE="$BUILD_DIR/bottle-cache" if [ -e "$BUILD_DIR" ] || [ -L "$BUILD_DIR" ]; then @@ -43,6 +50,14 @@ if [ -e "$BUILD_DIR" ] || [ -L "$BUILD_DIR" ]; then exit 1 fi mkdir "$BUILD_DIR" +if [ ! -f "$HOMEBREW_BOOTSTRAP" ] || [ -L "$HOMEBREW_BOOTSTRAP" ]; then + echo "ERROR: declared homebrew-bootstrap output is not a regular file: $HOMEBREW_BOOTSTRAP" >&2 + exit 2 +fi +if [ ! -f "$HOMEBREW_BREW_ENV" ] || [ -L "$HOMEBREW_BREW_ENV" ]; then + echo "ERROR: declared Homebrew environment output is not a regular file: $HOMEBREW_BREW_ENV" >&2 + exit 2 +fi cleanup() { rm -rf -- "$BUILD_DIR" } @@ -59,6 +74,9 @@ bash "$REPO_ROOT/scripts/build-homebrew-main-shell-closure.sh" \ --work-dir "$WORK_DIR" \ --report "$REPORT" \ --bottle-cache "$BOTTLE_CACHE" \ + --package-tree-spec "$REPO_ROOT/homebrew/main-shell-brew-package-tree.json" \ + --package-tree-archive "$HOMEBREW_BOOTSTRAP" \ + --homebrew-bootstrap-env "$HOMEBREW_BREW_ENV" \ --out "$VFS" [ -f "$VFS" ] || { echo "ERROR: $VFS not produced by builder" >&2; exit 1; } diff --git a/packages/registry/shell/build.toml b/packages/registry/shell/build.toml index 046582cc6d..7c718a5c61 100644 --- a/packages/registry/shell/build.toml +++ b/packages/registry/shell/build.toml @@ -8,6 +8,7 @@ inputs = [ "homebrew/main-shell.Brewfile", "homebrew/main-shell-default.json", "homebrew/main-shell-demo.json", + "homebrew/main-shell-brew-package-tree.json", "homebrew/main-shell-lazy-artifact-lock.json", "homebrew/main-shell-migration-lock.json", "homebrew/main-shell-materialization-policy.json", @@ -48,6 +49,7 @@ inputs = [ "host/src/vfs/hardlink-graph.ts", "host/src/vfs/image-helpers.ts", "host/src/vfs/memory-fs.ts", + "host/src/vfs/package-deferred-tree-contract.ts", "host/src/vfs/package-deferred-tree.ts", "host/src/vfs/sharedfs-vendor.ts", "host/src/vfs/tar.ts", @@ -58,7 +60,7 @@ inputs = [ ] repo_url = "https://github.com/Automattic/kandelo.git" commit = "8c53383229fab78f97b098c3207a655159c03041" -revision = 18 +revision = 19 [[git_inputs]] name = "homebrew_tap_core" diff --git a/packages/registry/shell/package.toml b/packages/registry/shell/package.toml index 37f7c1c4df..051ecb8ad7 100644 --- a/packages/registry/shell/package.toml +++ b/packages/registry/shell/package.toml @@ -4,9 +4,11 @@ version = "0.1.0" kernel_abi = 7 # The canonical shell closure is selected by the reviewed Brewfile and comes # from the immutable Homebrew tap declared in build.toml. It deliberately has -# no package-registry dependencies: resolving the retired rootfs/program graph -# would make a bottle-only build depend on artifacts it never consumes. -depends_on = [] +# no legacy rootfs/program dependencies: resolving the retired graph would make +# a bottle-only build depend on artifacts it never consumes. Homebrew itself is +# a separate product package because the shell only needs its source tree when +# a guest actually invokes `brew`. +depends_on = ["homebrew-bootstrap@6.0.3-4-g4ead861"] # Composite VFS image for the browser shell demo: a platform-only base plus the # exact reviewed Homebrew bottle closure and shell configuration. The image is diff --git a/run.sh b/run.sh index 1cb31746e5..17e972d391 100755 --- a/run.sh +++ b/run.sh @@ -358,7 +358,13 @@ has_msmtpd() { pkg_has_output msmtpd msmtpd.wasm || [ -f "$REPO_ROOT/pack has_cpython() { pkg_has_output cpython python.wasm || [ -f "$REPO_ROOT/packages/registry/cpython/bin/python.wasm" ]; } has_python_vfs() { pkg_has_output python-vfs python-vfs.vfs.zst || [ -f "$REPO_ROOT/apps/browser-demos/public/python.vfs.zst" ]; } has_perl_vfs() { pkg_has_output perl-vfs perl-vfs.vfs.zst || [ -f "$REPO_ROOT/apps/browser-demos/public/perl.vfs.zst" ]; } -has_shell_vfs() { pkg_has_output shell shell.vfs.zst; } +has_shell_vfs() { + pkg_has_output shell shell.vfs.zst && + # The VFS keeps this dependency lazy, but the browser must still be able + # to serve its exact package bytes when the guest first invokes brew. + pkg_has_output homebrew-bootstrap homebrew-bootstrap.zip && + pkg_has_output homebrew-bootstrap homebrew-brew.env +} has_node() { pkg_has_output node node.wasm; } has_spidermonkey_node() { pkg_has_output spidermonkey-node node.wasm || [ -f "$REPO_ROOT/packages/registry/spidermonkey-node/bin/node.wasm" ]; } has_node_vfs() { pkg_has_output node-vfs node-vfs.vfs.zst || [ -f "$REPO_ROOT/apps/browser-demos/public/node-vfs.vfs.zst" ]; } @@ -1012,6 +1018,14 @@ build_shell_vfs() { err "Package resolver did not materialize the declared shell.vfs.zst output" return 1 fi + if ! pkg_has_output homebrew-bootstrap homebrew-bootstrap.zip; then + err "Package resolver did not materialize shell's Homebrew source dependency" + return 1 + fi + if ! pkg_has_output homebrew-bootstrap homebrew-brew.env; then + err "Package resolver did not materialize shell's Homebrew launcher policy" + return 1 + fi info "Bottle-built Shell VFS image resolved" } diff --git a/scripts/build-homebrew-main-shell-closure.sh b/scripts/build-homebrew-main-shell-closure.sh index 9a5e38510b..7152499d28 100755 --- a/scripts/build-homebrew-main-shell-closure.sh +++ b/scripts/build-homebrew-main-shell-closure.sh @@ -8,6 +8,9 @@ WORK_DIR="" OUT="" REPORT="" BOTTLE_CACHE="" +PACKAGE_TREE_SPEC="" +PACKAGE_TREE_ARCHIVE="" +HOMEBREW_BOOTSTRAP_ENV="" BREWFILE="$REPO_ROOT/homebrew/main-shell.Brewfile" SHELL_CONFIG="$REPO_ROOT/homebrew/main-shell-default.json" DEMO_CONFIG="$REPO_ROOT/homebrew/main-shell-demo.json" @@ -17,6 +20,7 @@ LAZY_ARTIFACT_LOCK="$REPO_ROOT/homebrew/main-shell-lazy-artifact-lock.json" LAZY_ARTIFACT_CHECKER="$REPO_ROOT/scripts/verify-homebrew-main-shell-artifact-lock.sh" BOTTLE_MIRROR_REPOSITORY="kandelo-dev/homebrew-tap-core" LAZY_SHELL=false +MATERIALIZE_PACKAGE_TREE=false MAX_BYTES="$((512 * 1024 * 1024))" # The shell image is a content-addressed product artifact. Do not let a Nix @@ -43,6 +47,14 @@ Options: --out output image --report composition evidence --bottle-cache verified bottle cache + --package-tree-spec + reviewed package-owned lazy-tree recipe + --package-tree-archive + exact dependency output named by the recipe + --homebrew-bootstrap-env + exact package-owned launcher environment + --materialize-package-tree + embed that same tree for an eager derivative --migration-lock reviewed package/catalog lock --lazy-artifact-lock exact lazy-image digest and timestamp contract @@ -78,6 +90,18 @@ while [ "$#" -gt 0 ]; do BOTTLE_CACHE="${2:-}" shift 2 ;; + --package-tree-spec) + PACKAGE_TREE_SPEC="${2:-}" + shift 2 + ;; + --package-tree-archive) + PACKAGE_TREE_ARCHIVE="${2:-}" + shift 2 + ;; + --homebrew-bootstrap-env) + HOMEBREW_BOOTSTRAP_ENV="${2:-}" + shift 2 + ;; --migration-lock) MIGRATION_LOCK="${2:-}" shift 2 @@ -94,6 +118,10 @@ while [ "$#" -gt 0 ]; do LAZY_SHELL=true shift ;; + --materialize-package-tree) + MATERIALIZE_PACKAGE_TREE=true + shift + ;; -h|--help) usage exit 0 @@ -118,6 +146,35 @@ mkdir "$WORK_DIR" OUT="${OUT:-$WORK_DIR/main-shell.vfs.zst}" REPORT="${REPORT:-$WORK_DIR/main-shell-report.json}" BOTTLE_CACHE="${BOTTLE_CACHE:-$WORK_DIR/bottle-cache}" +if { [ -n "$PACKAGE_TREE_SPEC" ] && [ -z "$PACKAGE_TREE_ARCHIVE" ]; } || + { [ -z "$PACKAGE_TREE_SPEC" ] && [ -n "$PACKAGE_TREE_ARCHIVE" ]; }; then + echo "build-homebrew-main-shell-closure: package-tree spec and archive must be provided together" >&2 + exit 2 +fi +if { [ -n "$PACKAGE_TREE_SPEC" ] && [ -z "$HOMEBREW_BOOTSTRAP_ENV" ]; } || + { [ -z "$PACKAGE_TREE_SPEC" ] && [ -n "$HOMEBREW_BOOTSTRAP_ENV" ]; }; then + echo "build-homebrew-main-shell-closure: Homebrew bootstrap environment and package tree must be provided together" >&2 + exit 2 +fi +if [ "$MATERIALIZE_PACKAGE_TREE" = true ] && [ -z "$PACKAGE_TREE_SPEC" ]; then + echo "build-homebrew-main-shell-closure: --materialize-package-tree requires a package tree" >&2 + exit 2 +fi +if [ -n "$PACKAGE_TREE_SPEC" ] && + { [ ! -f "$PACKAGE_TREE_SPEC" ] || [ -L "$PACKAGE_TREE_SPEC" ]; }; then + echo "build-homebrew-main-shell-closure: package-tree spec must be a regular non-symlink file" >&2 + exit 2 +fi +if [ -n "$PACKAGE_TREE_ARCHIVE" ] && + { [ ! -f "$PACKAGE_TREE_ARCHIVE" ] || [ -L "$PACKAGE_TREE_ARCHIVE" ]; }; then + echo "build-homebrew-main-shell-closure: package-tree archive must be a regular non-symlink file" >&2 + exit 2 +fi +if [ -n "$HOMEBREW_BOOTSTRAP_ENV" ] && + { [ ! -f "$HOMEBREW_BOOTSTRAP_ENV" ] || [ -L "$HOMEBREW_BOOTSTRAP_ENV" ]; }; then + echo "build-homebrew-main-shell-closure: Homebrew bootstrap environment must be a regular non-symlink file" >&2 + exit 2 +fi if ! [[ "$MAX_BYTES" =~ ^[1-9][0-9]*$ ]] || [ $((MAX_BYTES % 4096)) -ne 0 ]; then echo "build-homebrew-main-shell-closure: --max-bytes must be a positive multiple of 4096" >&2 exit 2 @@ -187,7 +244,7 @@ for tool in git jq node ruby sha256sum wc; do } done -if [ "$LAZY_SHELL" = true ]; then +if [ "$LAZY_SHELL" = true ] && [ "$MATERIALIZE_PACKAGE_TREE" = false ]; then if [ ! -f "$LAZY_ARTIFACT_CHECKER" ] || [ -L "$LAZY_ARTIFACT_CHECKER" ]; then echo "build-homebrew-main-shell-closure: lazy artifact checker must be a regular non-symlink file" >&2 exit 2 @@ -277,6 +334,29 @@ node "$REPO_ROOT/tools/mkrootfs/bin/mkrootfs.mjs" build \ -o "$PLATFORM_BASE" MATERIALIZATION_ARGS=() +PACKAGE_TREE_ARGS=() +PACKAGE_TREE_JSON=null +PACKAGE_TREE_ARCHIVE_SHA="" +PACKAGE_TREE_ARCHIVE_BYTES=0 +HOMEBREW_BOOTSTRAP_ENV_SHA="" +HOMEBREW_BOOTSTRAP_ENV_BYTES=0 +if [ -n "$PACKAGE_TREE_SPEC" ]; then + PACKAGE_TREE_ARGS=( + --package-tree-spec "$PACKAGE_TREE_SPEC" + --package-tree-archive "$PACKAGE_TREE_ARCHIVE" + --homebrew-bootstrap-env "$HOMEBREW_BOOTSTRAP_ENV" + ) + if [ "$MATERIALIZE_PACKAGE_TREE" = true ]; then + PACKAGE_TREE_ARGS+=(--materialize-package-tree) + fi + PACKAGE_TREE_JSON="$(jq -c . "$PACKAGE_TREE_SPEC")" + PACKAGE_TREE_ARCHIVE_SHA="$(sha256sum "$PACKAGE_TREE_ARCHIVE")" + PACKAGE_TREE_ARCHIVE_SHA="${PACKAGE_TREE_ARCHIVE_SHA%% *}" + PACKAGE_TREE_ARCHIVE_BYTES="$(wc -c <"$PACKAGE_TREE_ARCHIVE" | tr -d '[:space:]')" + HOMEBREW_BOOTSTRAP_ENV_SHA="$(sha256sum "$HOMEBREW_BOOTSTRAP_ENV")" + HOMEBREW_BOOTSTRAP_ENV_SHA="${HOMEBREW_BOOTSTRAP_ENV_SHA%% *}" + HOMEBREW_BOOTSTRAP_ENV_BYTES="$(wc -c <"$HOMEBREW_BOOTSTRAP_ENV" | tr -d '[:space:]')" +fi MATERIALIZATION_JSON=null VFS_IMAGE_BUILDER="$REPO_ROOT/images/vfs/scripts/build-homebrew-vfs-image.ts" if [ "$LAZY_SHELL" = true ]; then @@ -303,6 +383,7 @@ fi --catalog-commit "$EXPECTED_TAP_SHA" \ --migration-lock "$MIGRATION_LOCK" \ "${MATERIALIZATION_ARGS[@]}" \ + "${PACKAGE_TREE_ARGS[@]}" \ --write-profile \ --shell-config "$SHELL_CONFIG" \ --demo-config "$DEMO_CONFIG" \ @@ -313,7 +394,7 @@ if [ ! -f "$OUT" ] || [ -L "$OUT" ] || [ ! -f "$REPORT" ] || [ -L "$REPORT" ]; t echo "build-homebrew-main-shell-closure: image builder did not produce regular image and report files" >&2 exit 1 fi -if [ "$LAZY_SHELL" = true ]; then +if [ "$LAZY_SHELL" = true ] && [ "$MATERIALIZE_PACKAGE_TREE" = false ]; then bash "$LAZY_ARTIFACT_CHECKER" \ --lock "$LAZY_ARTIFACT_LOCK" \ --expected-source-date-epoch "$SOURCE_DATE_EPOCH" \ @@ -325,6 +406,12 @@ jq -e \ --slurpfile tap "$TAP_ROOT/Kandelo/metadata.json" \ --slurpfile lock "$MIGRATION_LOCK" \ --argjson materialization "$MATERIALIZATION_JSON" \ + --argjson package_tree_spec "$PACKAGE_TREE_JSON" \ + --arg package_tree_archive_sha "$PACKAGE_TREE_ARCHIVE_SHA" \ + --argjson package_tree_archive_bytes "$PACKAGE_TREE_ARCHIVE_BYTES" \ + --arg homebrew_bootstrap_env_sha "$HOMEBREW_BOOTSTRAP_ENV_SHA" \ + --argjson homebrew_bootstrap_env_bytes "$HOMEBREW_BOOTSTRAP_ENV_BYTES" \ + --argjson materialize_package_tree "$MATERIALIZE_PACKAGE_TREE" \ --argjson lazy_shell "$LAZY_SHELL" \ --argjson abi "$ABI_VERSION" \ --arg catalog "$EXPECTED_TAP_SHA" \ @@ -483,7 +570,57 @@ jq -e \ (.image_capacity.max_byte_length == $max_bytes) and (.base_image.kernelAbi == $abi) and (.base_image.metadata.kernelAbi == $abi) and - (.base_image.metadata.homebrew == null) + (.base_image.metadata.homebrew == null) and + (if $package_tree_spec == null then + (.package_deferred_trees == null) + else + (.package_deferred_trees | length == 1) and + (.package_deferred_trees[0] as $tree | + $tree.schema == $package_tree_spec.schema and + $tree.kind == $package_tree_spec.kind and + $tree.id == $package_tree_spec.id and + $tree.content_role == $package_tree_spec.content_role and + $tree.package == $package_tree_spec.package and + $tree.archive.output == $package_tree_spec.package.output and + $tree.archive.url == $package_tree_spec.archive.url and + $tree.archive.sha256 == $package_tree_archive_sha and + $tree.archive.bytes == $package_tree_archive_bytes and + $tree.archive.expanded_bytes > 0 and + $tree.archive.source_entry_count > 0 and + ($tree.descriptor.sha256 | test("^[0-9a-f]{64}$")) and + $tree.descriptor.bytes > 0 and + $tree.mount_prefix == $package_tree_spec.mount_prefix and + $tree.owner == $package_tree_spec.owner and + $tree.activation == $package_tree_spec.activation and + $tree.state == (if $materialize_package_tree + then "materialized" + else "deferred" + end) and + .homebrew_bootstrap == { + environment: { + path: "/etc/homebrew/brew.env", + sha256: $homebrew_bootstrap_env_sha, + bytes: $homebrew_bootstrap_env_bytes + }, + entrypoint: { + path: "/usr/bin/brew", + target: "/home/linuxbrew/.linuxbrew/bin/brew" + }, + ownership: { + prefix: "/home/linuxbrew/.linuxbrew", + uid: 1000, + gid: 1000, + mutable_paths: [ + "/home/linuxbrew/.linuxbrew/Cellar", + "/home/linuxbrew/.linuxbrew/Library/Taps", + "/home/linuxbrew/.linuxbrew/var/homebrew/linked", + "/home/linuxbrew/.linuxbrew/var/homebrew/locks", + "/home/user/.cache/Homebrew" + ] + } + } + ) + end) ' "$REPORT" >/dev/null if [ "$LAZY_SHELL" = true ]; then diff --git a/scripts/check-homebrew-main-shell-brewfile.mjs b/scripts/check-homebrew-main-shell-brewfile.mjs index d0bd2b008c..0b10ed5082 100755 --- a/scripts/check-homebrew-main-shell-brewfile.mjs +++ b/scripts/check-homebrew-main-shell-brewfile.mjs @@ -5,7 +5,9 @@ import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); -const brewfile = resolve(process.argv[2] ?? `${repoRoot}/homebrew/main-shell.Brewfile`); +const brewfile = resolve( + process.argv[2] ?? `${repoRoot}/homebrew/main-shell.Brewfile`, +); const lockPath = resolve( process.argv[3] ?? `${repoRoot}/homebrew/main-shell-migration-lock.json`, ); @@ -22,17 +24,26 @@ const rootfsPackages = readDependencies( const shellDependencies = readDependencies( `${repoRoot}/packages/registry/shell/package.toml`, ); -if (shellDependencies.length !== 0) { - throw new Error( - "the canonical shell package must stay bottle-only (depends_on = []); " + - `found ${shellDependencies.map(({ name, version }) => `${name}@${version}`).join(", ")}`, - ); -} +const homebrewBootstrap = readPackageIdentity( + `${repoRoot}/packages/registry/homebrew-bootstrap/package.toml`, +); +// Bottle Formulae remain selected only by the reviewed Brewfile. The sole +// registry dependency is distribution machinery: exact Homebrew source bytes +// that the VFS registers without materializing until a guest invokes brew. +assertExactSequence( + shellDependencies, + [homebrewBootstrap], + "the canonical shell package must depend only on its exact Homebrew source package", + ({ name, version }) => `${name}@${version}`, +); const lockedRegistryPackages = lock.packages.map(({ registry }) => registry); const expectedFormulae = lock.packages.map(({ formula }) => formula.name); const actualFormulae = readBrewfilePackages(brewfile); -assertUnique(lockedRegistryPackages.map(({ name }) => name), "migration lock registry roots"); +assertUnique( + lockedRegistryPackages.map(({ name }) => name), + "migration lock registry roots", +); assertUnique(expectedFormulae, "migration lock Formulae"); assertUnique(actualFormulae, "main-shell Brewfile"); assertExactSequence( @@ -65,15 +76,20 @@ function readMigrationLock(path) { value.tap_repository !== tapRepository || value.tap_name !== tapName ) { - throw new Error(`invalid main-shell migration lock schema or tap identity: ${path}`); + throw new Error( + `invalid main-shell migration lock schema or tap identity: ${path}`, + ); } if ( !isRecord(value.catalog) || - JSON.stringify(Object.keys(value.catalog).sort()) !== JSON.stringify(["tap_commit"]) || + JSON.stringify(Object.keys(value.catalog).sort()) !== + JSON.stringify(["tap_commit"]) || typeof value.catalog.tap_commit !== "string" || !gitShaPattern.test(value.catalog.tap_commit) ) { - throw new Error(`main-shell migration lock must pin one exact catalog commit: ${path}`); + throw new Error( + `main-shell migration lock must pin one exact catalog commit: ${path}`, + ); } if ( !Array.isArray(value.packages) || @@ -89,17 +105,28 @@ function readMigrationLock(path) { value.consumer.profile !== "main-shell" || value.consumer.max_vfs_byte_length !== 512 * 1024 * 1024 ) { - throw new Error(`main-shell migration lock must declare the 512 MiB consumer profile: ${path}`); + throw new Error( + `main-shell migration lock must declare the 512 MiB consumer profile: ${path}`, + ); } const packages = value.packages.map((entry, index) => { - if (!isRecord(entry) || !isRecord(entry.registry) || !isRecord(entry.formula)) { + if ( + !isRecord(entry) || + !isRecord(entry.registry) || + !isRecord(entry.formula) + ) { throw new Error(`invalid migration lock package ${index}`); } - const registry = readIdentity(entry.registry, `packages[${index}].registry`); + const registry = readIdentity( + entry.registry, + `packages[${index}].registry`, + ); const formula = readIdentity(entry.formula, `packages[${index}].formula`); for (const field of ["revision", "bottle_rebuild"]) { if (!Number.isInteger(entry.formula[field]) || entry.formula[field] < 0) { - throw new Error(`packages[${index}].formula.${field} must be a non-negative integer`); + throw new Error( + `packages[${index}].formula.${field} must be a non-negative integer`, + ); } } return { @@ -112,10 +139,12 @@ function readMigrationLock(path) { }; }); const formulaClosure = value.formula_closure.map((entry, index) => - readFormulaIdentity(entry, `formula_closure[${index}]`) + readFormulaIdentity(entry, `formula_closure[${index}]`), ); if (packages.length === 0 || formulaClosure.length === 0) { - throw new Error(`main-shell migration lock must contain roots and a closure: ${path}`); + throw new Error( + `main-shell migration lock must contain roots and a closure: ${path}`, + ); } assertUnique(formulaClosure, "migration lock formula_closure"); const missingRoots = packages @@ -143,11 +172,10 @@ function readIdentity(value, label) { } function readFormulaIdentity(value, label) { - if ( - typeof value !== "string" || - !formulaIdentityPattern.test(value) - ) { - throw new Error(`${label} must be a canonical ${tapName}/ identity`); + if (typeof value !== "string" || !formulaIdentityPattern.test(value)) { + throw new Error( + `${label} must be a canonical ${tapName}/ identity`, + ); } return value; } @@ -184,12 +212,17 @@ function validateReviewedSubstitutions(lock) { } return { kind: entry.kind, - registry: readReviewedRegistryIdentity(entry.registry, `${label}.registry`), + registry: readReviewedRegistryIdentity( + entry.registry, + `${label}.registry`, + ), formula: readReviewedFormulaIdentity(entry.formula, `${label}.formula`), }; }); assertUnique( - actual.map(({ kind, registry, formula }) => `${kind}:${registry}->${formula}`), + actual.map( + ({ kind, registry, formula }) => `${kind}:${registry}->${formula}`, + ), "reviewed migration substitutions", ); assertExactSequence( @@ -218,12 +251,16 @@ function readReviewedRegistryIdentity(value, label) { function readReviewedFormulaIdentity(value, label) { const prefix = `${tapName}/`; if (typeof value !== "string" || !value.startsWith(prefix)) { - throw new Error(`${label} must be a ${tapName}/@ identity`); + throw new Error( + `${label} must be a ${tapName}/@ identity`, + ); } const unqualified = value.slice(prefix.length); const separator = unqualified.lastIndexOf("@"); if (separator <= 0 || separator === unqualified.length - 1) { - throw new Error(`${label} must be a ${tapName}/@ identity`); + throw new Error( + `${label} must be a ${tapName}/@ identity`, + ); } readIdentity( { @@ -261,10 +298,14 @@ function validateCompatibilityPolicy(lock) { typeof entry.reason !== "string" || entry.reason.trim().length === 0 ) { - throw new Error(`compatibility.link_conflict_owners[${index}] is invalid`); + throw new Error( + `compatibility.link_conflict_owners[${index}] is invalid`, + ); } if (conflictTargets.has(entry.target)) { - throw new Error(`compatibility link conflict target is duplicated: ${entry.target}`); + throw new Error( + `compatibility link conflict target is duplicated: ${entry.target}`, + ); } conflictTargets.add(entry.target); } @@ -277,14 +318,17 @@ function validateCompatibilityPolicy(lock) { !lockedPackages.has(entry.package) || (entry.source_kind !== "link" && entry.source_kind !== "keg") || typeof entry.source !== "string" || - !/^[a-z0-9][a-z0-9._+-]*(?:\/[a-z0-9][a-z0-9._+-]*)*$/.test(entry.source) || + !/^[a-z0-9][a-z0-9._+-]*(?:\/[a-z0-9][a-z0-9._+-]*)*$/.test( + entry.source, + ) || (entry.source_kind === "link" && !/^bin\/[a-z0-9][a-z0-9._+-]*$/.test(entry.source)) || !Array.isArray(entry.targets) || entry.targets.length === 0 || - entry.targets.some((target) => - typeof target !== "string" || - !/^\/(?:[a-z0-9._+-]+\/)*[a-z0-9._+-]+$/.test(target) + entry.targets.some( + (target) => + typeof target !== "string" || + !/^\/(?:[a-z0-9._+-]+\/)*[a-z0-9._+-]+$/.test(target), ) || new Set(entry.targets).size !== entry.targets.length ) { @@ -341,7 +385,9 @@ function validateCompatibilityPolicy(lock) { throw new Error(`compatibility.runtime_state[${index}] is invalid`); } if (runtimePaths.has(entry.path)) { - throw new Error(`compatibility runtime state path is duplicated: ${entry.path}`); + throw new Error( + `compatibility runtime state path is duplicated: ${entry.path}`, + ); } runtimePaths.set(entry.path, entry); } @@ -368,7 +414,9 @@ function validateTapMetadata(lock, path) { metadata.tap_name !== tapName || !Array.isArray(metadata.packages) ) { - throw new Error(`tap metadata has the wrong identity or package shape: ${path}`); + throw new Error( + `tap metadata has the wrong identity or package shape: ${path}`, + ); } const byName = new Map(); for (const [index, value] of metadata.packages.entries()) { @@ -383,9 +431,10 @@ function validateTapMetadata(lock, path) { if (!isRecord(pkg)) { throw new Error(`tap metadata is missing locked Formula ${formula.name}`); } - const expectedVersion = formula.revision === 0 - ? formula.version - : `${formula.version}_${formula.revision}`; + const expectedVersion = + formula.revision === 0 + ? formula.version + : `${formula.version}_${formula.revision}`; if ( pkg.full_name !== `${tapName}/${formula.name}` || pkg.version !== expectedVersion || @@ -438,7 +487,9 @@ function readTapMetadataPackage(value, label) { (dependency.full_name !== undefined && dependency.full_name !== `${tapName}/${dependency.name}`) ) { - throw new Error(`${dependencyLabel} is not a canonical same-tap dependency`); + throw new Error( + `${dependencyLabel} is not a canonical same-tap dependency`, + ); } return dependency.name; }); @@ -460,7 +511,10 @@ function resolveTapFormulaClosure(rootNames, byName) { } const pkg = byName.get(name); if (pkg === undefined) { - const context = requiredBy === undefined ? "registry root" : `dependency of ${requiredBy}`; + const context = + requiredBy === undefined + ? "registry root" + : `dependency of ${requiredBy}`; throw new Error(`tap metadata is missing ${context} Formula ${name}`); } state.set(name, "visiting"); @@ -479,25 +533,49 @@ function readDependencies(path) { const source = readFileSync(path, "utf8"); const match = /(?:^|\n)depends_on\s*=\s*\[([\s\S]*?)\]/.exec(source); if (!match) throw new Error(`cannot find depends_on array in ${path}`); - const entries = Array.from(match[1].matchAll(/"([^"]+)"/g), (item) => item[1]); + const entries = Array.from( + match[1].matchAll(/"([^"]+)"/g), + (item) => item[1], + ); return entries.map((entry) => { const at = entry.lastIndexOf("@"); if (at <= 0 || at === entry.length - 1) { - throw new Error(`dependency must be locked as name@version: ${entry} in ${path}`); + throw new Error( + `dependency must be locked as name@version: ${entry} in ${path}`, + ); } const name = entry.slice(0, at); const version = entry.slice(at + 1); if (!/^[a-z0-9][a-z0-9._-]*$/.test(name)) { - throw new Error(`unsupported dependency ${JSON.stringify(entry)} in ${path}`); + throw new Error( + `unsupported dependency ${JSON.stringify(entry)} in ${path}`, + ); } return { name, version }; }); } +function readPackageIdentity(path) { + const source = readFileSync(path, "utf8"); + const name = /(?:^|\n)name\s*=\s*"([^"]+)"/.exec(source)?.[1]; + const version = /(?:^|\n)version\s*=\s*"([^"]+)"/.exec(source)?.[1]; + if ( + name === undefined || + !/^[a-z0-9][a-z0-9._-]*$/.test(name) || + version === undefined || + version.length === 0 + ) { + throw new Error(`cannot read package identity from ${path}`); + } + return { name, version }; +} + function readBrewfilePackages(path) { const packages = []; let sawTap = false; - for (const [index, rawLine] of readFileSync(path, "utf8").split("\n").entries()) { + for (const [index, rawLine] of readFileSync(path, "utf8") + .split("\n") + .entries()) { const line = rawLine.trim(); if (line === "" || line.startsWith("#")) continue; if (line === `tap "${tapName}"`) { @@ -505,7 +583,9 @@ function readBrewfilePackages(path) { sawTap = true; continue; } - const match = /^brew "kandelo-dev\/tap-core\/([a-z0-9][a-z0-9._-]*)"$/.exec(line); + const match = /^brew "kandelo-dev\/tap-core\/([a-z0-9][a-z0-9._-]*)"$/.exec( + line, + ); if (!match) throw new Error(`unsupported ${path}:${index + 1}: ${rawLine}`); packages.push(match[1]); } @@ -517,7 +597,9 @@ function assertExactSequence(actual, expected, message, render) { const actualValues = actual.map(render); const expectedValues = expected.map(render); if (JSON.stringify(actualValues) === JSON.stringify(expectedValues)) return; - const missing = expectedValues.filter((value) => !actualValues.includes(value)); + const missing = expectedValues.filter( + (value) => !actualValues.includes(value), + ); const extra = actualValues.filter((value) => !expectedValues.includes(value)); throw new Error( `${message}\n missing: ${missing.join(", ") || "(none)"}` + @@ -529,7 +611,9 @@ function assertExactSet(actual, expected, message, render) { const actualValues = actual.map(render).sort(); const expectedValues = expected.map(render).sort(); if (JSON.stringify(actualValues) === JSON.stringify(expectedValues)) return; - const missing = expectedValues.filter((value) => !actualValues.includes(value)); + const missing = expectedValues.filter( + (value) => !actualValues.includes(value), + ); const extra = actualValues.filter((value) => !expectedValues.includes(value)); throw new Error( `${message}\n missing: ${missing.join(", ") || "(none)"}` + @@ -538,7 +622,9 @@ function assertExactSet(actual, expected, message, render) { } function assertUnique(values, label) { - const duplicate = values.find((value, index) => values.indexOf(value) !== index); + const duplicate = values.find( + (value, index) => values.indexOf(value) !== index, + ); if (duplicate) throw new Error(`${label} contains duplicate ${duplicate}`); } diff --git a/scripts/homebrew-closed-lazy-assets-contract.ts b/scripts/homebrew-closed-lazy-assets-contract.ts new file mode 100644 index 0000000000..2e87d76bb9 --- /dev/null +++ b/scripts/homebrew-closed-lazy-assets-contract.ts @@ -0,0 +1,125 @@ +import type { SerializedLazyArchiveEntry } from "../host/src/vfs/memory-fs"; +import { + encodeHomebrewBottleMirrorCollectionIdentity, + encodeHomebrewBottleMirrorPlan, + projectHomebrewBottleMirrorPlan, + type HomebrewBottleMirrorPlan, +} from "../host/src/homebrew-bottle-mirror-plan"; + +/** + * Decode the byte-canonical bottle mirror plan shared by Node and browser + * acceptance. Keeping this parser browser-safe prevents the two hosts from + * accepting different release identities. + */ +export function decodeHomebrewBottleMirrorPlan( + planBytes: Uint8Array, + label: string, +): HomebrewBottleMirrorPlan { + let decoded: unknown; + try { + decoded = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode( + planBytes, + )); + } catch (error) { + throw new Error(`${label} is not valid UTF-8 JSON: ${String(error)}`); + } + if (!isRecord(decoded) || !Array.isArray(decoded.assets)) { + throw new Error(`${label} does not declare a bottle mirror asset array`); + } + const plan = projectHomebrewBottleMirrorPlan(decoded); + if (!bytesEqual(planBytes, encodeHomebrewBottleMirrorPlan(plan))) { + throw new Error(`${label} bytes are not canonical`); + } + return plan; +} + +/** Validate the content-derived release identity with browser Web Crypto. */ +export async function assertHomebrewBottleMirrorPlanIdentity( + plan: HomebrewBottleMirrorPlan, +): Promise { + const normalized = projectHomebrewBottleMirrorPlan(plan); + const collectionSha = await sha256( + encodeHomebrewBottleMirrorCollectionIdentity( + normalized.repository, + normalized.assets, + ), + ); + const tag = `homebrew-shell-bottles-sha256-${collectionSha}`; + const releaseRoot = + `https://github.com/${normalized.repository}/releases/download/${tag}`; + if ( + normalized.collection_sha256 !== collectionSha || + normalized.tag !== tag || + normalized.release_root !== releaseRoot || + normalized.assets.some( + (asset) => asset.url !== `${releaseRoot}/${asset.asset}`, + ) + ) { + throw new Error( + "Homebrew bottle mirror plan has inconsistent derived identity", + ); + } +} + +export function assertPendingTreeHomebrewBottleMirrorBinding( + pendingTrees: readonly SerializedLazyArchiveEntry[], + plan: HomebrewBottleMirrorPlan, +): void { + if (pendingTrees.length !== plan.assets.length) { + throw new Error( + `pending tree count ${pendingTrees.length} differs from mirror asset count ` + + `${plan.assets.length}`, + ); + } + const assetByUrl = new Map(plan.assets.map((asset) => [asset.url, asset])); + if (assetByUrl.size !== plan.assets.length) { + throw new Error("bottle mirror plan duplicates a release URL"); + } + const seen = new Set(); + for (const tree of pendingTrees) { + const content = tree.content; + const primaryUrl = content?.transports[0]; + const asset = + primaryUrl === undefined ? undefined : assetByUrl.get(primaryUrl); + if ( + content === undefined || + asset === undefined || + content.sha256 !== asset.sha256 || + content.bytes !== asset.bytes + ) { + throw new Error( + `pending tree ${tree.mountPrefix} does not match one exact mirror asset`, + ); + } + if (seen.has(primaryUrl!)) { + throw new Error(`multiple pending trees use mirror URL ${primaryUrl}`); + } + seen.add(primaryUrl!); + } + if (seen.size !== plan.assets.length) { + throw new Error( + "pending trees do not cover the complete bottle mirror plan", + ); + } +} + +export function bytesEqual(left: Uint8Array, right: Uint8Array): boolean { + return left.byteLength === right.byteLength && + left.every((byte, index) => byte === right[index]); +} + +export function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +async function sha256(bytes: Uint8Array): Promise { + const owned = new ArrayBuffer(bytes.byteLength); + new Uint8Array(owned).set(bytes); + const digest = new Uint8Array( + await crypto.subtle.digest("SHA-256", owned), + ); + return Array.from( + digest, + (byte) => byte.toString(16).padStart(2, "0"), + ).join(""); +} diff --git a/scripts/homebrew-closed-lazy-assets.test.ts b/scripts/homebrew-closed-lazy-assets.test.ts new file mode 100644 index 0000000000..5e976b5d4f --- /dev/null +++ b/scripts/homebrew-closed-lazy-assets.test.ts @@ -0,0 +1,173 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { + mkdtempSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import type { SerializedLazyArchiveEntry } from "../host/src/vfs/memory-fs"; +import { + encodeHomebrewBottleMirrorCollectionIdentity, + encodeHomebrewBottleMirrorPlan, + HOMEBREW_BOTTLE_MIRROR_PLAN_ASSET, + HOMEBREW_BOTTLE_MIRROR_PLAN_KIND, + type HomebrewBottleMirrorPlan, +} from "../host/src/homebrew-vfs-composer"; +import { homebrewRuntimeLayerPayloadAsset } from + "../host/src/homebrew-runtime-layer-limits"; +import { + decodeHomebrewBottleMirrorPlan, + loadHomebrewBottleMirrorBindings, +} from "./homebrew-closed-lazy-assets"; + +const payloadBytes = new Uint8Array([1, 2, 3, 4]); + +test("loads exact local bytes under their immutable public URL identity", (t) => { + const root = mkdtempSync(join(tmpdir(), "kandelo-closed-bottles-")); + t.after(() => rmSync(root, { recursive: true, force: true })); + const fixture = createFixture(root); + + assert.deepEqual( + loadHomebrewBottleMirrorBindings( + fixture.planPath, + fixture.planBytes, + [fixture.pendingTree], + ), + [{ + url: fixture.plan.assets[0]!.url, + sha256: sha256(payloadBytes), + size: payloadBytes.byteLength, + bytes: payloadBytes, + }], + ); +}); + +test("rejects a plan that differs from the exact image-embedded bytes", (t) => { + const root = mkdtempSync(join(tmpdir(), "kandelo-closed-bottles-")); + t.after(() => rmSync(root, { recursive: true, force: true })); + const fixture = createFixture(root); + const changed = fixture.planBytes.slice(); + changed[0] ^= 1; + assert.throws( + () => loadHomebrewBottleMirrorBindings( + fixture.planPath, + changed, + [fixture.pendingTree], + ), + /differs from the exact VFS-embedded plan/, + ); +}); + +test("rejects symlinked payloads even when their bytes match", (t) => { + const root = mkdtempSync(join(tmpdir(), "kandelo-closed-bottles-")); + t.after(() => rmSync(root, { recursive: true, force: true })); + const fixture = createFixture(root, false); + const externalPayload = join(root, "external-payload"); + writeFileSync(externalPayload, payloadBytes); + symlinkSync(externalPayload, fixture.payloadPath); + assert.throws( + () => loadHomebrewBottleMirrorBindings( + fixture.planPath, + fixture.planBytes, + [fixture.pendingTree], + ), + /not a regular non-symlink file/, + ); +}); + +test("decodes only a structurally and derivationally valid mirror plan", () => { + const root = mkdtempSync(join(tmpdir(), "kandelo-closed-bottles-")); + try { + const fixture = createFixture(root); + assert.deepEqual( + decodeHomebrewBottleMirrorPlan(fixture.planBytes, "fixture"), + fixture.plan, + ); + const changed = encodeHomebrewBottleMirrorPlan({ + ...fixture.plan, + collection_sha256: "0".repeat(64), + }); + assert.throws( + () => decodeHomebrewBottleMirrorPlan(changed, "fixture"), + /inconsistent derived identity/, + ); + assert.throws( + () => decodeHomebrewBottleMirrorPlan( + new TextEncoder().encode(JSON.stringify(fixture.plan)), + "fixture", + ), + /bytes are not canonical/, + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +function createFixture(root: string, writePayload = true): { + plan: HomebrewBottleMirrorPlan; + planBytes: Uint8Array; + planPath: string; + payloadPath: string; + pendingTree: SerializedLazyArchiveEntry; +} { + const repository = "example/project"; + const identity = { + id: "bottle-test", + package: "example/tap/test", + asset: homebrewRuntimeLayerPayloadAsset("bottle-test"), + sha256: sha256(payloadBytes), + bytes: payloadBytes.byteLength, + }; + const collection = sha256( + encodeHomebrewBottleMirrorCollectionIdentity(repository, [identity]), + ); + const tag = `homebrew-shell-bottles-sha256-${collection}`; + const releaseRoot = + `https://github.com/${repository}/releases/download/${tag}`; + const plan: HomebrewBottleMirrorPlan = { + schema: 1, + kind: HOMEBREW_BOTTLE_MIRROR_PLAN_KIND, + repository, + collection_sha256: collection, + tag, + release_root: releaseRoot, + manifest_asset: HOMEBREW_BOTTLE_MIRROR_PLAN_ASSET, + assets: [{ ...identity, url: `${releaseRoot}/${identity.asset}` }], + }; + const planBytes = encodeHomebrewBottleMirrorPlan(plan); + const planPath = join(root, HOMEBREW_BOTTLE_MIRROR_PLAN_ASSET); + const payloadPath = join(root, identity.asset); + writeFileSync(planPath, planBytes); + if (writePayload) writeFileSync(payloadPath, payloadBytes); + return { + plan, + planBytes, + planPath, + payloadPath, + pendingTree: { + kind: "kandelo-deferred-tree-v2", + content: { + decoder: "homebrew-bottle-tar-gzip-v1", + mediaType: "application/vnd.oci.image.layer.v1.tar+gzip", + sha256: identity.sha256, + bytes: identity.bytes, + expandedBytes: 1, + sourceEntryCount: 1, + transports: [`${releaseRoot}/${identity.asset}`], + }, + url: `${releaseRoot}/${identity.asset}`, + mountPrefix: "/home/linuxbrew/.linuxbrew", + materialized: false, + entries: [], + }, + }; +} + +function sha256(bytes: Uint8Array): string { + return createHash("sha256").update(bytes).digest("hex"); +} diff --git a/scripts/homebrew-closed-lazy-assets.ts b/scripts/homebrew-closed-lazy-assets.ts new file mode 100644 index 0000000000..17ea8d7646 --- /dev/null +++ b/scripts/homebrew-closed-lazy-assets.ts @@ -0,0 +1,119 @@ +import { createHash } from "node:crypto"; +import { lstatSync, readFileSync } from "node:fs"; +import { basename, dirname, join } from "node:path"; + +import type { ClosedLazyAsset } from "../host/src/vfs/closed-lazy-assets"; +import type { SerializedLazyArchiveEntry } from "../host/src/vfs/memory-fs"; +import { + assertHomebrewBottleMirrorBundle, + assertHomebrewBottleMirrorPlan, + type HomebrewBottleMirrorPlan, +} from "../host/src/homebrew-vfs-composer"; +import { + assertPendingTreeHomebrewBottleMirrorBinding, + bytesEqual, + decodeHomebrewBottleMirrorPlan as decodeHomebrewBottleMirrorPlanStructure, + isRecord, +} from "./homebrew-closed-lazy-assets-contract"; + +export { + assertPendingTreeHomebrewBottleMirrorBinding, +} from "./homebrew-closed-lazy-assets-contract"; + +export function decodeHomebrewBottleMirrorPlan( + planBytes: Uint8Array, + label: string, +): HomebrewBottleMirrorPlan { + const plan = decodeHomebrewBottleMirrorPlanStructure(planBytes, label); + assertHomebrewBottleMirrorPlan(plan); + return plan; +} + +export function loadHomebrewBottleMirrorBindings( + planPath: string, + embeddedPlanBytes: Uint8Array, + pendingTrees: readonly SerializedLazyArchiveEntry[], +): ClosedLazyAsset[] { + const planStat = lstatSync(planPath); + if (!planStat.isFile() || planStat.isSymbolicLink()) { + throw new Error( + `bottle mirror plan is not a regular non-symlink file: ${planPath}`, + ); + } + const planBytes = new Uint8Array(readFileSync(planPath)); + if (!bytesEqual(planBytes, embeddedPlanBytes)) { + throw new Error( + "closed bottle mirror plan differs from the exact VFS-embedded plan", + ); + } + const plan = decodeHomebrewBottleMirrorPlan(planBytes, planPath); + const decoded = plan as unknown as Record; + if (!isRecord(decoded) || !Array.isArray(decoded.assets)) { + throw new Error("bottle mirror plan does not declare an asset array"); + } + if ( + typeof decoded.manifest_asset !== "string" || + basename(planPath) !== decoded.manifest_asset + ) { + throw new Error( + "bottle mirror plan filename differs from its declared asset name", + ); + } + + const mirrorDir = dirname(planPath); + const payloads = decoded.assets.map((value, index) => { + if ( + !isRecord(value) || + typeof value.id !== "string" || + typeof value.package !== "string" || + typeof value.asset !== "string" || + typeof value.sha256 !== "string" + ) { + throw new Error( + `bottle mirror asset ${index} has invalid identity fields`, + ); + } + if ( + value.asset === "." || + value.asset === ".." || + basename(value.asset) !== value.asset + ) { + throw new Error(`bottle mirror asset ${index} filename is not canonical`); + } + const assetPath = join(mirrorDir, value.asset); + const stat = lstatSync(assetPath); + if (!stat.isFile() || stat.isSymbolicLink()) { + throw new Error( + `bottle mirror asset is not a regular non-symlink file: ${assetPath}`, + ); + } + return { + id: value.id, + package: value.package, + asset: value.asset, + sha256: value.sha256, + bytes: new Uint8Array(readFileSync(assetPath)), + }; + }); + assertHomebrewBottleMirrorBundle(plan, payloads, { + asset: basename(planPath) as "kandelo-homebrew-bottle-mirror-plan.json", + sha256: createHash("sha256").update(planBytes).digest("hex"), + bytes: planBytes, + }); + assertPendingTreeHomebrewBottleMirrorBinding(pendingTrees, plan); + const payloadByPackage = new Map( + payloads.map((payload) => [payload.package, payload]), + ); + return plan.assets.map((asset): ClosedLazyAsset => { + const payload = payloadByPackage.get(asset.package); + if (payload === undefined) { + throw new Error(`bottle mirror payload is missing for ${asset.package}`); + } + return { + url: asset.url, + sha256: asset.sha256, + size: asset.bytes, + bytes: payload.bytes, + }; + }); +} diff --git a/scripts/homebrew-language-runtime-contract.ts b/scripts/homebrew-language-runtime-contract.ts index 3de0cf9bd0..ea1beee7c4 100644 --- a/scripts/homebrew-language-runtime-contract.ts +++ b/scripts/homebrew-language-runtime-contract.ts @@ -97,10 +97,10 @@ export const LANGUAGE_RUNTIME_INVOCATIONS: readonly LanguageRuntimeInvocation[] erlangInvocation("keg", `${ERLANG_KEG}/bin/erl`), ]; -export interface MainShellLanguageRuntimeInvocation - extends LanguageRuntimeInvocation { +export interface MainShellLanguageRuntimeInvocation extends LanguageRuntimeInvocation { packageName: string; dependencyPackages: readonly string[]; + launcherPackages: readonly string[]; terminalCommand: string; } @@ -154,16 +154,17 @@ const MAIN_SHELL_PERL_PROGRAM = [ 'print "main-shell-perl-ok:v5.40.3\\n"', ].join("; "); -const MAIN_SHELL_ERLANG_EXPRESSION = [ - 'ok = file:write_file("/tmp/kandelo-erlang-runtime.txt", <<"erlang-file-ok">>)', - '{ok, <<"erlang-file-ok">>} = file:read_file("/tmp/kandelo-erlang-runtime.txt")', - 'ok = file:delete("/tmp/kandelo-erlang-runtime.txt")', - 'Parent = self()', - 'spawn(fun() -> Parent ! {child, lists:sum([1,2,3])} end)', - 'receive {child, 6} -> ok after 5000 -> erlang:error(child_timeout) end', - 'io:format("main-shell-erlang-ok:28.2~n")', - "halt()", -].join(", ") + "."; +const MAIN_SHELL_ERLANG_EXPRESSION = + [ + 'ok = file:write_file("/tmp/kandelo-erlang-runtime.txt", <<"erlang-file-ok">>)', + '{ok, <<"erlang-file-ok">>} = file:read_file("/tmp/kandelo-erlang-runtime.txt")', + 'ok = file:delete("/tmp/kandelo-erlang-runtime.txt")', + "Parent = self()", + "spawn(fun() -> Parent ! {child, lists:sum([1,2,3])} end)", + "receive {child, 6} -> ok after 5000 -> erlang:error(child_timeout) end", + 'io:format("main-shell-erlang-ok:28.2~n")', + "halt()", + ].join(", ") + "."; const MAIN_SHELL_RUBY_PROGRAM = [ "raise 'RUBYLIB leaked' if ENV.key?('RUBYLIB')", @@ -171,7 +172,7 @@ const MAIN_SHELL_RUBY_PROGRAM = [ "require 'rbconfig'", "prefix = RbConfig::CONFIG['prefix']", `allowed = ['${HOMEBREW_PREFIX}/opt/ruby', '${HOMEBREW_PREFIX}/Cellar/ruby/4.0.5_1']`, - "raise \"wrong Ruby prefix: #{prefix}\" unless allowed.include?(prefix)", + 'raise "wrong Ruby prefix: #{prefix}" unless allowed.include?(prefix)', "require 'pathname'", "require 'json'", "require 'yaml'", @@ -195,6 +196,7 @@ function mainShellInvocation( label: string, packageName: string, dependencyPackages: readonly string[], + launcherPackages: readonly string[], command: string, args: readonly string[], expectedStdout: string, @@ -204,6 +206,7 @@ function mainShellInvocation( label, packageName, dependencyPackages, + launcherPackages, executable: SHELL, argv, expectedStdout, @@ -221,12 +224,13 @@ function shellQuote(value: string): string { * the main shell. These deliberately use only the normal PATH and package * wrappers: no language-specific runtime or library-path overrides are allowed. */ -export const MAIN_SHELL_LANGUAGE_RUNTIME_INVOCATIONS: - readonly MainShellLanguageRuntimeInvocation[] = [ +export const MAIN_SHELL_LANGUAGE_RUNTIME_INVOCATIONS: readonly MainShellLanguageRuntimeInvocation[] = + [ mainShellInvocation( "main-shell Python", "kandelo-dev/tap-core/python", ["kandelo-dev/tap-core/zlib"], + [], "python", ["-c", MAIN_SHELL_PYTHON_PROGRAM], "main-shell-python-ok:3.13.3\n", @@ -235,6 +239,7 @@ export const MAIN_SHELL_LANGUAGE_RUNTIME_INVOCATIONS: "main-shell Perl", "kandelo-dev/tap-core/perl", [], + [], "perl", ["-e", MAIN_SHELL_PERL_PROGRAM], "main-shell-perl-ok:v5.40.3\n", @@ -243,6 +248,9 @@ export const MAIN_SHELL_LANGUAGE_RUNTIME_INVOCATIONS: "main-shell Erlang", "kandelo-dev/tap-core/erlang", [], + // WHY: erl is installed as a #!/bin/sh wrapper. Track the selected + // shell bottle explicitly without pretending it is an Erlang library. + ["kandelo-dev/tap-core/dash"], "erl", [...ERLANG_ARGS, "-eval", MAIN_SHELL_ERLANG_EXPRESSION], "main-shell-erlang-ok:28.2\n", @@ -251,6 +259,7 @@ export const MAIN_SHELL_LANGUAGE_RUNTIME_INVOCATIONS: "main-shell Ruby", "kandelo-dev/tap-core/ruby", ["kandelo-dev/tap-core/zlib"], + [], "ruby", ["-e", MAIN_SHELL_RUBY_PROGRAM], "main-shell-ruby-ok:4.0.5:rubygems-4.0.10:bundler-4.0.10\n", diff --git a/scripts/homebrew-main-shell-catalog-contract.ts b/scripts/homebrew-main-shell-catalog-contract.ts new file mode 100644 index 0000000000..0d3f5096ca --- /dev/null +++ b/scripts/homebrew-main-shell-catalog-contract.ts @@ -0,0 +1,59 @@ +export interface MainShellCatalogIdentity { + tapRepository: string; + tapName: string; + tapCommit: string; +} + +/** + * Validate the immutable catalog identity from the guest-visible composition + * descriptor without importing Node-only image-build helpers. + */ +export function assertMainShellGuestCatalogIdentity( + guestManifest: unknown, + expected: MainShellCatalogIdentity, +): void { + const guest = requiredRecord(guestManifest, "guest Homebrew manifest"); + expectEqual(guest.schema, 1, "guest Homebrew manifest schema"); + const catalog = requiredRecord( + guest.catalog, + "guest Homebrew catalog", + ); + expectEqual( + catalog.tap_repository, + expected.tapRepository, + "guest Homebrew catalog tap_repository", + ); + expectEqual( + catalog.tap_name, + expected.tapName, + "guest Homebrew catalog tap_name", + ); + expectEqual( + catalog.checkout_commit, + expected.tapCommit, + "guest Homebrew catalog checkout_commit", + ); +} + +function requiredRecord( + value: unknown, + label: string, +): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error(`Homebrew main-shell image contract: ${label} must be an object`); + } + return value as Record; +} + +function expectEqual( + actual: unknown, + expected: unknown, + label: string, +): void { + if (actual !== expected) { + throw new Error( + `Homebrew main-shell image contract: ${label} is ` + + `${JSON.stringify(actual)}, expected ${JSON.stringify(expected)}`, + ); + } +} diff --git a/scripts/homebrew-main-shell-image-contract.test.ts b/scripts/homebrew-main-shell-image-contract.test.ts index 5660e628fd..23ccded14e 100644 --- a/scripts/homebrew-main-shell-image-contract.test.ts +++ b/scripts/homebrew-main-shell-image-contract.test.ts @@ -3,7 +3,10 @@ import { createHash } from "node:crypto"; import { readFileSync } from "node:fs"; import test from "node:test"; import { resolve } from "node:path"; -import { assertMainShellImageContract } from "./homebrew-main-shell-image-contract"; +import { + assertMainShellGuestCatalogIdentity, + assertMainShellImageContract, +} from "./homebrew-main-shell-image-contract"; const lock = JSON.parse( readFileSync(resolve("homebrew/main-shell-migration-lock.json"), "utf8"), @@ -206,6 +209,31 @@ test("accepts the exact reviewed root and Formula identities", () => { assert.doesNotThrow(() => assertMainShellImageContract(fixture())); }); +test("shares the authoritative guest catalog parser with narrow consumers", () => { + const guestManifest = fixture().guestManifest as Record; + const expected = { + tapRepository: lock.tap_repository, + tapName: lock.tap_name, + tapCommit: lock.catalog.tap_commit, + }; + assert.doesNotThrow(() => + assertMainShellGuestCatalogIdentity(guestManifest, expected) + ); + + for (const [key, replacement, message] of [ + ["tap_repository", "someone/else", "tap_repository"], + ["tap_name", "someone/else", "tap_name"], + ["checkout_commit", "0".repeat(40), "checkout_commit"], + ] as const) { + const changed = structuredClone(guestManifest); + changed.catalog[key] = replacement; + assert.throws( + () => assertMainShellGuestCatalogIdentity(changed, expected), + new RegExp(message), + ); + } +}); + for (const [name, mutate, expected] of [ [ "rejects a substituted requested root", diff --git a/scripts/homebrew-main-shell-image-contract.ts b/scripts/homebrew-main-shell-image-contract.ts index cd4cd6a2f8..398ef67ae1 100644 --- a/scripts/homebrew-main-shell-image-contract.ts +++ b/scripts/homebrew-main-shell-image-contract.ts @@ -12,6 +12,13 @@ import { DOOM_WAD_SHA256, DOOM_WAD_URL, } from "../web-libs/kandelo-session/src/demo-guides"; +import { + assertMainShellGuestCatalogIdentity, +} from "./homebrew-main-shell-catalog-contract"; +export { + assertMainShellGuestCatalogIdentity, + type MainShellCatalogIdentity, +} from "./homebrew-main-shell-catalog-contract"; const EXPECTED_ARCH = "wasm32"; const EXPECTED_SHELL_PATH = "/home/linuxbrew/.linuxbrew/bin/bash"; @@ -93,11 +100,11 @@ export function assertMainShellImageContract(input: MainShellImageContractInput) requestedPackagesSha256, "guest Homebrew requested_packages_sha256", ); - assertCatalog(requiredRecord(guest.catalog, "guest Homebrew catalog"), { + assertMainShellGuestCatalogIdentity(guest, { tapRepository, tapName, tapCommit, - }, "guest Homebrew catalog", "snake"); + }); assertLockBinding( requiredRecord(guest.migration_lock, "guest Homebrew migration_lock"), input, diff --git a/scripts/homebrew-main-shell-node-smoke.ts b/scripts/homebrew-main-shell-node-smoke.ts index 30ff7395f8..d5926cafeb 100755 --- a/scripts/homebrew-main-shell-node-smoke.ts +++ b/scripts/homebrew-main-shell-node-smoke.ts @@ -2,7 +2,8 @@ import { createHash } from "node:crypto"; import { lstatSync, readFileSync } from "node:fs"; -import { basename, dirname, join, posix, resolve } from "node:path"; +import { dirname, posix, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; import { NodeKernelHost } from "../host/src/node-kernel-host"; import { MemoryFileSystem, @@ -11,11 +12,19 @@ import { } from "../host/src/vfs/memory-fs"; import type { ClosedLazyAsset } from "../host/src/vfs/closed-lazy-assets"; import { - assertHomebrewBottleMirrorBundle, - assertHomebrewBottleMirrorPlan, + assertPackageDeferredZipTreeState, + derivePackageDeferredZipTree, + type DerivedPackageDeferredZipTree, +} from "../host/src/vfs/package-deferred-tree"; +import { HOMEBREW_BOTTLE_MIRROR_PLAN_VFS_PATH, type HomebrewBottleMirrorPlan, } from "../host/src/homebrew-vfs-composer"; +import { + assertPendingTreeHomebrewBottleMirrorBinding, + decodeHomebrewBottleMirrorPlan, + loadHomebrewBottleMirrorBindings, +} from "./homebrew-closed-lazy-assets"; import { assertMainShellImageContract } from "./homebrew-main-shell-image-contract"; import { KANDELO_DEMO_CONFIG_PATH } from "../web-libs/kandelo-session/src/demo-config"; import { @@ -32,6 +41,10 @@ import { MAIN_SHELL_LANGUAGE_RUNTIME_INVOCATIONS } from "./homebrew-language-run const { imagePath, migrationLockPath, + homebrewBootstrapSpecPath, + homebrewBootstrapArchivePath, + homebrewBootstrapEnvPath, + homebrewBootstrapState, demoConfigPath, transportMode, bottleMirrorPlanPath, @@ -41,10 +54,34 @@ const BASE_EXPECTED_FETCHED_PACKAGES = [ "kandelo-dev/tap-core/git", "kandelo-dev/tap-core/nethack", ] as const; +const BREW_EXPECTED_FETCHED_PACKAGES = [ + "kandelo-dev/tap-core/coreutils", + "kandelo-dev/tap-core/posix-utils-lite", + "kandelo-dev/tap-core/ruby", + "kandelo-dev/tap-core/zlib", +] as const; const LANGUAGE_PACKAGE_NAMES = new Set( MAIN_SHELL_LANGUAGE_RUNTIME_INVOCATIONS.map(({ packageName }) => packageName), ); const imageBytes = new Uint8Array(readFileSync(imagePath)); +const homebrewBootstrapArchiveBytes = readRegularFile( + homebrewBootstrapArchivePath, + "Homebrew bootstrap package output", +); +const homebrewBootstrapEnvBytes = readRegularFile( + homebrewBootstrapEnvPath, + "Homebrew bootstrap launcher environment", +); +const homebrewBootstrapTree = derivePackageDeferredZipTree( + parseJson( + readRegularFile( + homebrewBootstrapSpecPath, + "Homebrew bootstrap package-tree spec", + ), + homebrewBootstrapSpecPath, + ), + homebrewBootstrapArchiveBytes, +); const metadata = MemoryFileSystem.readImageMetadata(imageBytes); const capacity = MemoryFileSystem.readImageCapacity(imageBytes); assertVfsImageFitsProfile( @@ -57,6 +94,21 @@ assertVfsImageFitsProfile( const fs = MemoryFileSystem.fromImage(imageBytes, { maxByteLength: MAIN_SHELL_VFS_PROFILE_MAX_BYTES, }); +assertPackageDeferredZipTreeState( + fs, + homebrewBootstrapTree, + homebrewBootstrapState, +); +assertHomebrewBootstrapTreeMetadata( + metadata, + homebrewBootstrapTree, + homebrewBootstrapState, +); +assertHomebrewBootstrapConsumerContract( + fs, + metadata, + homebrewBootstrapEnvBytes, +); const migrationLockBytes = new Uint8Array(readFileSync(migrationLockPath)); const migrationLock = parseJson(migrationLockBytes, migrationLockPath); const demoConfigSource = readVfsFile(fs, KANDELO_DEMO_CONFIG_PATH); @@ -65,14 +117,18 @@ const guestManifest = parseJson( "/etc/kandelo/homebrew-vfs.json", ); const shellConfig = parseKandeloShellConfig( - new TextDecoder("utf-8", { fatal: true }).decode(readVfsFile(fs, KANDELO_SHELL_CONFIG_PATH)), + new TextDecoder("utf-8", { fatal: true }).decode( + readVfsFile(fs, KANDELO_SHELL_CONFIG_PATH), + ), ); if (shellConfig === null) { throw new Error(`${KANDELO_SHELL_CONFIG_PATH} has an unsupported schema`); } assertMainShellImageContract({ migrationLock, - migrationLockSha256: createHash("sha256").update(migrationLockBytes).digest("hex"), + migrationLockSha256: createHash("sha256") + .update(migrationLockBytes) + .digest("hex"), migrationLockBytes: migrationLockBytes.byteLength, guestManifest, imageMetadata: metadata, @@ -82,17 +138,45 @@ assertMainShellImageContract({ expectedDemoConfigSource: new Uint8Array(readFileSync(demoConfigPath)), runtimeState: readRuntimeState(fs, migrationLock), }); -const pendingTrees = fs.exportLazyArchiveEntries().filter( - (tree) => tree.content !== undefined, +const allPendingTrees = fs + .exportLazyArchiveEntries() + .filter((tree) => tree.content !== undefined); +const pendingTrees = allPendingTrees.filter((tree) => + tree.activation?.capabilities.some((capability) => + capability.startsWith("homebrew-bottle:"), + ), +); +const pendingBootstrapTrees = allPendingTrees.filter((tree) => + tree.activation?.capabilities.includes("homebrew:bootstrap"), +); +const unknownPendingTrees = allPendingTrees.filter( + (tree) => + !pendingTrees.includes(tree) && !pendingBootstrapTrees.includes(tree), ); +if (unknownPendingTrees.length !== 0) { + throw new Error( + `main-shell image has ${unknownPendingTrees.length} unclassified pending package trees`, + ); +} +if ( + pendingBootstrapTrees.length !== + (homebrewBootstrapState === "deferred" ? 1 : 0) +) { + throw new Error( + `main-shell image has ${pendingBootstrapTrees.length} pending Homebrew source trees; ` + + `expected ${homebrewBootstrapState === "deferred" ? 1 : 0}`, + ); +} if (fs.isPathDeferred(shellConfig.path)) { - throw new Error(`image-owned default shell remains deferred: ${shellConfig.path}`); + throw new Error( + `image-owned default shell remains deferred: ${shellConfig.path}`, + ); } const embeddedMirrorPlanBytes = readVfsFile( fs, HOMEBREW_BOTTLE_MIRROR_PLAN_VFS_PATH, ); -const mirrorPlan = decodeBottleMirrorPlan( +const mirrorPlan = decodeHomebrewBottleMirrorPlan( embeddedMirrorPlanBytes, HOMEBREW_BOTTLE_MIRROR_PLAN_VFS_PATH, ); @@ -102,17 +186,42 @@ if (pendingTrees.length !== mirrorPlan.assets.length) { `mirror plan declares ${mirrorPlan.assets.length}`, ); } -assertPendingTreeMirrorBinding(pendingTrees, mirrorPlan); -const closedLazyAssets = transportMode === "closed" - ? loadBottleMirrorBindings( - bottleMirrorPlanPath!, - embeddedMirrorPlanBytes, - pendingTrees, - ) - : undefined; -const posixShell = assertRetainedPosixShellAlias(fs, migrationLock, guestManifest); +assertPendingTreeHomebrewBottleMirrorBinding(pendingTrees, mirrorPlan); +const homebrewBootstrapLazyBase = + transportMode === "closed" + ? "https://closed.kandelo.invalid/main-shell/" + : pathToFileURL(`${dirname(homebrewBootstrapArchivePath)}/`).toString(); +const homebrewBootstrapTransportUrl = new URL( + homebrewBootstrapTree.descriptor.archive.url, + homebrewBootstrapLazyBase, +).toString(); +const closedLazyAssets = + transportMode === "closed" + ? [ + ...loadHomebrewBottleMirrorBindings( + bottleMirrorPlanPath!, + embeddedMirrorPlanBytes, + pendingTrees, + ), + ...(homebrewBootstrapState === "deferred" + ? [ + { + url: homebrewBootstrapTransportUrl, + sha256: homebrewBootstrapTree.descriptor.archive.sha256, + size: homebrewBootstrapTree.descriptor.archive.bytes, + bytes: homebrewBootstrapArchiveBytes, + } satisfies ClosedLazyAsset, + ] + : []), + ] + : undefined; +const posixShell = assertRetainedPosixShellAlias( + fs, + migrationLock, + guestManifest, +); const pendingPosixShellTrees = pendingTrees.filter((tree) => - tree.entries.some((entry) => entry.vfsPath === posixShell.executablePath) + tree.entries.some((entry) => entry.vfsPath === posixShell.executablePath), ); if (pendingPosixShellTrees.length !== 1) { throw new Error( @@ -121,16 +230,36 @@ if (pendingPosixShellTrees.length !== 1) { ); } const shellBytes = readVfsBinary(fs, shellConfig.path); +await proveLanguageIsolationOnFreshHost({ + imageBytes, + shellBytes, + homebrewBootstrapState, + homebrewBootstrapLazyBase, + homebrewBootstrapTransportUrl, + closedLazyAssets, + pendingTrees, + mirrorPlan, + guestManifest, +}); let stdout = ""; let stderr = ""; const lazyDownloads: LazyDownloadEvent[] = []; const host = new NodeKernelHost({ maxWorkers: 8, rootfsImage: imageBytes, + ...(homebrewBootstrapState === "deferred" + ? { rootfsLazyUrlBase: homebrewBootstrapLazyBase } + : {}), rootfsLazyAssets: closedLazyAssets, - onStdout: (_pid, data) => { stdout += new TextDecoder().decode(data); }, - onStderr: (_pid, data) => { stderr += new TextDecoder().decode(data); }, - onLazyDownload: (event) => { lazyDownloads.push(event); }, + onStdout: (_pid, data) => { + stdout += new TextDecoder().decode(data); + }, + onStderr: (_pid, data) => { + stderr += new TextDecoder().decode(data); + }, + onLazyDownload: (event) => { + lazyDownloads.push(event); + }, }); await host.init(); @@ -237,37 +366,98 @@ printf 'homebrew-nethack-state-ok\\n' "base shell compatibility surface", ); - for (const invocation of MAIN_SHELL_LANGUAGE_RUNTIME_INVOCATIONS) { - const eventStart = lazyDownloads.length; - const stdoutStart = stdout.length; - const stderrStart = stderr.length; - await spawnWithTimeout( - host, - shellBytes, - invocation.argv, - invocation.label, - () => ({ stdout: stdout.slice(stdoutStart), stderr: stderr.slice(stderrStart) }), + assertNoTransportForUrl( + lazyDownloads, + homebrewBootstrapTransportUrl, + "kernel initialization and non-brew shell commands", + ); + + const brewEventStart = lazyDownloads.length; + const brewStdoutStart = stdout.length; + const brewStderrStart = stderr.length; + const brewCommand = ` +set -eu +test -x /usr/bin/brew +brew_version="$(/usr/bin/brew --version 2>&1)" +case "$brew_version" in + "Homebrew "*) ;; + *) printf 'unexpected brew version: %s\n' "$brew_version" >&2; exit 1 ;; +esac +test "$(/usr/bin/brew --prefix 2>&1)" = /home/linuxbrew/.linuxbrew +test "$(/usr/bin/brew --repository 2>&1)" = /home/linuxbrew/.linuxbrew +test "$(/usr/bin/brew --cellar 2>&1)" = /home/linuxbrew/.linuxbrew/Cellar +test "$(/usr/bin/brew --cache 2>&1)" = /home/user/.cache/Homebrew +mkdir -p /home/linuxbrew/.linuxbrew/etc/homebrew /home/user/.homebrew +printf 'HOMEBREW_KANDELO_BOTTLE_TAG=wasm64_kandelo\n' > /home/linuxbrew/.linuxbrew/etc/homebrew/brew.env +printf 'HOMEBREW_KANDELO_BOTTLE_TAG=wasm64_kandelo\n' > /home/user/.homebrew/brew.env +test "$(/usr/bin/brew ruby -e 'print ENV.fetch("HOMEBREW_KANDELO_BOTTLE_TAG")' 2>&1)" = wasm32_kandelo +printf 'homebrew-ordinary-brew-ok\n' +`.trim(); + await spawnWithTimeout( + host, + shellBytes, + [shellConfig.argv[0], "-c", brewCommand], + "ordinary upstream Homebrew phase", + () => ({ + stdout: stdout.slice(brewStdoutStart), + stderr: stderr.slice(brewStderrStart), + }), + ); + const brewStdout = stdout.slice(brewStdoutStart); + const brewStderr = stderr.slice(brewStderrStart); + if (brewStdout !== "homebrew-ordinary-brew-ok\n" || brewStderr !== "") { + throw new Error( + `ordinary Homebrew command returned unexpected output; ` + + `stdout=${JSON.stringify(brewStdout)} stderr=${JSON.stringify(brewStderr)}`, ); - const runtimeStdout = stdout.slice(stdoutStart); - const runtimeStderr = stderr.slice(stderrStart); - if (runtimeStdout !== invocation.expectedStdout || runtimeStderr !== "") { - throw new Error( - `${invocation.label} returned unexpected output; ` + - `stdout=${JSON.stringify(runtimeStdout)} stderr=${JSON.stringify(runtimeStderr)}`, - ); - } - assertLanguageBottleIsolation( - invocation.packageName, - invocation.dependencyPackages, - lazyDownloads.slice(eventStart), - pendingTrees, - mirrorPlan, - guestManifest, - invocation.label, + } + const brewEvents = lazyDownloads.slice(brewEventStart); + assertHomebrewBootstrapTransport( + brewEvents, + homebrewBootstrapTree, + homebrewBootstrapTransportUrl, + homebrewBootstrapState, + ); + assertFetchedPackageSet( + withoutTransportUrl(brewEvents, homebrewBootstrapTransportUrl), + pendingTrees, + mirrorPlan, + BREW_EXPECTED_FETCHED_PACKAGES, + "ordinary upstream Homebrew first use", + ); + + const repeatBrewEventStart = lazyDownloads.length; + const repeatBrewStdoutStart = stdout.length; + const repeatBrewStderrStart = stderr.length; + await spawnWithTimeout( + host, + shellBytes, + [shellConfig.argv[0], "-c", "/usr/bin/brew --prefix"], + "ordinary upstream Homebrew repeat phase", + () => ({ + stdout: stdout.slice(repeatBrewStdoutStart), + stderr: stderr.slice(repeatBrewStderrStart), + }), + ); + const repeatBrewStdout = stdout.slice(repeatBrewStdoutStart); + const repeatBrewStderr = stderr.slice(repeatBrewStderrStart); + if ( + repeatBrewStdout !== "/home/linuxbrew/.linuxbrew\n" || + repeatBrewStderr !== "" + ) { + throw new Error( + `repeated ordinary Homebrew command returned unexpected output; ` + + `stdout=${JSON.stringify(repeatBrewStdout)} ` + + `stderr=${JSON.stringify(repeatBrewStderr)}`, ); } + assertNoLazyTransport( + lazyDownloads.slice(repeatBrewEventStart), + "repeated ordinary Homebrew use", + ); + const transportEvidence = assertBottleTransportEvents( - lazyDownloads, + withoutTransportUrl(lazyDownloads, homebrewBootstrapTransportUrl), pendingTrees, mirrorPlan, guestManifest, @@ -277,7 +467,7 @@ printf 'homebrew-nethack-state-ok\\n' `Homebrew main-shell Node smoke: exact ${counts.roots}-root/` + `${counts.formulae}-Formula archive, image-owned ` + "offline Bash, retained /bin/sh, metadata/runtime state, /dev/null, and " + - "isolated Python/Perl/Erlang/Ruby first use passed " + + "ordinary brew plus isolated Python/Perl/Erlang/Ruby first use passed " + `(${transportEvidence.bottles} bottles, ` + `${transportEvidence.bytes} bytes).`, ); @@ -285,127 +475,215 @@ printf 'homebrew-nethack-state-ok\\n' await host.destroy().catch(() => {}); } -function loadBottleMirrorBindings( - planPath: string, - embeddedManifestBytes: Uint8Array, - pendingTrees: readonly SerializedLazyArchiveEntry[], -): ClosedLazyAsset[] { - const planStat = lstatSync(planPath); - if (!planStat.isFile() || planStat.isSymbolicLink()) { - throw new Error(`bottle mirror plan is not a regular non-symlink file: ${planPath}`); +async function proveLanguageIsolationOnFreshHost(options: { + imageBytes: Uint8Array; + shellBytes: Uint8Array; + homebrewBootstrapState: "deferred" | "materialized"; + homebrewBootstrapLazyBase: string; + homebrewBootstrapTransportUrl: string; + closedLazyAssets: readonly ClosedLazyAsset[] | undefined; + pendingTrees: readonly SerializedLazyArchiveEntry[]; + mirrorPlan: HomebrewBottleMirrorPlan; + guestManifest: unknown; +}): Promise { + let stdout = ""; + let stderr = ""; + const lazyDownloads: LazyDownloadEvent[] = []; + const host = new NodeKernelHost({ + maxWorkers: 8, + rootfsImage: options.imageBytes, + ...(options.homebrewBootstrapState === "deferred" + ? { rootfsLazyUrlBase: options.homebrewBootstrapLazyBase } + : {}), + rootfsLazyAssets: options.closedLazyAssets, + onStdout: (_pid, data) => { + stdout += new TextDecoder().decode(data); + }, + onStderr: (_pid, data) => { + stderr += new TextDecoder().decode(data); + }, + onLazyDownload: (event) => { + lazyDownloads.push(event); + }, + }); + await host.init(); + try { + // WHY: brew itself starts the bottled Ruby runtime. A separate pristine + // machine keeps the language proof independent while the primary machine + // remains a truthful brew-first proof with Ruby still deferred. + for (const invocation of MAIN_SHELL_LANGUAGE_RUNTIME_INVOCATIONS) { + const eventStart = lazyDownloads.length; + const stdoutStart = stdout.length; + const stderrStart = stderr.length; + await spawnWithTimeout( + host, + options.shellBytes, + invocation.argv, + invocation.label, + () => ({ + stdout: stdout.slice(stdoutStart), + stderr: stderr.slice(stderrStart), + }), + ); + const runtimeStdout = stdout.slice(stdoutStart); + const runtimeStderr = stderr.slice(stderrStart); + if (runtimeStdout !== invocation.expectedStdout || runtimeStderr !== "") { + throw new Error( + `${invocation.label} returned unexpected output; ` + + `stdout=${JSON.stringify(runtimeStdout)} stderr=${JSON.stringify(runtimeStderr)}`, + ); + } + assertLanguageBottleIsolation( + invocation.packageName, + invocation.dependencyPackages, + invocation.launcherPackages, + lazyDownloads.slice(eventStart), + options.pendingTrees, + options.mirrorPlan, + options.guestManifest, + invocation.label, + ); + } + assertNoTransportForUrl( + lazyDownloads, + options.homebrewBootstrapTransportUrl, + "fresh-machine language commands", + ); + } finally { + await host.destroy().catch(() => {}); } - const manifestBytes = new Uint8Array(readFileSync(planPath)); +} + +function assertHomebrewBootstrapTreeMetadata( + metadata: unknown, + tree: DerivedPackageDeferredZipTree, + state: "deferred" | "materialized", +): void { + const imageMetadata = asRecord(metadata, "main-shell image metadata"); + if (!Array.isArray(imageMetadata.packageDeferredTrees)) { + throw new Error("main-shell image metadata omits packageDeferredTrees"); + } + const descriptor = tree.descriptor; + const expected = [ + { + schema: descriptor.schema, + kind: descriptor.kind, + id: descriptor.id, + content_role: descriptor.content_role, + package: descriptor.package, + descriptor: { + sha256: tree.descriptorSha256, + bytes: tree.descriptorBytes.byteLength, + }, + archive: { + output: descriptor.package.output, + url: descriptor.archive.url, + sha256: descriptor.archive.sha256, + bytes: descriptor.archive.bytes, + expanded_bytes: descriptor.archive.expanded_bytes, + source_entry_count: descriptor.archive.source_entry_count, + }, + mount_prefix: descriptor.mount_prefix, + owner: descriptor.owner, + activation: descriptor.activation, + state, + }, + ]; if ( - manifestBytes.byteLength !== embeddedManifestBytes.byteLength || - !manifestBytes.every((byte, index) => byte === embeddedManifestBytes[index]) + canonicalJson(imageMetadata.packageDeferredTrees) !== + canonicalJson(expected) ) { - throw new Error("closed bottle mirror plan differs from the exact VFS-embedded plan"); + throw new Error( + "main-shell package-tree metadata differs from the exact Homebrew package output", + ); } - const plan = decodeBottleMirrorPlan(manifestBytes, planPath); - const decoded = plan as unknown as Record; - if (!isRecord(decoded) || !Array.isArray(decoded.assets)) { - throw new Error("bottle mirror plan does not declare an asset array"); +} + +function assertHomebrewBootstrapConsumerContract( + fs: MemoryFileSystem, + metadata: unknown, + expectedEnvironment: Uint8Array, +): void { + const environmentPath = "/etc/homebrew/brew.env"; + const entrypointPath = "/usr/bin/brew"; + const target = "/home/linuxbrew/.linuxbrew/bin/brew"; + const actualEnvironment = readVfsFile(fs, environmentPath); + if ( + actualEnvironment.byteLength !== expectedEnvironment.byteLength || + !actualEnvironment.every( + (byte, index) => byte === expectedEnvironment[index], + ) + ) { + throw new Error( + "main-shell Homebrew environment differs from its package output", + ); } + const entrypoint = fs.lstat(entrypointPath); if ( - typeof decoded.manifest_asset !== "string" || - basename(planPath) !== decoded.manifest_asset + (entrypoint.mode & 0xf000) !== 0xa000 || + fs.readlink(entrypointPath) !== target ) { - throw new Error("bottle mirror plan filename differs from its declared asset name"); + throw new Error( + "main-shell does not expose the canonical /usr/bin/brew alias", + ); } + assertTreeOwner(fs, "/home/linuxbrew/.linuxbrew", 1000, 1000); + assertTreeOwner(fs, "/home/user/.cache", 1000, 1000); - const mirrorDir = dirname(planPath); - const payloads = decoded.assets.map((value, index) => { - if ( - !isRecord(value) || typeof value.id !== "string" || - typeof value.package !== "string" || typeof value.asset !== "string" || - typeof value.sha256 !== "string" - ) { - throw new Error(`bottle mirror asset ${index} has invalid identity fields`); - } - if (value.asset === "." || value.asset === ".." || basename(value.asset) !== value.asset) { - throw new Error(`bottle mirror asset ${index} filename is not canonical`); - } - const assetPath = join(mirrorDir, value.asset); - const stat = lstatSync(assetPath); - if (!stat.isFile() || stat.isSymbolicLink()) { - throw new Error( - `bottle mirror asset is not a regular non-symlink file: ${assetPath}`, - ); - } - const bytes = new Uint8Array(readFileSync(assetPath)); - return { - id: value.id, - package: value.package, - asset: value.asset, - sha256: value.sha256, - bytes, - }; - }); - assertHomebrewBottleMirrorBundle(plan, payloads, { - asset: basename(planPath) as "kandelo-homebrew-bottle-mirror-plan.json", - sha256: createHash("sha256").update(manifestBytes).digest("hex"), - bytes: manifestBytes, - }); - assertPendingTreeMirrorBinding(pendingTrees, plan); - const payloadByPackage = new Map(payloads.map((payload) => [payload.package, payload])); - return plan.assets.map((asset): ClosedLazyAsset => { - const payload = payloadByPackage.get(asset.package)!; - return { - url: asset.url, - sha256: asset.sha256, - size: asset.bytes, - bytes: payload.bytes, - }; - }); -} - -function decodeBottleMirrorPlan( - manifestBytes: Uint8Array, - label: string, -): HomebrewBottleMirrorPlan { - const decoded = parseJson(manifestBytes, label); - if (!isRecord(decoded) || !Array.isArray(decoded.assets)) { - throw new Error(`${label} does not declare a bottle mirror asset array`); + const imageMetadata = asRecord(metadata, "main-shell image metadata"); + const expected = { + environment: { + path: environmentPath, + sha256: createHash("sha256").update(expectedEnvironment).digest("hex"), + bytes: expectedEnvironment.byteLength, + }, + entrypoint: { path: entrypointPath, target }, + ownership: { + prefix: "/home/linuxbrew/.linuxbrew", + uid: 1000, + gid: 1000, + mutable_paths: [ + "/home/linuxbrew/.linuxbrew/Cellar", + "/home/linuxbrew/.linuxbrew/Library/Taps", + "/home/linuxbrew/.linuxbrew/var/homebrew/linked", + "/home/linuxbrew/.linuxbrew/var/homebrew/locks", + "/home/user/.cache/Homebrew", + ], + }, + }; + if ( + canonicalJson(imageMetadata.homebrewBootstrap) !== canonicalJson(expected) + ) { + throw new Error("main-shell Homebrew consumer metadata changed"); } - const plan = decoded as unknown as HomebrewBottleMirrorPlan; - assertHomebrewBottleMirrorPlan(plan); - return plan; } -function assertPendingTreeMirrorBinding( - pendingTrees: readonly SerializedLazyArchiveEntry[], - plan: HomebrewBottleMirrorPlan, +function assertTreeOwner( + fs: MemoryFileSystem, + root: string, + uid: number, + gid: number, ): void { - if (pendingTrees.length !== plan.assets.length) { - throw new Error( - `pending tree count ${pendingTrees.length} differs from mirror asset count ` + - `${plan.assets.length}`, - ); - } - const assetByUrl = new Map(plan.assets.map((asset) => [asset.url, asset])); - if (assetByUrl.size !== plan.assets.length) { - throw new Error("bottle mirror plan duplicates a release URL"); + const stat = fs.lstat(root); + if (stat.uid !== uid || stat.gid !== gid) { + throw new Error(`main-shell Homebrew path has the wrong owner: ${root}`); } - const seen = new Set(); - for (const tree of pendingTrees) { - const content = tree.content; - const primaryUrl = content?.transports[0]; - const asset = primaryUrl === undefined ? undefined : assetByUrl.get(primaryUrl); - if ( - content === undefined || asset === undefined || - content.sha256 !== asset.sha256 || content.bytes !== asset.bytes - ) { - throw new Error( - `pending tree ${tree.mountPrefix} does not match one exact mirror asset`, + if ((stat.mode & 0xf000) !== 0x4000) return; + const handle = fs.opendir(root); + try { + for (;;) { + const entry = fs.readdir(handle); + if (entry === null) break; + if (entry.name === "." || entry.name === "..") continue; + assertTreeOwner( + fs, + root === "/" ? `/${entry.name}` : `${root}/${entry.name}`, + uid, + gid, ); } - if (seen.has(primaryUrl!)) { - throw new Error(`multiple pending trees use mirror URL ${primaryUrl}`); - } - seen.add(primaryUrl!); - } - if (seen.size !== plan.assets.length) { - throw new Error("pending trees do not cover the complete bottle mirror plan"); + } finally { + fs.closedir(handle); } } @@ -421,6 +699,82 @@ function assertNoLazyTransport( } } +function assertNoTransportForUrl( + events: readonly LazyDownloadEvent[], + url: string, + label: string, +): void { + const event = events.find((candidate) => candidate.url === url); + if (event !== undefined) { + throw new Error( + `${label} unexpectedly fetched the Homebrew source tree from ${url}`, + ); + } +} + +function withoutTransportUrl( + events: readonly LazyDownloadEvent[], + url: string, +): LazyDownloadEvent[] { + return events.filter((event) => event.url !== url); +} + +function assertHomebrewBootstrapTransport( + events: readonly LazyDownloadEvent[], + tree: DerivedPackageDeferredZipTree, + url: string, + state: "deferred" | "materialized", +): void { + const matching = events.filter((event) => event.url === url); + if (state === "materialized") { + if (matching.length !== 0) { + throw new Error( + "eager Homebrew source tree unexpectedly used lazy transport", + ); + } + return; + } + if (matching.length === 0) { + throw new Error( + "first brew use did not fetch the deferred Homebrew source tree", + ); + } + const ids = new Set(matching.map((event) => event.id)); + const started = matching.filter((event) => event.status === "started"); + const completed = matching.filter((event) => event.status === "complete"); + const errors = matching.filter((event) => event.status === "error"); + const expectedBytes = tree.descriptor.archive.bytes; + if ( + ids.size !== 1 || + started.length !== 1 || + completed.length !== 1 || + errors.length !== 0 || + matching[0]!.status !== "started" || + matching.at(-1)!.status !== "complete" || + started[0]!.loadedBytes !== 0 || + completed[0]!.loadedBytes !== expectedBytes || + matching.some( + (event) => + event.kind !== "tree" || + event.mountPrefix !== tree.descriptor.mount_prefix || + event.totalBytes !== expectedBytes || + event.loadedBytes < 0 || + event.loadedBytes > expectedBytes, + ) + ) { + throw new Error( + "first brew use did not retrieve the complete Homebrew source package exactly once", + ); + } + let previousLoaded = -1; + for (const event of matching) { + if (event.loadedBytes < previousLoaded) { + throw new Error("Homebrew source-tree download progress moved backwards"); + } + previousLoaded = event.loadedBytes; + } +} + function assertSingleBottleTransport( events: readonly LazyDownloadEvent[], tree: SerializedLazyArchiveEntry, @@ -454,17 +808,28 @@ function assertBottleTransportEvents( const fetchedPackages = packagesForUrls(evidence.urls, plan); const requiredPackages = [ ...BASE_EXPECTED_FETCHED_PACKAGES, - ...MAIN_SHELL_LANGUAGE_RUNTIME_INVOCATIONS.map(({ packageName }) => packageName), + ...BREW_EXPECTED_FETCHED_PACKAGES, + ...MAIN_SHELL_LANGUAGE_RUNTIME_INVOCATIONS.map( + ({ packageName }) => packageName, + ), ]; - const missing = requiredPackages.filter((name) => !fetchedPackages.includes(name)); + const missing = requiredPackages.filter( + (name) => !fetchedPackages.includes(name), + ); if (missing.length !== 0) { throw new Error( `main-shell smoke did not fetch required bottles ${JSON.stringify(missing)}; ` + `fetched ${JSON.stringify(fetchedPackages)}`, ); } - const allowedPackages = new Set(BASE_EXPECTED_FETCHED_PACKAGES); - for (const { packageName, dependencyPackages } of MAIN_SHELL_LANGUAGE_RUNTIME_INVOCATIONS) { + const allowedPackages = new Set([ + ...BASE_EXPECTED_FETCHED_PACKAGES, + ...BREW_EXPECTED_FETCHED_PACKAGES, + ]); + for (const { + packageName, + dependencyPackages, + } of MAIN_SHELL_LANGUAGE_RUNTIME_INVOCATIONS) { for (const dependency of reviewedPackageClosure( guestManifest, packageName, @@ -473,7 +838,9 @@ function assertBottleTransportEvents( allowedPackages.add(dependency); } } - const unexpected = fetchedPackages.filter((name) => !allowedPackages.has(name)); + const unexpected = fetchedPackages.filter( + (name) => !allowedPackages.has(name), + ); if (unexpected.length !== 0) { throw new Error( `main-shell smoke fetched bottles outside its reviewed runtime closures: ` + @@ -483,11 +850,17 @@ function assertBottleTransportEvents( const fetchedUrls = new Set(evidence.urls); const remaining = plan.assets.filter((asset) => !fetchedUrls.has(asset.url)); if (remaining.length === 0) { - throw new Error("main-shell smoke unexpectedly materialized every deferred bottle"); + throw new Error( + "main-shell smoke unexpectedly materialized every deferred bottle", + ); } - const initiallyPendingUrls = new Set(pendingTrees.map((tree) => tree.content!.transports[0]!)); + const initiallyPendingUrls = new Set( + pendingTrees.map((tree) => tree.content!.transports[0]!), + ); if (remaining.some((asset) => !initiallyPendingUrls.has(asset.url))) { - throw new Error("main-shell smoke remaining bottle set differs from the initial pending set"); + throw new Error( + "main-shell smoke remaining bottle set differs from the initial pending set", + ); } return { bottles: evidence.bottles, bytes: evidence.bytes }; } @@ -513,13 +886,18 @@ function assertFetchedPackageSet( function assertLanguageBottleIsolation( packageName: string, declaredDependencies: readonly string[], + launcherPackages: readonly string[], events: readonly LazyDownloadEvent[], pendingTrees: readonly SerializedLazyArchiveEntry[], plan: HomebrewBottleMirrorPlan, guestManifest: unknown, label: string, ): void { - const evidence = assertCompleteBottleTransport(events, pendingTrees, `${label} first use`); + const evidence = assertCompleteBottleTransport( + events, + pendingTrees, + `${label} first use`, + ); const fetchedPackages = packagesForUrls(evidence.urls, plan); if (!fetchedPackages.includes(packageName)) { throw new Error( @@ -527,11 +905,10 @@ function assertLanguageBottleIsolation( `fetched ${JSON.stringify(fetchedPackages)}`, ); } - const allowed = reviewedPackageClosure( - guestManifest, - packageName, - declaredDependencies, - ); + const allowed = reviewedPackageClosure(guestManifest, packageName, [ + ...declaredDependencies, + ...launcherPackages, + ]); const outsideClosure = fetchedPackages.filter((name) => !allowed.has(name)); if (outsideClosure.length !== 0) { throw new Error( @@ -553,14 +930,20 @@ function packagesForUrls( urls: readonly string[], plan: HomebrewBottleMirrorPlan, ): string[] { - const packageByUrl = new Map(plan.assets.map((asset) => [asset.url, asset.package])); - return urls.map((url) => { - const packageName = packageByUrl.get(url); - if (packageName === undefined) { - throw new Error(`completed bottle URL is absent from the mirror plan: ${url}`); - } - return packageName; - }).sort(); + const packageByUrl = new Map( + plan.assets.map((asset) => [asset.url, asset.package]), + ); + return urls + .map((url) => { + const packageName = packageByUrl.get(url); + if (packageName === undefined) { + throw new Error( + `completed bottle URL is absent from the mirror plan: ${url}`, + ); + } + return packageName; + }) + .sort(); } function reviewedPackageClosure( @@ -594,7 +977,10 @@ function reviewedPackageClosure( return reviewed; } -function mainShellCounts(migrationLock: unknown): { roots: number; formulae: number } { +function mainShellCounts(migrationLock: unknown): { + roots: number; + formulae: number; +} { const lock = asRecord(migrationLock, "migration lock"); if (!Array.isArray(lock.packages) || !Array.isArray(lock.formula_closure)) { throw new Error("migration lock package counts are unavailable"); @@ -614,7 +1000,9 @@ function assertCompleteBottleTransport( for (const tree of pendingTrees) { const content = tree.content; if (content === undefined || content.transports.length === 0) { - throw new Error(`pending tree ${tree.mountPrefix} has no bottle transport`); + throw new Error( + `pending tree ${tree.mountPrefix} has no bottle transport`, + ); } const primaryUrl = content.transports[0]!; if (treeByPrimaryUrl.has(primaryUrl)) { @@ -664,14 +1052,20 @@ function assertCompleteBottleTransport( const completed = grouped.filter((event) => event.status === "complete"); const errors = grouped.filter((event) => event.status === "error"); if ( - started.length !== 1 || completed.length !== 1 || errors.length !== 0 || - grouped[0]!.status !== "started" || grouped.at(-1)!.status !== "complete" + started.length !== 1 || + completed.length !== 1 || + errors.length !== 0 || + grouped[0]!.status !== "started" || + grouped.at(-1)!.status !== "complete" ) { throw new Error( `${label} transport ${id} must have one start, one completion, and no fallback error`, ); } - if (started[0]!.loadedBytes !== 0 || completed[0]!.loadedBytes !== expectedBytes) { + if ( + started[0]!.loadedBytes !== 0 || + completed[0]!.loadedBytes !== expectedBytes + ) { throw new Error( `${label} transport ${id} did not retrieve the complete original bottle ` + `(${completed[0]!.loadedBytes}/${expectedBytes} bytes)`, @@ -680,7 +1074,9 @@ function assertCompleteBottleTransport( let previousLoaded = -1; for (const event of grouped) { if (event.loadedBytes < previousLoaded) { - throw new Error(`${label} transport ${id} byte progress moved backwards`); + throw new Error( + `${label} transport ${id} byte progress moved backwards`, + ); } previousLoaded = event.loadedBytes; } @@ -703,7 +1099,10 @@ function assertRetainedPosixShellAlias( guestManifest: unknown, ): { executablePath: string } { const lock = asRecord(migrationLock, "migration lock"); - const compatibility = asRecord(lock.compatibility, "migration lock compatibility"); + const compatibility = asRecord( + lock.compatibility, + "migration lock compatibility", + ); if (!Array.isArray(compatibility.aliases)) { throw new Error("migration lock compatibility aliases are missing"); } @@ -712,15 +1111,20 @@ function assertRetainedPosixShellAlias( return entry.targets.includes("/bin/sh"); }); if (matches.length !== 1) { - throw new Error(`migration lock declares ${matches.length} /bin/sh aliases, expected one`); + throw new Error( + `migration lock declares ${matches.length} /bin/sh aliases, expected one`, + ); } const alias = matches[0]! as Record; const packageName = "kandelo-dev/tap-core/dash"; if ( - alias.package !== packageName || alias.source_kind !== "link" || + alias.package !== packageName || + alias.source_kind !== "link" || alias.source !== "bin/dash" ) { - throw new Error("migration lock /bin/sh alias is not the reviewed Dash link"); + throw new Error( + "migration lock /bin/sh alias is not the reviewed Dash link", + ); } const guest = asRecord(guestManifest, "guest Homebrew manifest"); @@ -731,7 +1135,9 @@ function assertRetainedPosixShellAlias( (entry) => isRecord(entry) && entry.full_name === packageName, ); if (packages.length !== 1) { - throw new Error(`guest Homebrew manifest has ${packages.length} Dash packages, expected one`); + throw new Error( + `guest Homebrew manifest has ${packages.length} Dash packages, expected one`, + ); } const prefix = packages[0]!.prefix; if (typeof prefix !== "string" || !prefix.startsWith("/")) { @@ -747,7 +1153,9 @@ function assertRetainedPosixShellAlias( } const executablePath = resolveVfsSymlinkPath(fs, "/bin/sh"); if (!executablePath.startsWith(`${prefix}/`)) { - throw new Error(`resolved /bin/sh executable escapes the Dash prefix: ${executablePath}`); + throw new Error( + `resolved /bin/sh executable escapes the Dash prefix: ${executablePath}`, + ); } return { executablePath }; } @@ -808,6 +1216,10 @@ async function spawnWithTimeout( function parseArgs(args: string[]): { imagePath: string; migrationLockPath: string; + homebrewBootstrapSpecPath: string; + homebrewBootstrapArchivePath: string; + homebrewBootstrapEnvPath: string; + homebrewBootstrapState: "deferred" | "materialized"; demoConfigPath: string; transportMode: "closed" | "public"; bottleMirrorPlanPath?: string; @@ -816,6 +1228,10 @@ function parseArgs(args: string[]): { const allowed = new Set([ "--image", "--migration-lock", + "--homebrew-bootstrap-spec", + "--homebrew-bootstrap-archive", + "--homebrew-bootstrap-env", + "--homebrew-bootstrap-state", "--demo-config", "--transport-mode", "--bottle-mirror-plan", @@ -824,7 +1240,9 @@ function parseArgs(args: string[]): { const option = args[index]; const value = args[index + 1]; if ( - option === undefined || value === undefined || !allowed.has(option) || + option === undefined || + value === undefined || + !allowed.has(option) || values.has(option) ) { return smokeUsage(); @@ -833,11 +1251,22 @@ function parseArgs(args: string[]): { } const image = values.get("--image"); const migrationLock = values.get("--migration-lock"); + const homebrewBootstrapSpec = values.get("--homebrew-bootstrap-spec"); + const homebrewBootstrapArchive = values.get("--homebrew-bootstrap-archive"); + const homebrewBootstrapEnv = values.get("--homebrew-bootstrap-env"); + const homebrewBootstrapState = values.get("--homebrew-bootstrap-state"); const demoConfig = values.get("--demo-config"); const mode = values.get("--transport-mode"); const plan = values.get("--bottle-mirror-plan"); if ( - !image || !migrationLock || !demoConfig || + !image || + !migrationLock || + !homebrewBootstrapSpec || + !homebrewBootstrapArchive || + !homebrewBootstrapEnv || + !demoConfig || + (homebrewBootstrapState !== "deferred" && + homebrewBootstrapState !== "materialized") || (mode !== "closed" && mode !== "public") || (mode === "closed" && !plan) || (mode === "public" && plan !== undefined) @@ -847,6 +1276,10 @@ function parseArgs(args: string[]): { return { imagePath: resolve(image), migrationLockPath: resolve(migrationLock), + homebrewBootstrapSpecPath: resolve(homebrewBootstrapSpec), + homebrewBootstrapArchivePath: resolve(homebrewBootstrapArchive), + homebrewBootstrapEnvPath: resolve(homebrewBootstrapEnv), + homebrewBootstrapState, demoConfigPath: resolve(demoConfig), transportMode: mode, ...(plan === undefined ? {} : { bottleMirrorPlanPath: resolve(plan) }), @@ -854,13 +1287,17 @@ function parseArgs(args: string[]): { } function smokeUsage(): never { - throw new Error( - "usage: npx tsx scripts/homebrew-main-shell-node-smoke.ts " + - "--image --migration-lock " + - "--demo-config --transport-mode " + - "[--bottle-mirror-plan ] " + - "(the plan is required only in closed mode)", - ); + throw new Error( + "usage: npx tsx scripts/homebrew-main-shell-node-smoke.ts " + + "--image --migration-lock " + + "--homebrew-bootstrap-spec " + + "--homebrew-bootstrap-archive " + + "--homebrew-bootstrap-env " + + "--homebrew-bootstrap-state " + + "--demo-config --transport-mode " + + "[--bottle-mirror-plan ] " + + "(the plan is required only in closed mode)", + ); } function readRuntimeState( @@ -875,7 +1312,9 @@ function readRuntimeState( contents?: Uint8Array; }> { const lock = migrationLock as { - compatibility?: { runtime_state?: Array<{ path?: unknown; kind?: unknown }> }; + compatibility?: { + runtime_state?: Array<{ path?: unknown; kind?: unknown }>; + }; }; const declarations = lock.compatibility?.runtime_state; if (!Array.isArray(declarations)) { @@ -891,11 +1330,14 @@ function readRuntimeState( throw new Error(`migration lock runtime_state[${index}] is invalid`); } const stat = fs.lstat(declaration.path); - const actualKind = (stat.mode & 0xf000) === 0x4000 - ? "directory" - : (stat.mode & 0xf000) === 0x8000 - ? declaration.kind === "text_file" ? "text_file" : "empty_file" - : "unsupported"; + const actualKind = + (stat.mode & 0xf000) === 0x4000 + ? "directory" + : (stat.mode & 0xf000) === 0x8000 + ? declaration.kind === "text_file" + ? "text_file" + : "empty_file" + : "unsupported"; if (actualKind === "unsupported") { throw new Error(`${declaration.path} is not a regular file or directory`); } @@ -905,9 +1347,11 @@ function readRuntimeState( mode: stat.mode & 0o7777, uid: stat.uid, gid: stat.gid, - ...(actualKind === "directory" ? {} : { - contents: readVfsFile(fs, declaration.path, stat.size), - }), + ...(actualKind === "directory" + ? {} + : { + contents: readVfsFile(fs, declaration.path, stat.size), + }), }; }); } @@ -932,7 +1376,32 @@ function asRecord(value: unknown, label: string): Record { return value; } -function readVfsFile(fs: MemoryFileSystem, path: string, knownSize?: number): Uint8Array { +function canonicalJson(value: unknown): string { + const normalize = (candidate: unknown): unknown => { + if (Array.isArray(candidate)) return candidate.map(normalize); + if (!isRecord(candidate)) return candidate; + return Object.fromEntries( + Object.keys(candidate) + .sort() + .map((key) => [key, normalize(candidate[key])]), + ); + }; + return JSON.stringify(normalize(value)); +} + +function readRegularFile(path: string, label: string): Uint8Array { + const stat = lstatSync(path); + if (!stat.isFile() || stat.isSymbolicLink() || stat.size <= 0) { + throw new Error(`${label} is not a nonempty regular file: ${path}`); + } + return new Uint8Array(readFileSync(path)); +} + +function readVfsFile( + fs: MemoryFileSystem, + path: string, + knownSize?: number, +): Uint8Array { const stat = knownSize === undefined ? fs.stat(path) : undefined; const size = knownSize ?? stat!.size; if (stat !== undefined && (stat.mode & 0xf000) !== 0x8000) { diff --git a/scripts/test-homebrew-main-shell-closure.sh b/scripts/test-homebrew-main-shell-closure.sh index 5ab279da61..983120e68e 100755 --- a/scripts/test-homebrew-main-shell-closure.sh +++ b/scripts/test-homebrew-main-shell-closure.sh @@ -21,6 +21,10 @@ PREPARE_MERGE_WORKFLOW="$REPO_ROOT/.github/workflows/prepare-merge.yml" FORCE_REBUILD_WORKFLOW="$REPO_ROOT/.github/workflows/force-rebuild.yml" SHELL_BUILD_TOML="$REPO_ROOT/packages/registry/shell/build.toml" SHELL_BUILDER="$REPO_ROOT/packages/registry/shell/build-shell.sh" +SHELL_PACKAGE_TOML="$REPO_ROOT/packages/registry/shell/package.toml" +HOMEBREW_BOOTSTRAP_PACKAGE_TOML="$REPO_ROOT/packages/registry/homebrew-bootstrap/package.toml" +PACKAGE_TREE_SPEC="$REPO_ROOT/homebrew/main-shell-brew-package-tree.json" +LAZY_ARCHIVE_RESOLVER="$REPO_ROOT/apps/browser-demos/lib/init/lazy-archives.ts" RUN_SH="$REPO_ROOT/run.sh" TMP_ROOT="$(mktemp -d)" trap 'rm -rf "$TMP_ROOT"' EXIT @@ -171,7 +175,9 @@ grep -Fq -- '--artifact "$OUT"' "$BUILDER" || for variable in \ KANDELO_HOMEBREW_MAIN_SHELL_STRICT \ - KANDELO_HOMEBREW_MAIN_SHELL_SHA256 + KANDELO_HOMEBREW_MAIN_SHELL_SHA256 \ + KANDELO_HOMEBREW_MAIN_SHELL_BOOTSTRAP_SHA256 \ + KANDELO_HOMEBREW_MAIN_SHELL_BOOTSTRAP_BYTES do grep -Fq -- "\"$variable=\$$variable\"" "$WORKFLOW" || fail "main-shell workflow must pass $variable explicitly to its isolated consumer" @@ -221,6 +227,15 @@ grep -Fq -- '--lazy-shell \' "$WORKFLOW" || fail "candidate proof must explicitly opt into lazy shell composition" grep -Fq 'scripts/build-homebrew-main-shell-closure.sh \' "$WORKFLOW" || fail "candidate proof must invoke the strict shell composer" +[ "$(grep -Fc -- '--materialize-package-tree \' "$WORKFLOW")" -eq 1 ] || + fail "candidate proof must build exactly one source-materialized derivative" +[ "$(grep -Fc -- '--package-tree-spec homebrew/main-shell-brew-package-tree.json' \ + "$WORKFLOW")" -eq 2 ] || + fail "lazy and eager candidate builds must use the same package-tree recipe" +[ "$(grep -Fc -- '--package-tree-archive "$bootstrap"' "$WORKFLOW")" -eq 2 ] || + fail "lazy and eager candidate builds must use the same package output bytes" +grep -Fq 'del(.state)' "$WORKFLOW" || + fail "candidate proof must compare lazy and eager package-tree identity" candidate_install_workflow_block="$(sed -n \ "/- name: Install the candidate's exact shell bytes/,/- name: Recover the exact bottle mirror/p" \ "$WORKFLOW")" @@ -261,6 +276,14 @@ grep -Fq -- '--image "${{ steps.candidate.outputs.image }}"' "$WORKFLOW" || fail "Node proof must boot the exact candidate bytes directly" grep -Fq -- '--migration-lock homebrew/main-shell-migration-lock.json' "$WORKFLOW" || fail "post-archive Node proof must validate against the reviewed migration lock" +grep -Fq -- '--homebrew-bootstrap-spec homebrew/main-shell-brew-package-tree.json' \ + "$WORKFLOW" || + fail "Node proof must derive the exact Homebrew package tree" +grep -Fq -- '--homebrew-bootstrap-archive "${{ steps.candidate.outputs.bootstrap }}"' \ + "$WORKFLOW" || + fail "Node proof must bind the exact standalone Homebrew package bytes" +grep -Fq -- '--homebrew-bootstrap-state "$state"' "$WORKFLOW" || + fail "Node proof must assert lazy versus eager source state" grep -Fq -- '--demo-config homebrew/main-shell-demo.json' "$WORKFLOW" || fail "post-archive Node proof must validate the canonical demo config bytes" node_smoke_workflow_block="$(sed -n \ @@ -268,6 +291,12 @@ node_smoke_workflow_block="$(sed -n \ "$WORKFLOW")" grep -Fq 'node_smoke_args=(' <<<"$node_smoke_workflow_block" || fail "Node proof must build one explicit transport-aware argument vector" +grep -Fq 'run_node_smoke "${{ steps.candidate.outputs.image }}" deferred' \ + <<<"$node_smoke_workflow_block" || + fail "Node proof must boot the deferred shell candidate" +grep -Fq 'run_node_smoke "${{ steps.candidate.outputs.eager_image }}" materialized' \ + <<<"$node_smoke_workflow_block" || + fail "Node proof must boot the source-materialized derivative" grep -Fq 'case "$TRANSPORT_MODE" in' <<<"$node_smoke_workflow_block" || fail "Node proof must branch explicitly on closed versus public transport" grep -Fq '"${node_smoke_args[@]}"' <<<"$node_smoke_workflow_block" || @@ -289,13 +318,74 @@ grep -Fq '${{ steps.candidate.outputs.image }}' "$WORKFLOW" || fail "main-shell evidence must retain the exact candidate image" grep -Fq '${{ steps.candidate.outputs.report }}' "$WORKFLOW" || fail "main-shell evidence must retain the candidate composition report" +for evidence in \ + '${{ steps.candidate.outputs.bootstrap }}' \ + '${{ steps.candidate.outputs.eager_image }}' \ + '${{ steps.candidate.outputs.eager_report }}' +do + grep -Fq "$evidence" "$WORKFLOW" || + fail "main-shell evidence must retain $evidence" +done grep -Fq 'apps/browser-demos/test-results' "$WORKFLOW" || fail "main-shell evidence must retain browser failure traces" grep -Fq '${{ runner.temp }}/homebrew-main-shell-modeset-playwright.json' \ "$WORKFLOW" || fail "main-shell evidence must retain the isolated MODESET report" -[ "$(grep -Fc 'bash ../../scripts/dev-shell.sh env \' "$WORKFLOW")" -eq 2 ] || - fail "shell and MODESET proofs must run in separate isolated browser processes" +# WHY: process isolation is a contract of each heavyweight browser proof, not +# an incidental total invocation count. Name every standalone command so adding +# another legitimate proof cannot silently relabel which contracts are isolated. +browser_invocation_for() { + local test_path="$1" + awk -v test_path="$test_path" ' + index($0, "bash ../../scripts/dev-shell.sh env \\") { + invocation = $0 ORS + active = 1 + matched = 0 + next + } + active { + invocation = invocation $0 ORS + if (index($0, "npx playwright test " test_path " \\")) { + matched = 1 + } + if (matched && $0 !~ /\\[[:space:]]*$/) { + printf "%s", invocation + exit + } + if (!matched && $0 !~ /\\[[:space:]]*$/) { + active = 0 + } + } + ' "$WORKFLOW" +} +guest_lifecycle_browser_invocation="$( + browser_invocation_for "test/homebrew-guest-lifecycle.spec.ts" +)" +grep -Fq 'bash ../../scripts/dev-shell.sh env \' \ + <<<"$guest_lifecycle_browser_invocation" && + grep -Fq -- '--grep "rejects a guest lifecycle fixture"' \ + <<<"$guest_lifecycle_browser_invocation" || + fail "offline guest-lifecycle rejection must run in its own browser process" +shell_browser_invocation="$( + browser_invocation_for "test/kandelo-homebrew-main-shell.spec.ts" +)" +grep -Fq 'bash ../../scripts/dev-shell.sh env \' \ + <<<"$shell_browser_invocation" && + grep -Fq '"PLAYWRIGHT_JSON_OUTPUT_FILE=$shell_report" \' \ + <<<"$shell_browser_invocation" && + grep -Fq -- '--project=chromium --reporter=json' \ + <<<"$shell_browser_invocation" || + fail "shell acceptance must run in its own reporting browser process" +modeset_browser_invocation="$( + browser_invocation_for "test/kandelo-modeset.spec.ts" +)" +grep -Fq 'bash ../../scripts/dev-shell.sh env \' \ + <<<"$modeset_browser_invocation" && + grep -Fq '"PLAYWRIGHT_JSON_OUTPUT_FILE=$modeset_report" \' \ + <<<"$modeset_browser_invocation" && + grep -Fq -- '--project=chromium --reporter=json' \ + <<<"$modeset_browser_invocation" || + fail "MODESET acceptance must run in its own reporting browser process" grep -Fq '"PLAYWRIGHT_JSON_OUTPUT_FILE=$shell_report"' "$WORKFLOW" || fail "shell acceptance must have Playwright write JSON directly to its report file" grep -Fq '"PLAYWRIGHT_JSON_OUTPUT_FILE=$modeset_report"' "$WORKFLOW" || @@ -304,8 +394,14 @@ grep -Fq 'npx playwright test test/kandelo-homebrew-main-shell.spec.ts \' "$WORK fail "browser acceptance must run the exact Homebrew shell proof" grep -Fq 'npx playwright test test/kandelo-modeset.spec.ts \' "$WORKFLOW" || fail "browser acceptance must preserve MODESET in a fresh process" -grep -Fq 'for report in "$shell_report" "$modeset_report"; do' "$WORKFLOW" || - fail "browser acceptance must validate both isolated Playwright reports" +[ "$(grep -Fc '.stats.expected == 2 and .stats.unexpected == 0 and' "$WORKFLOW")" -eq 1 ] || + fail "shell acceptance must require both pristine-machine browser proofs" +grep -Fq "' \"\$shell_report\" >/dev/null" "$WORKFLOW" || + fail "shell acceptance must validate its exact two-test report" +[ "$(grep -Fc '.stats.expected == 1 and .stats.unexpected == 0 and' "$WORKFLOW")" -eq 1 ] || + fail "MODESET acceptance must remain one browser proof" +grep -Fq "' \"\$modeset_report\" >/dev/null" "$WORKFLOW" || + fail "MODESET acceptance must validate its isolated report" grep -Fq 'page.goto("/?demo=modeset"' "$BROWSER_SMOKE" && fail "Homebrew shell acceptance must not start a second VFS in its browser process" grep -Fq 'gotoOrSkip(page, "/?demo=modeset")' "$MODESET_SMOKE" || @@ -393,8 +489,8 @@ grep -Fq 'repository = "https://github.com/Kandelo-dev/homebrew-tap-core.git"' \ locked_tap_sha="$(jq -er '.catalog.tap_commit' "$SOURCE_LOCK")" grep -Fq "commit = \"$locked_tap_sha\"" "$SHELL_BUILD_TOML" || fail "shell Git input commit must equal the reviewed migration lock" -grep -Eq '^revision[[:space:]]*=[[:space:]]*18$' "$SHELL_BUILD_TOML" || - fail "language-expanded lazy shell must publish canonical shell revision 18" +grep -Eq '^revision[[:space:]]*=[[:space:]]*19$' "$SHELL_BUILD_TOML" || + fail "brew-enabled lazy shell must publish canonical shell revision 19" for shell_input in \ homebrew/main-shell-demo.json \ web-libs/kandelo-session/src/demo-config.ts @@ -430,7 +526,8 @@ grep -Fq 'from "../../../host/src/homebrew-vfs-composer"' \ fail "materialized image entrypoint must own the candidate composer import" for generic_input in \ WASM_POSIX_BUILD_GIT_HOMEBREW_TAP_CORE_DIR \ - WASM_POSIX_BUILD_GIT_HOMEBREW_TAP_CORE_COMMIT + WASM_POSIX_BUILD_GIT_HOMEBREW_TAP_CORE_COMMIT \ + WASM_POSIX_DEP_HOMEBREW_BOOTSTRAP_DIR do grep -Fq "$generic_input" "$SHELL_BUILDER" || fail "shell builder must consume generic resolver input $generic_input" @@ -441,7 +538,13 @@ grep -Fq 'KANDELO_HOMEBREW_MAIN_SHELL_TAP_' "$SHELL_BUILDER" && fail "canonical package wrapper must activate lazy composition exactly once" grep -Fq 'build-shell-vfs-image.sh' "$SHELL_BUILDER" && fail "shell builder must not retain the legacy registry-composition fallback" -for isolated_flag in '--work-dir "$WORK_DIR"' '--report "$REPORT"' '--bottle-cache "$BOTTLE_CACHE"'; do +for isolated_flag in \ + '--work-dir "$WORK_DIR"' \ + '--report "$REPORT"' \ + '--bottle-cache "$BOTTLE_CACHE"' \ + '--package-tree-spec "$REPO_ROOT/homebrew/main-shell-brew-package-tree.json"' \ + '--package-tree-archive "$HOMEBREW_BOOTSTRAP"' +do grep -Fq -- "$isolated_flag" "$SHELL_BUILDER" || fail "shell builder must pass isolated composer option $isolated_flag" done @@ -451,8 +554,50 @@ grep -Fq 'homebrew-main-shell-node-smoke.ts' "$BUILDER" && fail "cached shell composition must not consume ambient runtime acceptance artifacts" grep -Fq 'scripts/homebrew-main-shell-node-smoke.ts' "$WORKFLOW" || fail "exact candidate shell bytes must retain post-build Node acceptance" -grep -Eq '^depends_on = \[\]$' "$REPO_ROOT/packages/registry/shell/package.toml" || - fail "canonical bottle-only shell package must not pre-resolve the legacy registry graph" +jq -e ' + (keys | sort) == [ + "activation", "archive", "content_role", "id", "kind", + "mount_prefix", "owner", "package", "schema" + ] and + .schema == 1 and + .kind == "kandelo-package-deferred-zip-tree" and + .id == "homebrew-bootstrap/source-tree" and + .content_role == "source-tree" and + .package == { + name: "homebrew-bootstrap", + output: "homebrew-bootstrap.zip" + } and + .archive == { + url: "homebrew-bootstrap.zip", + mode_policy: "portable-posix-v1" + } and + .mount_prefix == "/home/linuxbrew/.linuxbrew" and + .owner == { uid: 1000, gid: 1000 } and + .activation == { + mode: "first-use", + capabilities: ["homebrew:bootstrap"], + roots: ["/home/linuxbrew/.linuxbrew/bin/brew"] + } +' "$PACKAGE_TREE_SPEC" >/dev/null || + fail "Homebrew package-tree spec is not the exact reviewed contract" +grep -Fq 'depends_on = ["homebrew-bootstrap@6.0.3-4-g4ead861"]' \ + "$SHELL_PACKAGE_TOML" || + fail "shell package must depend on the exact standalone Homebrew source package" +[ "$(grep -Fc '[[outputs]]' "$SHELL_PACKAGE_TOML")" -eq 1 ] || + fail "shell package must publish only its VFS image" +grep -Fq 'name = "homebrew-bootstrap"' "$HOMEBREW_BOOTSTRAP_PACKAGE_TOML" || + fail "standalone Homebrew source package is missing" +grep -Fq 'wasm = "homebrew-bootstrap.zip"' "$HOMEBREW_BOOTSTRAP_PACKAGE_TOML" || + fail "standalone Homebrew source package omits its exact ZIP output" +grep -Fq '"homebrew/main-shell-brew-package-tree.json"' "$SHELL_BUILD_TOML" || + fail "shell build identity omits the package-tree recipe" +grep -Fq \ + 'import homebrewBootstrapZipUrl from "@binaries/programs/homebrew-bootstrap/homebrew-bootstrap.zip?url";' \ + "$LAZY_ARCHIVE_RESOLVER" || + fail "browser shell does not resolve the standalone Homebrew package output" +grep -Fq '"homebrew-bootstrap.zip": homebrewBootstrapZipUrl' \ + "$LAZY_ARCHIVE_RESOLVER" || + fail "browser shell does not bind the descriptor-relative Homebrew asset" shell_build_function="$TMP_ROOT/build-shell-vfs-function.sh" sed -n '/^build_shell_vfs()/,/^}/p' "$RUN_SH" >"$shell_build_function" @@ -494,6 +639,14 @@ grep -Fq -- '--binaries-dir "$REPO_ROOT/local-binaries"' "$RUN_SH" || fail "run.sh must materialize the resolved shell package for local consumers" grep -Fq 'pkg_has_output shell shell.vfs.zst' "$RUN_SH" || fail "run.sh must validate the shell package's declared output" +has_shell_vfs_function="$TMP_ROOT/has-shell-vfs-function.sh" +sed -n '/^has_shell_vfs()/,/^}/p' "$RUN_SH" >"$has_shell_vfs_function" +grep -Fq 'pkg_has_output homebrew-bootstrap homebrew-bootstrap.zip' \ + "$has_shell_vfs_function" || + fail "shell availability must include its lazily served Homebrew package" +grep -Fq "Package resolver did not materialize shell's Homebrew source dependency" \ + "$shell_build_function" || + fail "shell resolution must verify its Homebrew package dependency" grep -Fq 'packages/registry/shell/build-shell.sh' "$RUN_SH" && fail "run.sh must not bypass the resolver by invoking the shell recipe directly" grep -Fq 'build_fbdoom' "$shell_build_function" && @@ -553,32 +706,49 @@ done echo "canonical shell wrapper did not pin SOURCE_DATE_EPOCH=0" >&2 exit 79 } -work="" report="" cache="" out="" lazy_shell=false +work="" report="" cache="" out="" spec="" archive="" bootstrap_env="" lazy_shell=false while [ "$#" -gt 0 ]; do case "$1" in --lazy-shell) lazy_shell=true; shift ;; --work-dir) work="$2"; shift 2 ;; --report) report="$2"; shift 2 ;; --bottle-cache) cache="$2"; shift 2 ;; + --package-tree-spec) spec="$2"; shift 2 ;; + --package-tree-archive) archive="$2"; shift 2 ;; + --homebrew-bootstrap-env) bootstrap_env="$2"; shift 2 ;; --out) out="$2"; shift 2 ;; --tap-root|--expected-tap-sha) shift 2 ;; *) echo "unexpected fake-composer option: $1" >&2; exit 81 ;; esac done -[ -n "$work" ] && [ -n "$report" ] && [ -n "$cache" ] && [ -n "$out" ] +[ -n "$work" ] && [ -n "$report" ] && [ -n "$cache" ] && [ -n "$out" ] && + [ "$spec" = "$PACKAGE_TREE_SPEC" ] && + [ "$archive" = "$WASM_POSIX_DEP_HOMEBREW_BOOTSTRAP_DIR/homebrew-bootstrap.zip" ] && + [ "$bootstrap_env" = "$WASM_POSIX_DEP_HOMEBREW_BOOTSTRAP_DIR/homebrew-brew.env" ] [ "$lazy_shell" = true ] [ ! -e "$work" ] && [ ! -L "$work" ] mkdir "$work" mkdir "$cache" printf '%s\n' "$WASM_POSIX_DEP_OUT_DIR" >"$out" printf '{}\n' >"$report" -printf '%s|%s|%s|%s|%s\n' \ - "$WASM_POSIX_DEP_OUT_DIR" "$work" "$report" "$cache" "$out" \ +printf '%s|%s|%s|%s|%s|%s|%s\n' \ + "$WASM_POSIX_DEP_OUT_DIR" "$work" "$report" "$cache" "$out" "$archive" \ + "$bootstrap_env" \ >>"$FAKE_COMPOSER_LOG" FAKE_COMPOSER chmod 0755 "$apply_fake_composer" tap_sha=1111111111111111111111111111111111111111 +bootstrap_dir="$TMP_ROOT/homebrew-bootstrap-dependency" +mkdir "$bootstrap_dir" +printf '%s\n' 'exact standalone Homebrew package bytes' > \ + "$bootstrap_dir/homebrew-bootstrap.zip" +printf '%s\n' \ + 'HOMEBREW_NO_ANALYTICS=1' \ + 'HOMEBREW_NO_AUTO_UPDATE=1' \ + 'HOMEBREW_SYSTEM_ENV_TAKES_PRIORITY=1' \ + 'HOMEBREW_KANDELO_BOTTLE_TAG=wasm32_kandelo' \ + >"$bootstrap_dir/homebrew-brew.env" parallel_one="$TMP_ROOT/parallel-shell-one" parallel_two="$TMP_ROOT/parallel-shell-two" mkdir "$parallel_one" "$parallel_two" @@ -587,6 +757,7 @@ run_fake_shell_build() { env \ PATH="$fake_bin:$PATH" \ FAKE_COMPOSER_LOG="$fake_log" \ + PACKAGE_TREE_SPEC="$PACKAGE_TREE_SPEC" \ GH_TOKEN=forbidden \ GITHUB_TOKEN=forbidden \ HOMEBREW_GITHUB_API_TOKEN=forbidden \ @@ -596,6 +767,7 @@ run_fake_shell_build() { WASM_POSIX_DEP_TARGET_ARCH=wasm32 \ WASM_POSIX_BUILD_GIT_HOMEBREW_TAP_CORE_DIR="$TMP_ROOT/fake-tap" \ WASM_POSIX_BUILD_GIT_HOMEBREW_TAP_CORE_COMMIT="$tap_sha" \ + WASM_POSIX_DEP_HOMEBREW_BOOTSTRAP_DIR="$bootstrap_dir" \ /bin/bash "$SHELL_BUILDER" } run_fake_shell_build "$parallel_one" & @@ -626,6 +798,13 @@ expect_failure "requires build.toml git input homebrew_tap_core" \ WASM_POSIX_DEP_TARGET_ARCH=wasm32 \ bash "$SHELL_BUILDER" +expect_failure "requires its declared homebrew-bootstrap dependency" \ + env WASM_POSIX_DEP_OUT_DIR="$TMP_ROOT/missing-bootstrap-input" \ + WASM_POSIX_DEP_TARGET_ARCH=wasm32 \ + WASM_POSIX_BUILD_GIT_HOMEBREW_TAP_CORE_DIR="$TMP_ROOT/fake-tap" \ + WASM_POSIX_BUILD_GIT_HOMEBREW_TAP_CORE_COMMIT="$tap_sha" \ + bash "$SHELL_BUILDER" + tap="$TMP_ROOT/tap" mkdir -p "$tap/Kandelo" git -C "$tap" init -q @@ -647,6 +826,15 @@ expect_failure "must match locked catalog" \ --migration-lock "$lock" \ --expected-tap-sha 0000000000000000000000000000000000000000 +expect_failure "package-tree spec and archive must be provided together" \ + "$BUILDER" --tap-root "$tap" \ + --work-dir "$TMP_ROOT/work-package-tree-without-archive" \ + --migration-lock "$lock" --package-tree-spec "$PACKAGE_TREE_SPEC" +expect_failure "--materialize-package-tree requires a package tree" \ + "$BUILDER" --tap-root "$tap" \ + --work-dir "$TMP_ROOT/work-materialize-without-package-tree" \ + --migration-lock "$lock" --materialize-package-tree + printf '%s\n' "untracked" >"$tap/untracked-file" expect_failure "exact tap checkout is dirty" \ "$BUILDER" --tap-root "$tap" --work-dir "$TMP_ROOT/work-dirty-tap" \ diff --git a/tests/package-system/homebrew-bootstrap-package.test.ts b/tests/package-system/homebrew-bootstrap-package.test.ts index 225e45d5db..781af5fc64 100644 --- a/tests/package-system/homebrew-bootstrap-package.test.ts +++ b/tests/package-system/homebrew-bootstrap-package.test.ts @@ -49,7 +49,7 @@ afterEach(() => { }); describe("homebrew-bootstrap package contract", () => { - it("pins one exact portable recipe, output, license boundary, and sealed Git input", () => { + it("pins one exact portable recipe, output closure, license boundary, and sealed Git input", () => { const manifest = readFileSync(join(packageDir, "package.toml"), "utf8"); const build = readFileSync(join(packageDir, "build.toml"), "utf8"); const lock = loadHomebrewBootstrapSourceLock(lockPath); @@ -61,6 +61,7 @@ describe("homebrew-bootstrap package contract", () => { expect(manifest).toContain(`sha256 = "${lock.source.archive_sha256}"`); expect(manifest).toContain(`spdx = "${lock.license.expression}"`); expect(manifest).toContain('name = "homebrew-bootstrap"\nwasm = "homebrew-bootstrap.zip"'); + expect(manifest).toContain('name = "homebrew-brew"\nwasm = "homebrew-brew.env"'); expect(manifest).toContain('fork_instrumentation = "disabled"'); expect(build).toContain('name = "homebrew_brew"'); @@ -102,13 +103,22 @@ describe("homebrew-bootstrap package contract", () => { expect(projection.packages["homebrew-bootstrap"]).toMatchObject({ arches: ["wasm32"], dependencyClosures: { wasm32: [] }, - members: [{ - kind: "output", - sourceArtifact: "homebrew-bootstrap.zip", - mirrorPath: "homebrew-bootstrap.zip", - outputName: "homebrew-bootstrap", - forkInstrumentation: "disabled", - }], + members: [ + { + kind: "output", + sourceArtifact: "homebrew-bootstrap.zip", + mirrorPath: "homebrew-bootstrap/homebrew-bootstrap.zip", + outputName: "homebrew-bootstrap", + forkInstrumentation: "disabled", + }, + { + kind: "output", + sourceArtifact: "homebrew-brew.env", + mirrorPath: "homebrew-bootstrap/homebrew-brew.env", + outputName: "homebrew-brew", + forkInstrumentation: "disabled", + }, + ], }); });