[Packaging] Resolve every package from one verified build

.github/actions/detect-change-scope/ci-scope-paths.sh

@@ -58,8 +58,11 @@ package_publish_flow_changed_files() {
 binary_materialization_changed_files() {
   grep -E \
     -e '^tools/xtask/src/(index_toml|remote_fetch|util)\.rs$' \
-    -e '^scripts/(fetch-binaries|install-local-binary|materialize-pr-overlays|resolve-binary|test-wasm-artifact-guards|wasm-artifact-guards)\.sh$' \
+    -e '^scripts/(fetch-binaries|install-local-binary|materialize-pr-overlays|pack-ci-test-workspace|resolve-binary|test-wasm-artifact-guards|wasm-artifact-guards)\.sh$' \
+    -e '^scripts/(build-resolve-binary-bundle|test-resolve-binary-bundle)\.sh$' \
+    -e '^scripts/resolve-binary\.(ts|bundle\.mjs|bundle\.LICENSES\.txt)$' \
     -e '^scripts/vfs-has-stale-abi\.mjs$' \
+    -e '^host/src/binary-resolver\.ts$' \
     -e '^tests/package-system/' \
     || true
 }

.github/actions/detect-change-scope/test-ci-scope-paths.sh

@@ -117,6 +117,9 @@ assert_matches binary_materialization_changed_files \
 assert_matches binary_materialization_changed_files \
   "scripts/fetch-binaries.sh" \
   "scripts/fetch-binaries.sh"
+assert_matches binary_materialization_changed_files \
+  "scripts/pack-ci-test-workspace.sh" \
+  "scripts/pack-ci-test-workspace.sh"
 assert_matches binary_materialization_changed_files \
   "scripts/wasm-artifact-guards.sh" \
   "scripts/wasm-artifact-guards.sh"
@@ -126,6 +129,17 @@ assert_matches binary_materialization_changed_files \
 assert_matches binary_materialization_changed_files \
   "scripts/vfs-has-stale-abi.mjs" \
   "scripts/vfs-has-stale-abi.mjs"
+for resolver_input in \
+  host/src/binary-resolver.ts \
+  scripts/resolve-binary.ts \
+  scripts/resolve-binary.bundle.mjs \
+  scripts/resolve-binary.bundle.LICENSES.txt \
+  scripts/build-resolve-binary-bundle.sh \
+  scripts/test-resolve-binary-bundle.sh; do
+  assert_matches binary_materialization_changed_files \
+    "$resolver_input" \
+    "$resolver_input"
+done
 assert_matches binary_materialization_changed_files \
   "tests/package-system/fetch-binaries-allow-stale.test.ts" \
   "tests/package-system/fetch-binaries-allow-stale.test.ts"

.github/scripts/test-merge-candidate-workflows.sh

@@ -654,6 +654,28 @@ for step in "Compute matrix" "Materialize binaries"; do
   fi
 done
 
+for workflow in "$STAGING_WORKFLOW" "$PREPARE"; do
+  validation_job="$(job_block "$workflow" test-gate-validation)"
+  root_install_line="$(
+    grep -nF -- '- name: Install root npm deps' <<<"$validation_job" |
+      head -n 1 |
+      cut -d: -f1
+  )"
+  materialization_line="$(
+    grep -nF -- '- name: Test binary materialization flow' <<<"$validation_job" |
+      head -n 1 |
+      cut -d: -f1
+  )"
+  [ -n "$root_install_line" ] &&
+    [ -n "$materialization_line" ] &&
+    [ "$root_install_line" -lt "$materialization_line" ] ||
+    fail "$(basename "$workflow") must install root npm dependencies before materialization tests"
+  root_install_step="$(step_block "$workflow" "Install root npm deps")"
+  grep -Fq 'run: bash scripts/dev-shell.sh npm ci --no-audit --no-fund' \
+    <<<"$root_install_step" ||
+    fail "$(basename "$workflow") materialization validation must install the root esbuild dependency"
+done
+
 grep -Fq 'cleanup-merge-candidates.sh' "$CLEANUP_WORKFLOW" || \
   fail "staging cleanup must delegate candidate lifecycle to the tested helper"
 cleanup_sweep=$(job_block "$CLEANUP_WORKFLOW" sweep)

.github/workflows/browser-demos-pages.yml

@@ -28,8 +28,7 @@ on:
       - "libc/**"
       - "package.json"
       - "package-lock.json"
-      - "packages/registry/**/package.toml"
-      - "packages/registry/**/build.toml"
+      - "packages/registry/**"
       - "programs/**"
       - "rust-toolchain.toml"
       - "images/rootfs/**"
@@ -38,6 +37,7 @@ on:
       - "scripts/build-musl.sh"
       - "scripts/build-programs.sh"
       - "scripts/build-rootfs.sh"
+      - "scripts/browser-binary-package-roots.mjs"
       - "scripts/check-pages-publish-size.mjs"
       - "scripts/check-pages-run-freshness.sh"
       - "scripts/ci-check-pages-deployment.sh"
@@ -49,6 +49,11 @@ on:
       - "scripts/install-overlay-headers.sh"
       - "scripts/install-local-binary.sh"
       - "scripts/resolve-binary.sh"
+      - "scripts/resolve-binary.ts"
+      - "scripts/resolve-binary.bundle.mjs"
+      - "scripts/resolve-binary.bundle.LICENSES.txt"
+      - "scripts/build-resolve-binary-bundle.sh"
+      - "scripts/test-resolve-binary-bundle.sh"
       - "sdk/**"
       - "tools/mkrootfs/**"
       - "tools/xtask/**"
@@ -105,6 +110,15 @@ jobs:
       - name: Set up Nix
         uses: ./.github/actions/setup-nix
 
+      - name: Verify browser package projection is current
+        run: |
+          bash scripts/dev-shell.sh bash -c '
+            host_target="$(rustc -vV | sed -n "s/^host: //p")"
+            cargo run -p xtask --target "$host_target" --quiet -- \
+              build-deps program-index-check \
+              packages/registry packages/registry/program-packages.json
+          '
+
       - name: Prepare browser demo assets
         run: bash scripts/dev-shell.sh ./run.sh prepare-browser --allow-stale
 

.github/workflows/homebrew-main-shell-ci.yml

@@ -46,6 +46,11 @@ on:
       - "scripts/install-local-binary.sh"
       - "scripts/install-overlay-headers.sh"
       - "scripts/resolve-binary.sh"
+      - "scripts/resolve-binary.ts"
+      - "scripts/resolve-binary.bundle.mjs"
+      - "scripts/resolve-binary.bundle.LICENSES.txt"
+      - "scripts/build-resolve-binary-bundle.sh"
+      - "scripts/test-resolve-binary-bundle.sh"
       - "scripts/recover-homebrew-bottle-mirror.ts"
       - "scripts/test-homebrew-main-shell-closure.sh"
       - "scripts/verify-homebrew-main-shell-artifact-lock.sh"
@@ -102,6 +107,11 @@ on:
       - "scripts/install-local-binary.sh"
       - "scripts/install-overlay-headers.sh"
       - "scripts/resolve-binary.sh"
+      - "scripts/resolve-binary.ts"
+      - "scripts/resolve-binary.bundle.mjs"
+      - "scripts/resolve-binary.bundle.LICENSES.txt"
+      - "scripts/build-resolve-binary-bundle.sh"
+      - "scripts/test-resolve-binary-bundle.sh"
       - "scripts/recover-homebrew-bottle-mirror.ts"
       - "scripts/test-homebrew-main-shell-closure.sh"
       - "scripts/verify-homebrew-main-shell-artifact-lock.sh"
@@ -292,16 +302,24 @@ jobs:
           CANDIDATE_PATH: ${{ steps.candidate.outputs.image }}
         run: |
           set -euo pipefail
-          installed=local-binaries/programs/wasm32/shell.vfs.zst
           browser_copy=apps/browser-demos/public/shell.vfs.zst
+          install_session="homebrew-main-shell-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_JOB}"
           test -f "$CANDIDATE_PATH"
-          mkdir -p "$(dirname "$installed")" "$(dirname "$browser_copy")"
-          cp "$CANDIDATE_PATH" "$installed"
+          mkdir -p "$(dirname "$browser_copy")"
+          bash scripts/dev-shell.sh bash -c '
+            set -euo pipefail
+            host_target="$(rustc -vV | sed -n "s/^host: //p")"
+            WASM_POSIX_LOCAL_INSTALL_SOURCE="$1" \
+            WASM_POSIX_LOCAL_INSTALL_SESSION="$2" \
+              cargo run -p xtask --target "$host_target" --quiet -- \
+                build-deps --arch wasm32 --binaries-dir local-binaries \
+                install-local-artifact shell shell.vfs.zst
+          ' bash "$CANDIDATE_PATH" "$install_session"
+          resolved=$(bash scripts/resolve-binary.sh programs/shell.vfs.zst)
+          test -f "$resolved"
+          cmp "$CANDIDATE_PATH" "$resolved"
           cp "$CANDIDATE_PATH" "$browser_copy"
-          cmp "$CANDIDATE_PATH" "$installed"
           cmp "$CANDIDATE_PATH" "$browser_copy"
-          resolved=$(bash scripts/resolve-binary.sh programs/shell.vfs.zst)
-          test "$(realpath "$resolved")" = "$(realpath "$installed")"
           image_sha=$(sha256sum "$CANDIDATE_PATH" | awk '{print $1}')
           [[ "$image_sha" =~ ^[0-9a-f]{64}$ ]]
           echo "sha256=$image_sha" >> "$GITHUB_OUTPUT"

.github/workflows/prepare-merge.yml

@@ -1291,6 +1291,9 @@ jobs:
       - name: Set up Nix
         uses: ./.github/actions/setup-nix
 
+      - name: Install root npm deps
+        run: bash scripts/dev-shell.sh npm ci --no-audit --no-fund
+
       - name: Install host npm deps
         run: |
           bash scripts/dev-shell.sh bash -c '

.github/workflows/staging-build.yml

@@ -707,6 +707,9 @@ jobs:
       - name: Set up Nix
         uses: ./.github/actions/setup-nix
 
+      - name: Install root npm deps
+        run: bash scripts/dev-shell.sh npm ci --no-audit --no-fund
+
       - name: Install host npm deps
         run: |
           bash scripts/dev-shell.sh bash -c '

.gitignore

@@ -15,6 +15,7 @@ host/wasm/
 # override tree that takes priority. Neither is tracked.
 /binaries/
 /local-binaries/
+/.ci-test-binary-cache/
 
 # PR-build overlay file. Created by scripts/fetch-binaries.sh when
 # checked out on a PR branch with a corresponding pr-<NNN>-staging
@@ -132,3 +133,8 @@ benchmarks/wasm/
 
 # Phase C: per-package PR overlay (CI-generated; merged over package.toml at parse time).
 packages/registry/*/package.pr.toml
+
+# Durable advisory-lock inode used to serialize generated package-index
+# replacement across cooperating xtask processes. It is intentionally retained
+# so waiters never split onto different lock inodes.
+.*.kandelo-index.lock

apps/browser-demos/binary-dev-access.ts

@@ -0,0 +1,160 @@
+import fs from "node:fs";
+import path from "node:path";
+import {
+  normalizePath,
+  type ViteDevServer,
+} from "vite";
+
+export interface BinaryDevAccess {
+  approve(file: string): string;
+  approveBatch(files: readonly string[]): string[];
+  attachServer(server: ViteDevServer): void;
+}
+
+export interface BinaryDevAccessOptions {
+  repoRoot: string;
+  programCacheRoot: string;
+  caseInsensitivePaths: boolean;
+}
+
+const invalidFsRequest = Symbol("invalid-fs-request");
+
+function fsRequestPath(
+  url: string | undefined,
+): string | typeof invalidFsRequest | null {
+  if (!url) return null;
+  const pathname = new URL(url, "http://127.0.0.1").pathname;
+  if (!pathname.startsWith("/@fs/")) return null;
+  let decoded: string;
+  try {
+    decoded = decodeURIComponent(pathname.slice("/@fs/".length));
+  } catch {
+    return invalidFsRequest;
+  }
+  let file = normalizePath(decoded);
+  if (!path.isAbsolute(file) && !/^[A-Za-z]:\//.test(file)) file = `/${file}`;
+  return normalizePath(file);
+}
+
+export function pathIsWithin(
+  root: string,
+  file: string,
+  caseInsensitivePaths: boolean,
+): boolean {
+  const comparableRoot = caseInsensitivePaths ? root.toLowerCase() : root;
+  const comparableFile = caseInsensitivePaths ? file.toLowerCase() : file;
+  const fromRoot = path.relative(comparableRoot, comparableFile);
+  return fromRoot === ""
+    || (fromRoot !== ".."
+      && !fromRoot.startsWith(`..${path.sep}`)
+      && !path.isAbsolute(fromRoot));
+}
+
+/**
+ * Give Vite access only to exact resolver-approved files outside the checkout.
+ *
+ * Batch approval deliberately validates every path before publishing any
+ * capability. A malformed later package member must not make earlier members
+ * from the same failed graph directly servable through Vite's broad lexical
+ * transport allow-list.
+ */
+export function createBinaryDevAccess(
+  options: BinaryDevAccessOptions,
+): BinaryDevAccess {
+  const approvedExternalFiles = new Set<string>();
+  const attachedServers = new WeakSet<ViteDevServer>();
+  const programCacheRoot = normalizePath(options.programCacheRoot);
+  const repoRoot = normalizePath(options.repoRoot);
+
+  function validate(file: string): {
+    canonical: string;
+    externalCapability: string | null;
+  } {
+    const canonical = normalizePath(fs.realpathSync(file));
+    if (!fs.lstatSync(canonical).isFile()) {
+      throw new Error(
+        `Resolved browser artifact is not a regular file: ${canonical}`,
+      );
+    }
+    const isInsideProgramCache = pathIsWithin(
+      programCacheRoot,
+      canonical,
+      options.caseInsensitivePaths,
+    );
+    const isInsideRepo = pathIsWithin(
+      repoRoot,
+      canonical,
+      options.caseInsensitivePaths,
+    );
+    if (isInsideProgramCache) {
+      // The middleware guards the cache namespace even when an explicit cache
+      // root overlaps the checkout, so every cache file needs an exact
+      // capability regardless of repository containment.
+      return { canonical, externalCapability: canonical };
+    }
+    if (!isInsideRepo) {
+      throw new Error(
+        `Resolved browser artifact is outside the Kandelo program cache: ${canonical}`,
+      );
+    }
+    return { canonical, externalCapability: null };
+  }
+
+  const access: BinaryDevAccess = {
+    approve(file: string): string {
+      return access.approveBatch([file])[0]!;
+    },
+    approveBatch(files: readonly string[]): string[] {
+      const validated = files.map(validate);
+      // Commit only after every member is canonical, regular, and inside an
+      // allowed root. This is the capability-side half of atomic graph
+      // resolution; validation failure above leaves the set unchanged.
+      for (const { externalCapability } of validated) {
+        if (externalCapability !== null) {
+          approvedExternalFiles.add(externalCapability);
+        }
+      }
+      return validated.map(({ canonical }) => canonical);
+    },
+    attachServer(nextServer: ViteDevServer): void {
+      if (attachedServers.has(nextServer)) return;
+      attachedServers.add(nextServer);
+      nextServer.middlewares.use((request, response, next) => {
+        const requested = fsRequestPath(request.url);
+        if (requested === invalidFsRequest) {
+          response.statusCode = 403;
+          response.end("Malformed filesystem path");
+          return;
+        }
+        if (
+          !requested
+          || !pathIsWithin(
+            programCacheRoot,
+            requested,
+            options.caseInsensitivePaths,
+          )
+        ) {
+          next();
+          return;
+        }
+        try {
+          if (
+            !approvedExternalFiles.has(requested)
+            || !fs.lstatSync(requested).isFile()
+            || normalizePath(fs.realpathSync(requested)) !== requested
+          ) {
+            response.statusCode = 403;
+            response.end("Forbidden resolver-cache path");
+            return;
+          }
+        } catch {
+          response.statusCode = 403;
+          response.end("Forbidden resolver-cache path");
+          return;
+        }
+        next();
+      });
+    },
+  };
+  return access;
+}