diff --git a/.github/actions/detect-change-scope/ci-scope-paths.sh b/.github/actions/detect-change-scope/ci-scope-paths.sh index 651debc12f..9cdbf68c1b 100644 --- a/.github/actions/detect-change-scope/ci-scope-paths.sh +++ b/.github/actions/detect-change-scope/ci-scope-paths.sh @@ -59,7 +59,10 @@ binary_materialization_changed_files() { grep -E \ -e '^tools/xtask/src/(index_toml|remote_fetch|util)\.rs$' \ -e '^scripts/(fetch-binaries|install-local-binary|materialize-pr-overlays|resolve-binary|test-wasm-artifact-guards|wasm-artifact-guards)\.sh$' \ + -e '^scripts/(build-resolve-binary-bundle|test-resolve-binary-bundle)\.sh$' \ + -e '^scripts/resolve-binary\.(ts|bundle\.mjs|bundle\.LICENSES\.txt)$' \ -e '^scripts/vfs-has-stale-abi\.mjs$' \ + -e '^host/src/binary-resolver\.ts$' \ -e '^tests/package-system/' \ || true } diff --git a/.github/actions/detect-change-scope/test-ci-scope-paths.sh b/.github/actions/detect-change-scope/test-ci-scope-paths.sh index f0a398d7b1..6d5d8bd49b 100755 --- a/.github/actions/detect-change-scope/test-ci-scope-paths.sh +++ b/.github/actions/detect-change-scope/test-ci-scope-paths.sh @@ -126,6 +126,17 @@ assert_matches binary_materialization_changed_files \ assert_matches binary_materialization_changed_files \ "scripts/vfs-has-stale-abi.mjs" \ "scripts/vfs-has-stale-abi.mjs" +for resolver_input in \ + host/src/binary-resolver.ts \ + scripts/resolve-binary.ts \ + scripts/resolve-binary.bundle.mjs \ + scripts/resolve-binary.bundle.LICENSES.txt \ + scripts/build-resolve-binary-bundle.sh \ + scripts/test-resolve-binary-bundle.sh; do + assert_matches binary_materialization_changed_files \ + "$resolver_input" \ + "$resolver_input" +done assert_matches binary_materialization_changed_files \ "tests/package-system/fetch-binaries-allow-stale.test.ts" \ "tests/package-system/fetch-binaries-allow-stale.test.ts" diff --git a/.github/scripts/test-merge-candidate-workflows.sh b/.github/scripts/test-merge-candidate-workflows.sh index 6b2de8744a..70006b642f 100755 --- a/.github/scripts/test-merge-candidate-workflows.sh +++ b/.github/scripts/test-merge-candidate-workflows.sh @@ -654,6 +654,28 @@ for step in "Compute matrix" "Materialize binaries"; do fi done +for workflow in "$STAGING_WORKFLOW" "$PREPARE"; do + validation_job="$(job_block "$workflow" test-gate-validation)" + root_install_line="$( + grep -nF -- '- name: Install root npm deps' <<<"$validation_job" | + head -n 1 | + cut -d: -f1 + )" + materialization_line="$( + grep -nF -- '- name: Test binary materialization flow' <<<"$validation_job" | + head -n 1 | + cut -d: -f1 + )" + [ -n "$root_install_line" ] && + [ -n "$materialization_line" ] && + [ "$root_install_line" -lt "$materialization_line" ] || + fail "$(basename "$workflow") must install root npm dependencies before materialization tests" + root_install_step="$(step_block "$workflow" "Install root npm deps")" + grep -Fq 'run: bash scripts/dev-shell.sh npm ci --no-audit --no-fund' \ + <<<"$root_install_step" || + fail "$(basename "$workflow") materialization validation must install the root esbuild dependency" +done + grep -Fq 'cleanup-merge-candidates.sh' "$CLEANUP_WORKFLOW" || \ fail "staging cleanup must delegate candidate lifecycle to the tested helper" cleanup_sweep=$(job_block "$CLEANUP_WORKFLOW" sweep) diff --git a/.github/workflows/browser-demos-pages.yml b/.github/workflows/browser-demos-pages.yml index e96126f334..225707f81e 100644 --- a/.github/workflows/browser-demos-pages.yml +++ b/.github/workflows/browser-demos-pages.yml @@ -28,8 +28,7 @@ on: - "libc/**" - "package.json" - "package-lock.json" - - "packages/registry/**/package.toml" - - "packages/registry/**/build.toml" + - "packages/registry/**" - "programs/**" - "rust-toolchain.toml" - "images/rootfs/**" @@ -38,6 +37,7 @@ on: - "scripts/build-musl.sh" - "scripts/build-programs.sh" - "scripts/build-rootfs.sh" + - "scripts/browser-binary-package-roots.mjs" - "scripts/check-pages-publish-size.mjs" - "scripts/check-pages-run-freshness.sh" - "scripts/ci-check-pages-deployment.sh" @@ -49,6 +49,11 @@ on: - "scripts/install-overlay-headers.sh" - "scripts/install-local-binary.sh" - "scripts/resolve-binary.sh" + - "scripts/resolve-binary.ts" + - "scripts/resolve-binary.bundle.mjs" + - "scripts/resolve-binary.bundle.LICENSES.txt" + - "scripts/build-resolve-binary-bundle.sh" + - "scripts/test-resolve-binary-bundle.sh" - "sdk/**" - "tools/mkrootfs/**" - "tools/xtask/**" @@ -105,6 +110,15 @@ jobs: - name: Set up Nix uses: ./.github/actions/setup-nix + - name: Verify browser package projection is current + run: | + bash scripts/dev-shell.sh bash -c ' + host_target="$(rustc -vV | sed -n "s/^host: //p")" + cargo run -p xtask --target "$host_target" --quiet -- \ + build-deps program-index-check \ + packages/registry packages/registry/program-packages.json + ' + - name: Prepare browser demo assets run: bash scripts/dev-shell.sh ./run.sh prepare-browser --allow-stale diff --git a/.github/workflows/homebrew-main-shell-ci.yml b/.github/workflows/homebrew-main-shell-ci.yml index fdcd881da8..f50c759c2c 100644 --- a/.github/workflows/homebrew-main-shell-ci.yml +++ b/.github/workflows/homebrew-main-shell-ci.yml @@ -46,6 +46,11 @@ on: - "scripts/install-local-binary.sh" - "scripts/install-overlay-headers.sh" - "scripts/resolve-binary.sh" + - "scripts/resolve-binary.ts" + - "scripts/resolve-binary.bundle.mjs" + - "scripts/resolve-binary.bundle.LICENSES.txt" + - "scripts/build-resolve-binary-bundle.sh" + - "scripts/test-resolve-binary-bundle.sh" - "scripts/recover-homebrew-bottle-mirror.ts" - "scripts/test-homebrew-main-shell-closure.sh" - "scripts/verify-homebrew-main-shell-artifact-lock.sh" @@ -102,6 +107,11 @@ on: - "scripts/install-local-binary.sh" - "scripts/install-overlay-headers.sh" - "scripts/resolve-binary.sh" + - "scripts/resolve-binary.ts" + - "scripts/resolve-binary.bundle.mjs" + - "scripts/resolve-binary.bundle.LICENSES.txt" + - "scripts/build-resolve-binary-bundle.sh" + - "scripts/test-resolve-binary-bundle.sh" - "scripts/recover-homebrew-bottle-mirror.ts" - "scripts/test-homebrew-main-shell-closure.sh" - "scripts/verify-homebrew-main-shell-artifact-lock.sh" @@ -292,16 +302,24 @@ jobs: CANDIDATE_PATH: ${{ steps.candidate.outputs.image }} run: | set -euo pipefail - installed=local-binaries/programs/wasm32/shell.vfs.zst browser_copy=apps/browser-demos/public/shell.vfs.zst + install_session="homebrew-main-shell-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_JOB}" test -f "$CANDIDATE_PATH" - mkdir -p "$(dirname "$installed")" "$(dirname "$browser_copy")" - cp "$CANDIDATE_PATH" "$installed" + mkdir -p "$(dirname "$browser_copy")" + bash scripts/dev-shell.sh bash -c ' + set -euo pipefail + host_target="$(rustc -vV | sed -n "s/^host: //p")" + WASM_POSIX_LOCAL_INSTALL_SOURCE="$1" \ + WASM_POSIX_LOCAL_INSTALL_SESSION="$2" \ + cargo run -p xtask --target "$host_target" --quiet -- \ + build-deps --arch wasm32 --binaries-dir local-binaries \ + install-local-artifact shell shell.vfs.zst + ' bash "$CANDIDATE_PATH" "$install_session" + resolved=$(bash scripts/resolve-binary.sh programs/shell.vfs.zst) + test -f "$resolved" + cmp "$CANDIDATE_PATH" "$resolved" cp "$CANDIDATE_PATH" "$browser_copy" - cmp "$CANDIDATE_PATH" "$installed" cmp "$CANDIDATE_PATH" "$browser_copy" - resolved=$(bash scripts/resolve-binary.sh programs/shell.vfs.zst) - test "$(realpath "$resolved")" = "$(realpath "$installed")" image_sha=$(sha256sum "$CANDIDATE_PATH" | awk '{print $1}') [[ "$image_sha" =~ ^[0-9a-f]{64}$ ]] echo "sha256=$image_sha" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/prepare-merge.yml b/.github/workflows/prepare-merge.yml index fff6908654..7b38739601 100644 --- a/.github/workflows/prepare-merge.yml +++ b/.github/workflows/prepare-merge.yml @@ -1291,6 +1291,9 @@ jobs: - name: Set up Nix uses: ./.github/actions/setup-nix + - name: Install root npm deps + run: bash scripts/dev-shell.sh npm ci --no-audit --no-fund + - name: Install host npm deps run: | bash scripts/dev-shell.sh bash -c ' diff --git a/.github/workflows/staging-build.yml b/.github/workflows/staging-build.yml index e0167e309f..50332dae8e 100644 --- a/.github/workflows/staging-build.yml +++ b/.github/workflows/staging-build.yml @@ -707,6 +707,9 @@ jobs: - name: Set up Nix uses: ./.github/actions/setup-nix + - name: Install root npm deps + run: bash scripts/dev-shell.sh npm ci --no-audit --no-fund + - name: Install host npm deps run: | bash scripts/dev-shell.sh bash -c ' diff --git a/.gitignore b/.gitignore index d1e6ebaca7..79884f82ab 100644 --- a/.gitignore +++ b/.gitignore @@ -132,3 +132,8 @@ benchmarks/wasm/ # Phase C: per-package PR overlay (CI-generated; merged over package.toml at parse time). packages/registry/*/package.pr.toml + +# Durable advisory-lock inode used to serialize generated package-index +# replacement across cooperating xtask processes. It is intentionally retained +# so waiters never split onto different lock inodes. +.*.kandelo-index.lock diff --git a/Cargo.lock b/Cargo.lock index 1f7bfb6cbb..d3b5b947cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1983,6 +1983,7 @@ dependencies = [ "flate2", "jsonschema", "regex", + "rustix", "serde", "serde_json", "sha2", diff --git a/apps/browser-demos/test/vite-binary-cache-boundary.spec.ts b/apps/browser-demos/test/vite-binary-cache-boundary.spec.ts new file mode 100644 index 0000000000..0404fb8865 --- /dev/null +++ b/apps/browser-demos/test/vite-binary-cache-boundary.spec.ts @@ -0,0 +1,335 @@ +import { expect, test } from "@playwright/test"; +import { createHash, randomUUID } from "node:crypto"; +import { + existsSync, + mkdirSync, + mkdtempSync, + realpathSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import type { AddressInfo } from "node:net"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { createServer, normalizePath, type ViteDevServer } from "vite"; + +const appRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const repoRoot = resolve(appRoot, "../.."); + +function fsUrl(origin: string, file: string): string { + const normalized = normalizePath(file).replace(/^\//, ""); + return `${origin}/@fs/${encodeURI(normalized)}`; +} + +function writeProgramProjection( + registryRoot: string, + packageName: string, + cacheKey: string, +): void { + const manifest = [ + "[package]", + `name = ${JSON.stringify(packageName)}`, + 'version = "1.0.0"', + "revision = 1", + "", + ].join("\n"); + const packageRoot = join(registryRoot, packageName); + mkdirSync(packageRoot, { recursive: true }); + writeFileSync(join(packageRoot, "package.toml"), manifest); + writeFileSync( + join(registryRoot, "program-packages.json"), + `${JSON.stringify({ + format: "kandelo-program-packages-v2", + identities: { + [packageName]: { + manifestSha256: createHash("sha256").update(manifest).digest("hex"), + cacheKeys: { + wasm32: cacheKey, + wasm64: createHash("sha256") + .update(`${packageName}:wasm64`) + .digest("hex"), + }, + }, + }, + packages: { + [packageName]: { + manifestSha256: createHash("sha256").update(manifest).digest("hex"), + arches: ["wasm32"], + cacheKeys: { wasm32: cacheKey }, + dependencyClosures: { wasm32: [] }, + members: [ + { + kind: "output", + sourceArtifact: "artifact.dat", + mirrorPath: `${packageName}/artifact.dat`, + outputName: "artifact", + forkInstrumentation: "disabled", + }, + { + kind: "output", + sourceArtifact: "sidecar.dat", + mirrorPath: `${packageName}/sidecar.dat`, + outputName: "sidecar", + forkInstrumentation: "disabled", + }, + ], + }, + }, + }, null, 2)}\n`, + ); +} + +test("Vite serves an approved bottle member without exposing its cache", async () => { + const savedXdgCacheHome = process.env.XDG_CACHE_HOME; + const savedBinaryCacheRoot = process.env.WASM_POSIX_BINARY_CACHE_ROOT; + const savedRegistry = process.env.WASM_POSIX_DEPS_REGISTRY; + const savedNoHmr = process.env.KANDELO_BROWSER_TEST_NO_HMR; + const testRoot = mkdtempSync(join(tmpdir(), "kandelo-vite-cache-boundary-")); + const namespace = `vite-cache-boundary-${randomUUID()}`; + const cacheKey = "a".repeat(64); + const cacheRoot = join(testRoot, "kandelo"); + const registryRoot = join(testRoot, "registry"); + const generation = join( + cacheRoot, + "programs", + `${namespace}-1.0.0-rev1-wasm32-${cacheKey}`, + ); + const artifact = join(generation, "artifact.dat"); + const sidecar = join(generation, "sidecar.dat"); + const privateSource = join(cacheRoot, "sources", "private.dat"); + const cacheEscape = join(cacheRoot, "programs", "escape.dat"); + const mirror = join( + repoRoot, + "binaries", + "programs", + "wasm32", + namespace, + "artifact.dat", + ); + const sidecarMirror = join( + repoRoot, + "binaries", + "programs", + "wasm32", + namespace, + "sidecar.dat", + ); + const entryDirectory = join(appRoot, "test-runs", namespace); + const entry = join(entryDirectory, "entry.ts"); + const artifactBytes = "approved bottle member\n".repeat(512); + let server: ViteDevServer | null = null; + + try { + mkdirSync(dirname(artifact), { recursive: true }); + mkdirSync(dirname(privateSource), { recursive: true }); + mkdirSync(dirname(mirror), { recursive: true }); + mkdirSync(entryDirectory, { recursive: true }); + writeFileSync(artifact, artifactBytes); + writeFileSync(sidecar, "package sidecar\n"); + writeFileSync(privateSource, "private source bytes\n"); + symlinkSync(privateSource, cacheEscape); + symlinkSync(artifact, mirror); + symlinkSync(sidecar, sidecarMirror); + writeProgramProjection(registryRoot, namespace, cacheKey); + writeFileSync( + entry, + `import artifactUrl from "@binaries/programs/wasm32/${namespace}/artifact.dat?url";\nexport default artifactUrl;\n`, + ); + + process.env.XDG_CACHE_HOME = testRoot; + delete process.env.WASM_POSIX_BINARY_CACHE_ROOT; + process.env.WASM_POSIX_DEPS_REGISTRY = registryRoot; + process.env.KANDELO_BROWSER_TEST_NO_HMR = "1"; + server = await createServer({ + configFile: join(appRoot, "vite.config.ts"), + root: appRoot, + logLevel: "silent", + server: { host: "127.0.0.1", port: 0, hmr: false }, + }); + await server.listen(); + const address = server.httpServer!.address() as AddressInfo; + const origin = `http://127.0.0.1:${address.port}`; + const canonicalArtifact = realpathSync(artifact); + const canonicalProgramRoot = realpathSync(join(cacheRoot, "programs")); + + expect((await fetch(fsUrl(origin, canonicalArtifact))).status).toBe(403); + const transformedEntry = await fetch(fsUrl(origin, entry)); + const transformedSource = await transformedEntry.text(); + expect(transformedEntry.status, transformedSource).toBe(200); + expect(transformedSource).toContain("artifact.dat"); + + const modulePath = transformedSource.match( + /from\s+("\/@fs\/[^"\n]+artifact\.dat\?import&url")/, + )?.[1]; + expect(modulePath).toBeDefined(); + const assetModule = await fetch(new URL(JSON.parse(modulePath!), origin)); + const assetModuleSource = await assetModule.text(); + expect(assetModule.status, assetModuleSource).toBe(200); + const assetPath = assetModuleSource.match( + /export default ("[^"\n]+")\s*;?/, + )?.[1]; + expect(assetPath, assetModuleSource).toBeDefined(); + const importedAsset = await fetch(new URL(JSON.parse(assetPath!), origin)); + expect(importedAsset.status).toBe(200); + expect(await importedAsset.text()).toBe(artifactBytes); + + const approvedResponse = await fetch(fsUrl(origin, canonicalArtifact)); + const approvedBody = await approvedResponse.text(); + expect( + approvedResponse.status, + JSON.stringify({ + approvedBody, + transformedSource, + allow: server.config.server.fs.allow, + }, null, 2), + ).toBe(200); + expect(approvedBody).toBe(artifactBytes); + expect((await fetch(fsUrl(origin, realpathSync(privateSource)))).status).toBe(403); + expect((await fetch( + fsUrl(origin, join(canonicalProgramRoot, "escape.dat")), + )).status).toBe(403); + const caseVariant = canonicalArtifact.replace( + namespace, + namespace.toUpperCase(), + ); + if (caseVariant !== canonicalArtifact && existsSync(caseVariant)) { + expect((await fetch(fsUrl(origin, caseVariant))).status).toBe(403); + } + expect((await fetch(`${origin}/@fs/%E0%A4%A`)).status).toBe(403); + + rmSync(artifact); + mkdirSync(artifact); + const descendant = join(artifact, "private.dat"); + writeFileSync(descendant, "replacement directory bytes\n"); + expect((await fetch(fsUrl(origin, canonicalArtifact))).status).toBe(403); + expect((await fetch(fsUrl( + origin, + join(canonicalArtifact, "private.dat"), + ))).status).toBe(403); + } finally { + await server?.close(); + rmSync(join(repoRoot, "binaries", "programs", "wasm32", namespace), { + recursive: true, + force: true, + }); + rmSync(entryDirectory, { recursive: true, force: true }); + rmSync(testRoot, { recursive: true, force: true }); + if (savedXdgCacheHome === undefined) { + delete process.env.XDG_CACHE_HOME; + } else { + process.env.XDG_CACHE_HOME = savedXdgCacheHome; + } + if (savedBinaryCacheRoot === undefined) { + delete process.env.WASM_POSIX_BINARY_CACHE_ROOT; + } else { + process.env.WASM_POSIX_BINARY_CACHE_ROOT = savedBinaryCacheRoot; + } + if (savedRegistry === undefined) { + delete process.env.WASM_POSIX_DEPS_REGISTRY; + } else { + process.env.WASM_POSIX_DEPS_REGISTRY = savedRegistry; + } + if (savedNoHmr === undefined) { + delete process.env.KANDELO_BROWSER_TEST_NO_HMR; + } else { + process.env.KANDELO_BROWSER_TEST_NO_HMR = savedNoHmr; + } + } +}); + +test("Vite approves an explicit program cache that overlaps the checkout", async () => { + const savedBinaryCacheRoot = process.env.WASM_POSIX_BINARY_CACHE_ROOT; + const savedRegistry = process.env.WASM_POSIX_DEPS_REGISTRY; + const savedNoHmr = process.env.KANDELO_BROWSER_TEST_NO_HMR; + const namespace = `vite-overlap-${randomUUID()}`; + const cacheKey = "b".repeat(64); + const cacheRoot = join(repoRoot, `.vite-overlap-cache-${namespace}`); + const registryRoot = join(cacheRoot, "registry"); + const generation = join( + cacheRoot, + "programs", + `${namespace}-1.0.0-rev1-wasm32-${cacheKey}`, + ); + const artifact = join(generation, "artifact.dat"); + const sidecar = join(generation, "sidecar.dat"); + const mirror = join( + repoRoot, + "binaries", + "programs", + "wasm32", + namespace, + "artifact.dat", + ); + const sidecarMirror = join( + repoRoot, + "binaries", + "programs", + "wasm32", + namespace, + "sidecar.dat", + ); + const entryDirectory = join(appRoot, "test-runs", namespace); + const entry = join(entryDirectory, "entry.ts"); + let server: ViteDevServer | null = null; + + try { + mkdirSync(dirname(artifact), { recursive: true }); + mkdirSync(dirname(mirror), { recursive: true }); + mkdirSync(entryDirectory, { recursive: true }); + writeFileSync(artifact, "repo-overlap bottle member\n"); + writeFileSync(sidecar, "package sidecar\n"); + symlinkSync(artifact, mirror); + symlinkSync(sidecar, sidecarMirror); + writeProgramProjection(registryRoot, namespace, cacheKey); + writeFileSync( + entry, + `import artifactUrl from "@binaries/programs/wasm32/${namespace}/artifact.dat?url";\nexport default artifactUrl;\n`, + ); + process.env.WASM_POSIX_BINARY_CACHE_ROOT = cacheRoot; + process.env.WASM_POSIX_DEPS_REGISTRY = registryRoot; + process.env.KANDELO_BROWSER_TEST_NO_HMR = "1"; + + server = await createServer({ + configFile: join(appRoot, "vite.config.ts"), + root: appRoot, + logLevel: "silent", + server: { host: "127.0.0.1", port: 0, hmr: false }, + }); + await server.listen(); + const address = server.httpServer!.address() as AddressInfo; + const origin = `http://127.0.0.1:${address.port}`; + const canonicalArtifact = realpathSync(artifact); + expect((await fetch(fsUrl(origin, canonicalArtifact))).status).toBe(403); + + const transformedEntry = await fetch(fsUrl(origin, entry)); + const transformedSource = await transformedEntry.text(); + expect(transformedEntry.status, transformedSource).toBe(200); + expect(transformedSource).toContain("artifact.dat"); + expect((await fetch(fsUrl(origin, canonicalArtifact))).status).toBe(200); + } finally { + await server?.close(); + rmSync(join(repoRoot, "binaries", "programs", "wasm32", namespace), { + recursive: true, + force: true, + }); + rmSync(entryDirectory, { recursive: true, force: true }); + rmSync(cacheRoot, { recursive: true, force: true }); + if (savedBinaryCacheRoot === undefined) { + delete process.env.WASM_POSIX_BINARY_CACHE_ROOT; + } else { + process.env.WASM_POSIX_BINARY_CACHE_ROOT = savedBinaryCacheRoot; + } + if (savedRegistry === undefined) { + delete process.env.WASM_POSIX_DEPS_REGISTRY; + } else { + process.env.WASM_POSIX_DEPS_REGISTRY = savedRegistry; + } + if (savedNoHmr === undefined) { + delete process.env.KANDELO_BROWSER_TEST_NO_HMR; + } else { + process.env.KANDELO_BROWSER_TEST_NO_HMR = savedNoHmr; + } + } +}); diff --git a/apps/browser-demos/vite.config.ts b/apps/browser-demos/vite.config.ts index c8006fccd4..b849085be8 100644 --- a/apps/browser-demos/vite.config.ts +++ b/apps/browser-demos/vite.config.ts @@ -4,18 +4,144 @@ import fs from "fs"; import { execSync } from "child_process"; import { defineConfig, + normalizePath, type Plugin, type PreviewServer, type ViteDevServer, } from "vite"; import react from "@vitejs/plugin-react"; -import { tryResolveBinary } from "../../host/src/binary-resolver"; +import { + binaryProgramCacheRoot, + tryResolveBinary, +} from "../../host/src/binary-resolver"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const repoRoot = path.resolve(__dirname, "../.."); + +function canonicalizeFromExistingAncestor(file: string): string { + const suffix: string[] = []; + let existing = path.resolve(file); + while (!fs.existsSync(existing)) { + const parent = path.dirname(existing); + if (parent === existing) return normalizePath(path.resolve(file)); + suffix.unshift(path.basename(existing)); + existing = parent; + } + return normalizePath(path.resolve(fs.realpathSync(existing), ...suffix)); +} + +const configuredProgramCacheRoot = binaryProgramCacheRoot(); +const browserProgramCacheRoot = canonicalizeFromExistingAncestor( + configuredProgramCacheRoot, +); +const caseInsensitivePaths = fs.existsSync( + path.join(__dirname, "VITE.CONFIG.TS"), +); const DEFAULT_CORS_PROXY_URL = "https://wordpress-playground-cors-proxy.net/?"; const preferredLocalPort = 5401; +interface BinaryDevAccess { + approve(file: string): string; + attachServer(server: ViteDevServer): void; +} + +const invalidFsRequest = Symbol("invalid-fs-request"); + +function fsRequestPath( + url: string | undefined, +): string | typeof invalidFsRequest | null { + if (!url) return null; + const pathname = new URL(url, "http://127.0.0.1").pathname; + if (!pathname.startsWith("/@fs/")) return null; + let decoded: string; + try { + decoded = decodeURIComponent(pathname.slice("/@fs/".length)); + } catch { + return invalidFsRequest; + } + let file = normalizePath(decoded); + if (!path.isAbsolute(file) && !/^[A-Za-z]:\//.test(file)) file = `/${file}`; + return normalizePath(file); +} + +function pathIsWithin(root: string, file: string): boolean { + const comparableRoot = caseInsensitivePaths ? root.toLowerCase() : root; + const comparableFile = caseInsensitivePaths ? file.toLowerCase() : file; + const fromRoot = path.relative(comparableRoot, comparableFile); + return fromRoot === "" + || (fromRoot !== ".." + && !fromRoot.startsWith(`..${path.sep}`) + && !path.isAbsolute(fromRoot)); +} + +/** + * Give Vite access only to exact resolver-approved files outside the checkout. + * Vite needs the program-cache directory in its transport allow list, while + * this pre-serving guard turns that broad lexical rule into exact, rechecked + * regular-file capabilities. + */ +function createBinaryDevAccess(): BinaryDevAccess { + const approvedExternalFiles = new Set(); + const attachedServers = new WeakSet(); + const programCacheRoot = browserProgramCacheRoot; + + return { + approve(file: string): string { + const canonical = fs.realpathSync(file); + if (!fs.lstatSync(canonical).isFile()) { + throw new Error(`Resolved browser artifact is not a regular file: ${canonical}`); + } + const isInsideProgramCache = pathIsWithin(programCacheRoot, canonical); + const isInsideRepo = pathIsWithin(repoRoot, canonical); + if (isInsideProgramCache) { + // The middleware guards the cache namespace even when an explicit + // cache root overlaps the checkout, so every cache file needs an + // exact capability regardless of repository containment. + approvedExternalFiles.add(normalizePath(canonical)); + } else if (!isInsideRepo) { + throw new Error( + `Resolved browser artifact is outside the Kandelo program cache: ${canonical}`, + ); + } + return canonical; + }, + attachServer(nextServer: ViteDevServer): void { + if (attachedServers.has(nextServer)) return; + attachedServers.add(nextServer); + nextServer.middlewares.use((request, response, next) => { + const requested = fsRequestPath(request.url); + if (requested === invalidFsRequest) { + response.statusCode = 403; + response.end("Malformed filesystem path"); + return; + } + if (!requested || !pathIsWithin(programCacheRoot, requested)) { + next(); + return; + } + try { + if ( + !approvedExternalFiles.has(requested) + || !fs.lstatSync(requested).isFile() + || normalizePath(fs.realpathSync(requested)) !== requested + ) { + response.statusCode = 403; + response.end("Forbidden resolver-cache path"); + return; + } + } catch { + response.statusCode = 403; + response.end("Forbidden resolver-cache path"); + return; + } + next(); + }); + }, + }; +} + +const binaryDevAccess = createBinaryDevAccess(); + const crossOriginIsolationHeaders = { "Cross-Origin-Opener-Policy": "same-origin", "Cross-Origin-Embedder-Policy": "require-corp", @@ -87,7 +213,7 @@ function injectBlobIframeInterceptorPlaceholder(content: string): string { * these aliases can run without a kernel build present. Pages that do * import them get a clear error pointing at the build script. */ -function resolveKernelArtifactsAlias(): Plugin { +function resolveKernelArtifactsAlias(access: BinaryDevAccess): Plugin { const KERNEL = "@kernel-wasm"; const ROOTFS = "@rootfs-vfs"; return { @@ -100,7 +226,7 @@ function resolveKernelArtifactsAlias(): Plugin { if (pathPart === KERNEL) { const resolved = tryResolveBinary("kernel.wasm"); - if (resolved) return resolved + query; + if (resolved) return access.approve(resolved) + query; const local = path.resolve(repoRoot, "local-binaries/kernel.wasm"); const fetched = path.resolve(repoRoot, "binaries/kernel.wasm"); this.error( @@ -117,7 +243,7 @@ function resolveKernelArtifactsAlias(): Plugin { path.resolve(repoRoot, "binaries/programs/wasm32/rootfs.vfs"), ]; for (const file of candidates) { - if (fs.existsSync(file)) return file + query; + if (fs.existsSync(file)) return access.approve(file) + query; } this.error( "rootfs.vfs not found. Run `bash build.sh` from the repo root, or fetch/build the rootfs package.\n" + @@ -126,6 +252,9 @@ function resolveKernelArtifactsAlias(): Plugin { } return null; }, + configureServer(server) { + access.attachServer(server); + }, }; } @@ -184,7 +313,7 @@ function dropWorkerEntryExports(): Plugin { * which can't express "try this directory first, then that one." A * `resolveId` hook can. */ -function resolveBinariesAlias(): Plugin { +function resolveBinariesAlias(access: BinaryDevAccess): Plugin { const PREFIX = "@binaries/"; const applyDefaultArch = (rel: string): string => { if (!rel.startsWith("programs/")) return rel; @@ -204,7 +333,7 @@ function resolveBinariesAlias(): Plugin { const query = queryIdx === -1 ? "" : source.slice(queryIdx); const rest = applyDefaultArch(pathPart.slice(PREFIX.length)); const resolved = tryResolveBinary(rest); - if (resolved) return resolved + query; + if (resolved) return access.approve(resolved) + query; const local = path.resolve(repoRoot, "local-binaries", rest); const fetched = path.resolve(repoRoot, "binaries", rest); this.error( @@ -213,6 +342,9 @@ function resolveBinariesAlias(): Plugin { `Run \`./run.sh fetch\` to install release archives, or build the artifact locally.`, ); }, + configureServer(server) { + access.attachServer(server); + }, }; } @@ -511,8 +643,8 @@ export default defineConfig({ }, plugins: [ react(), - resolveKernelArtifactsAlias(), - resolveBinariesAlias(), + resolveKernelArtifactsAlias(binaryDevAccess), + resolveBinariesAlias(binaryDevAccess), rewriteNavLinks(), injectGitRevision(), injectCoiServiceWorker(), @@ -531,7 +663,11 @@ export default defineConfig({ ], } : undefined, fs: { - allow: [repoRoot], + // Multi-member package resolution returns canonical generation paths so + // a live mirror swap cannot change the bytes after validation. Resolver + // plugins approve exact files, and the pre-serving guard rejects every + // other cache path (including symlinks and approved-path descendants). + allow: [repoRoot, browserProgramCacheRoot], }, }, preview: { @@ -553,8 +689,8 @@ export default defineConfig({ worker: { format: "es", plugins: () => [ - resolveKernelArtifactsAlias(), - resolveBinariesAlias(), + resolveKernelArtifactsAlias(binaryDevAccess), + resolveBinariesAlias(binaryDevAccess), dropWorkerEntryExports(), ], }, diff --git a/docs/binary-releases.md b/docs/binary-releases.md index 19cd331acc..d5d8f96140 100644 --- a/docs/binary-releases.md +++ b/docs/binary-releases.md @@ -23,6 +23,39 @@ for the resolver behavior, schema, and build-script contract. For third-party repositories that publish their own package archives, see [docs/package-sources.md](package-sources.md). +Program archives have a second, source-controlled index: +`packages/registry/program-packages.json`. It is not a release ledger and does +not select archive URLs. Rust generates it from `package.toml` so every +consumer agrees on output/runtime closure membership, mirror placement, target +arches, and fork policy. Schema `kandelo-program-packages-v2` also records an +identity for every package kind and each program's full transitive dependency +identity per consumer architecture. Repository TypeScript, shell resolution, +external registry roots, and the standalone host npm package consume that +projection. The generated manifest digests and cache keys prevent a changed +selected recipe or dependency from silently using old policy. For an ordered +multi-root registry, the highest-priority existing index contains the complete +first-hit identity and program projection across lower roots too. A +dependency-only override rekeys affected lower programs in that combined +context; lower indexes remain standalone suffix-context fallbacks. The +first-party `kernel` and `userspace` boot artifacts retain identities in the +index but are excluded from its guest-program map because their outputs publish +at the binary root rather than below `programs//`. Regenerate the +projection whenever a package manifest or ordered dependency context changes; +package checks reject stale committed output. + +Source-checkout program resolution runs +`xtask build-deps program-index-context-check` synchronously before each public +program-resolution boundary. That one Rust implementation recomputes every +existing registry root in its ordered suffix context, including `build.toml` +revision and declared inputs, global toolchain inputs, and transitive +dependency identities. It fails closed when an index is missing or stale. +The standalone npm package does not reach back into a checkout; its projection +is checked before packaging and shipped as immutable package content. +Index generation stages complete JSON and serializes cooperating publishers +through a persistent advisory sidecar lock held across source refresh, target +validation, replacement, and directory sync. The lock file is intentionally +retained so concurrent writers always coordinate on one inode. + Homebrew bottles use a separate publication model. Bottle tarballs are Homebrew-native artifacts published through the `kandelo-dev/homebrew-tap-core` tap and GHCR/Homebrew bottle URL shape; Kandelo-specific sidecars and @@ -646,9 +679,49 @@ For each declared arch in the package's `arches = [...]` (default cache-key sha (catches recipe drift). 6. Places each program output under `binaries/programs//` using the manifest's output layout, and places declared non-Wasm runtime files under - `binaries/programs///`. Both are symlinks into the - validated cache, so browser/Node image builders load the same bytes without - re-fetching. Local builds use the identical layout under `local-binaries/`. + `binaries/programs///`. When the combined output + and runtime-file count is greater than one, every member—including the sole + executable of an executable-plus-runtime package—lives below one package + directory. Members are symlinks into the validated cache, so browser/Node + image builders load the same bytes without re-fetching. The host verifies + that every link ends in its declared source-artifact suffix and that all + links resolve into one canonical program-cache generation. It returns those + canonical member paths instead of the mutable mirror paths. + +For a multi-member package closure, the fetched materializer validates every +output and runtime file before changing the live mirror. It creates a +same-parent staging directory, renames the old live directory to a +transaction-owned backup, then renames the stage to the live name. A concurrent +path lookup can therefore see the complete old directory, a brief absence +between renames, or the complete new directory, but not a partially populated +live directory. Normal failures roll back or clean up only paths owned by that +transaction. An abrupt process crash can leave an inert stage or backup; those +orphans are not automatically scavenged because there is no lease proving that +another process is not still using them. + +Direct local builds use the same public layout but different backing storage. +One build-helper session collects exact declared suffixes under +`local-binaries/.kandelo-local-generations/////`. +Members are create-once regular files. Only a complete, validated tree can +claim its one publication attempt and atomically replace the live package +directory or scalar link; a claimed missing generation is never recreated. A +one-member package keeps its historical flat mirror name as a symlink to the +immutable generation member. + +The stage and live directory must be on the same filesystem so rename remains +atomic. Unix uses file symlinks for mirror members. Windows uses file symlinks +too, but missing symlink privileges or open handles that prevent a rename make +the transaction abort and roll back; the implementation does not depend on +Windows overwrite semantics. + +Canonical paths protect a consumer from later live-mirror swaps; they are not +open file descriptors or operating-system leases. Normal fetched cache entries +are treated as immutable, but force-source rebuild and stale-cache repair can +remove and recreate one canonical cache-key directory. Those maintenance +operations must not run concurrently with consumers of the same package or a +previously returned path may disappear or name replacement bytes. Append-only +local session generations do not have this maintenance exception unless a user +manually deletes resolver-owned state. The no-argument form above materializes every publishable registry root. A bounded consumer should repeat `--package` for its direct roots instead: diff --git a/docs/browser-support.md b/docs/browser-support.md index 41de31ed79..93505ff46b 100644 --- a/docs/browser-support.md +++ b/docs/browser-support.md @@ -307,7 +307,16 @@ file can be created. `saveShellDerivedVfsImage()` rejects a product build unless at least 64 MiB of data blocks and 8,192 inode slots remain after its immutable contents are written. This makes runtime allocation space a checked artifact contract instead of allowing an image to build successfully and then -fail with `ENOSPC` during normal browser initialization. +fail with `ENOSPC` during normal browser initialization. The shared save helper +also requires the serialized artifact's encoded growth ceiling to equal the +768 MiB product profile. A future product that intentionally needs a larger +reviewed profile must pass that exact ceiling explicitly rather than silently +drifting from its browser consumer; an override cannot select a smaller +profile. The Homebrew main-shell composer applies the same serialized-ceiling +check against its selected `--max-bytes` contract before it creates the output +artifact. Host-tree copies fail the build on any read or VFS write error. +Intentional omissions are declared through the copy helper's `exclude` option, +and every unexcluded symlink must be preserved explicitly or the build fails. ```typescript // Typical demo pattern @@ -674,6 +683,15 @@ export default { }; ``` +During development, `@binaries/...` imports can resolve to canonical package +members outside the checkout. Vite's directory allow list is only transport +plumbing: a pre-serving guard permits the exact regular files approved by the +binary resolver and rechecks their real paths on every request. Other program +cache entries, source-cache files, symlink escapes, malformed filesystem URLs, +and descendants created by replacing an approved file with a directory receive +HTTP 403. Production builds emit ordinary bundled assets and do not expose the +local package cache. + ## Known Limitations ### SharedArrayBuffer restrictions diff --git a/docs/package-management.md b/docs/package-management.md index 268523bc35..d455ff8571 100644 --- a/docs/package-management.md +++ b/docs/package-management.md @@ -32,7 +32,7 @@ Most readers want one of these. Detailed sections follow further down. | Pull pre-built binaries without compiling | [`scripts/fetch-binaries.sh`](#release-archives) — walks every `package.toml`, calls the resolver. Run with `--allow-stale` in CI. | | Add a new package to the registry | [Schema: `package.toml`](#schema-packagetoml) + [docs/porting-guide.md](porting-guide.md#adding-a-new-package-to-the-registry) for the end-to-end workflow. | | Resolve one package on demand | `cargo xtask build-deps resolve ` — handles fetch/source-build, populates the cache. | -| Find where an output lands | `cargo xtask build-deps output-path ` — single source of truth for the layout convention (flat for 1-output packages, nested under `/` for ≥2-output packages). | +| Find where an output lands | `cargo xtask build-deps output-path ` — single source of truth for the layout convention (flat for a one-member output/runtime closure, nested under `/` for two or more members). A basename is accepted only when unique. | | Migrate a build script to consume cached deps | [Migrating a consumer to the cache](#migrating-a-consumer-to-the-cache) — the `WASM_POSIX_DEP_*_DIR` contract + CPPFLAGS/LDFLAGS pattern. | | Override a published archive locally | Drop the file at `local-binaries/programs//` or `local-libs//build/`. The resolver prefers these. | | Override an archive in a PR for testing | Per-PR builds publish to `pr--staging` tags. Locally, run `./run.sh --pr-staging ` or set `WASM_POSIX_USE_PR_STAGING=1` so `run.sh` exports the matching staging `WASM_POSIX_BINARY_INDEX_URL`. Manual `WASM_POSIX_BINARY_INDEX_URL` values still win. | @@ -224,9 +224,16 @@ Build scripts install declared artifacts through `scripts/install-local-binary.sh`. When `WASM_POSIX_INSTALL_LOCAL_MIRROR=0` and `WASM_POSIX_DEP_OUT_DIR` are set, both executable outputs and runtime files are copied only into that -caller-owned output root. This packaging-only mode does not require Rust, -Cargo, or `xtask`; the resolver validates the completed output root against -`package.toml` after the build script returns. +caller-owned output root. The root is a single-writer scratch contract for the +duration of each install: the current user must own it and every destination +directory created or traversed within it, and group/other users may not write +those directories. The packaging-only shell helper enforces those observable +properties, publishes through a private create-once transaction, and compares +filesystem identity plus exact bytes before cleanup. It does not claim general +shared-directory race freedom; that would require dirfd/openat-style operations +rather than shell pathname commands. This mode does not require Rust, Cargo, or +`xtask`; the resolver validates the completed output root against `package.toml` +after the build script returns. Repo-side VFS/test builders query the authoritative path and mode with `xtask build-deps runtime-file-metadata `; they must not @@ -240,9 +247,163 @@ a complete fetched package, but local, fetched, and installed-package tiers are never combined. If artifacts exist but no tier has the complete accepted closure, resolution fails loudly. +The host resolver applies the same rule automatically when any member of a +program package with more than one total `[[outputs]]` plus +`[[runtime_files]]` entry is requested. This includes a package with one +executable and one runtime archive: both paths move under the package directory +and form one closure. Rust's complete TOML parser generates +`packages/registry/program-packages.json`, a closed, versioned projection of +package identity, target arches, exact source artifacts, resolver mirror paths, +fork policy, and runtime-file metadata. TypeScript and shell consumers read +that JSON; neither reparses `package.toml`. + +Schema `kandelo-program-packages-v2` records every library, program, and source +package's manifest SHA-256 and cache key in both wasm32 and wasm64 consumer +contexts. A source package has the same cache key in both contexts because its +identity is architecture-independent. Each projected program also records its +complete transitive selected dependency identity for every supported +architecture. The host resolver, standalone shell bundle, and browser scanner +therefore validate the same first-hit registry context before accepting a +program mirror. Dependency order has no selection meaning and names must be +unique. The Rust generator emits a deterministic order so the checked-in JSON +is reproducible and CI can detect stale projections. Runtime consumers compare +the closure as a unique package-identity set, so reordering the same entries +does not change the accepted program identity. + +An index describes the complete ordered registry context beginning at its +owning root, not only packages physically stored in that root. Its `identities` +map covers every first-hit package, and its `packages` map covers every +first-hit guest program across that root and all lower roots. The generator +reads each selected program's physical manifest for member policy, then +recomputes its cache key and dependency closure in the complete context. A +dependency-only external override can therefore rekey affected main-registry +programs without copying their manifests into the external root. The +highest-priority existing root's index is authoritative to consumers; lower +indexes are self-contained suffix-context fallbacks when higher roots are +absent, not fragments that consumers merge. + +The program map covers guest programs published under +`binaries/programs//`. A higher first-hit non-program package naturally +removes a same-named lower program from that map, while the lower physical +index retains a fail-closed claim against stale flat mirror fallback. The map +deliberately excludes Kandelo's first-party `kernel` and `userspace` boot +artifacts: their single outputs publish at `binaries/kernel.wasm` and +`binaries/userspace.wasm`, and retain their existing root-artifact ABI and +export validation instead of pretending to be guest program packages. Their +package identities remain in the all-package identity map so +dependency-context validation stays complete. + +Generate or verify an index with: + +```bash +cargo xtask build-deps program-index \ + /program-packages.json +cargo xtask build-deps program-index-check \ + /program-packages.json +cargo xtask build-deps program-index-context-check +``` + +The root passed to `program-index` or `program-index-check` must be the +highest-priority existing root in `WASM_POSIX_DEPS_REGISTRY`. Generate a lower +root's committed fallback with the registry suffix beginning at that root. +`build-deps check` verifies every present index against its own suffix context. +`program-index-context-check` is the stricter consumer boundary: it skips +nonexistent optional roots, requires an index for every existing configured +root, and validates each in its exact suffix context. Source-checkout program +resolution runs that Rust check before consuming generated policy. +The standalone `wasm-posix-host` package ships the same projection under +`wasm/`, so installed consumers retain closure and fork policy without carrying +source manifests. + +`scripts/resolve-binary.sh` runs a checked-in standalone Node bundle generated +from the same TypeScript resolver, so clean checkouts do not need +`node_modules` merely to probe fetched artifacts. After changing the resolver +or its policy dependencies, regenerate and verify that bundle with +`scripts/build-resolve-binary-bundle.sh` and +`scripts/test-resolve-binary-bundle.sh`. +For program paths the wrapper incrementally builds the current release xtask +and exports its path as `WASM_POSIX_XTASK_BIN`. A direct caller may provide that +override, but doing so is an attestation that the executable was prepared from +the current source; the resolver deliberately does not rebuild an explicit +caller-owned tool path. + +A registry directory without a regular `package.toml` is an ordinary +non-package path, matching Rust's lookup. A regular manifest is a first-hit +claim on that package name. If that selected package has no contextual +identity, a selected program has no program projection, or either manifest +digest is stale, package-owned lookup fails. Undeclared +nested members and the former flat spelling of a package-owned output are +errors too; they never fall through to scalar lookup. This also rejects stale +nested paths after a package changes from multi-member to scalar. All public +resolver entries reject +absolute, backslash, drive-prefixed, empty-component, `.`/`..`, and NUL path +spellings. `tryResolveBinary` returns `null` only for genuine absence and +rethrows corruption, policy rejection, and malformed package state. + +Tier membership alone is not a package identity. Every selected symlink under +the mutable `local-binaries/` or `binaries/` mirrors must resolve through its +declared source-artifact suffix into one approved generation root. A fetched +generation root must be a canonically named direct child of the program-cache +namespace; a local generation root must be a direct child of that package's +hidden local-generation namespace. A preexisting directory whose links point +at different or arbitrary roots is rejected even when every mirror path +exists. A versioned installed host package is one immutable installation +identity, so a complete +all-regular-file closure under its `wasm/` tree remains supported; +regular-file/symlink mixtures and installed symlink closures are rejected. + +Changing a package from a multi-member directory layout to a scalar flat path +does not delete the former package directory. The generated projection makes +that stale nested spelling inert immediately, while retaining the directory +avoids guessing whether another publisher or a user owns it. This safe +transition can leave stale mirror directories for explicit operational +cleanup; automatic ownership-marked retirement is deferred. An extensionless +scalar output whose exact flat name is that retained directory will fail +closed until the directory is deliberately retired. + +Normal direct builds collect package closures under +`local-binaries/.kandelo-local-generations/////`, +using the exact declared source suffix for each member. One sourced install +helper shares a session across its calls. The collector accepts only +create-once regular files, validates the complete tree, creates a one-shot +publication claim, and only then swaps the live package directory or scalar +link. A claimed generation is never recreated after its root disappears. +One-member packages retain their flat mirror name as a symlink to the +immutable generation member. + +Scalar replacement and package-directory replacement reserve a unique private +transaction parent (mode 0700 on Unix), validate filesystem identity and exact +bytes or link maps before and after quarantine, and delete only unchanged +private entries. Scalar rollback uses an atomic no-replace rename, so a writer +that appears after a failed publication or quarantine check remains the winner; +changed or unvalidated quarantined state is preserved privately rather than +overwriting that winner. Concurrent publishers must replace resolver paths through +this pathname transaction; mutating an already quarantined regular file +through a previously held file descriptor is outside the supported writer +protocol and causes digest validation to fail whenever it is observed before +unlink. + +For symlink-backed closures the host returns canonical generation-member paths, +not live mirror paths, so a later mirror-directory swap cannot retarget an +already resolved string. Local claimed generations are append-only unless a +user manually removes both resolver-owned state and backing bytes. Fetched +cache repair is a narrower boundary: force-source rebuild and stale-entry +recovery may remove and recreate the same cache-key directory, and the resolver +does not support that operation concurrently with same-package consumers. A +path retained across such repair can temporarily disappear or name replacement +bytes. See [Binary releases](binary-releases.md) for producer replacement, +rollback, crash-orphan, cache-repair, and platform boundaries. + Build scripts register executable outputs with `install_local_binary` and declared data with `install_local_runtime_file`. Normal local builds mirror -both into `local-binaries/`. A sealed publisher instead sets +both into `local-binaries/`. A normal executable install performs one +structured Rust lookup for the destination, exact declared artifact, and fork +policy before instrumentation or filesystem mutation. It never guesses a path +for an unregistered or malformed package. Package publication does not create +a second `sh` resolver output; guest images own their explicit `/bin/sh` +symlink to the shell they include. + +A sealed publisher instead sets `WASM_POSIX_INSTALL_LOCAL_MIRROR=0`, provides `WASM_POSIX_DEP_OUT_DIR`, and supplies the reviewed fork-instrumentation policy for executable outputs. In that mode the helpers copy the exact declared @@ -294,6 +455,14 @@ index_url = "https://github.com/Automattic/kandelo/releases/download/binaries-ab package-system import-closure test enforces that relationship for derived images. Type-only modules elided by `tsx` are not inferred, but an authored schema that belongs in artifact provenance may be declared explicitly. + Canonical `packages/registry//...` inputs select the first registry + root containing that package's `package.toml`, then require every declared + file below that one selected package directory. A partial external shadow + fails instead of filling missing files from a lower package generation, and + a stray higher directory without `package.toml` cannot shadow main. The + first-party non-package helper trees below `packages/registry` remain + main-checkout inputs. Existing registry-relative input spellings apply the + same package-level rule for third-party sources. - `repo_url` + `commit` record the project's recipe provenance. - `revision` is the publish-time counter the resolver hashes into the cache-key. Bump when output bytes legitimately change (build @@ -516,8 +685,13 @@ in turn, it checks: validate declared outputs, atomically install into the canonical cache. -`cache_root` is `$XDG_CACHE_HOME/kandelo` if set, else -`$HOME/.cache/kandelo`. +`cache_root` is `WASM_POSIX_BINARY_CACHE_ROOT` when set, otherwise +`$XDG_CACHE_HOME/kandelo` or `$HOME/.cache/kandelo`. Rust, TypeScript, the +standalone resolver bundle, and Vite share this override. Absolute values are +used directly; relative values are anchored at the Kandelo repository root so +different process working directories cannot silently select different +caches. Installed npm consumers without a Kandelo source root must use an +absolute value. ## Build-script contract @@ -526,7 +700,7 @@ that doesn't respect them cannot be cached safely. | Variable | Meaning | |---|---| -| `WASM_POSIX_DEP_OUT_DIR` | Temp dir the script must install into. Layout matches `outputs.libs` / `outputs.headers` / `outputs.pkgconfig` / `outputs.files` relative paths. | +| `WASM_POSIX_DEP_OUT_DIR` | Caller-owned, single-writer temp dir the script must install into. The current user owns it and each destination directory traversed within it; none is group/other-writable. Layout matches `outputs.libs` / `outputs.headers` / `outputs.pkgconfig` / `outputs.files` relative paths. | | `WASM_POSIX_DEP_NAME` | `name` from package.toml. | | `WASM_POSIX_DEP_VERSION` | `version` from package.toml. | | `WASM_POSIX_DEP_REVISION` | Effective package revision after `build.toml` is overlaid. | @@ -931,6 +1105,12 @@ continue through ordinary resolution. For example, the main-shell proof uses it to guarantee that the shell composer runs while its reviewed Homebrew bottles remain ordinary immutable inputs. +When `archive-stage --cache-root ` also uses `--binaries-dir`, later +processes consuming that symlink mirror must set +`WASM_POSIX_BINARY_CACHE_ROOT=`. This couples the mirror's canonical +targets to the same non-default cache in Rust, Node, shell, and browser +consumers instead of rejecting a valid generation as foreign. + Each matrix entry then publishes via `scripts/index-update.sh`: ```bash @@ -1233,6 +1413,14 @@ The second notices the canonical path exists and discards its own temp dir. Identical inputs yield identical outputs, so keeping either copy is correct. +This race rule covers creation of a previously absent cache key. Maintenance +that deliberately removes an existing key—force-source rebuild or stale-cache +repair—uses the resolver's existing no-concurrent-same-package assumption. +Consumers must not retain or read canonical member paths concurrently with +that maintenance because the directory can be absent and then recreated under +the same pathname. Live mirror publication remains atomic; this boundary is +about maintenance of the backing cache itself. + A crashed build (process killed mid-script) leaves its `.tmp-/` behind. The next resolve of the same key starts a fresh temp with a new pid — no conflict — and the leftover is harmless until manually @@ -1250,7 +1438,20 @@ WASM_POSIX_DEPS_REGISTRY="./packages/registry:~/my-wasm-packages" \ Colon-separated. First hit wins — later entries have lower priority, like `$PATH`. This is how third parties bring their own packages without patching the repo: they drop a `/package.toml` into their -own directory tree and prepend it to the registry path. +own directory tree, generate that root's `program-packages.json`, and prepend +it to the registry path. Rust builds, TypeScript resolution, and +`scripts/resolve-binary.sh` use the same exact roots and first-hit package +selection. + +Generate the external root's index against `external:main`. That complete top +index includes external programs and every selected lower program, each with +the exact combined first-hit cache key and dependency closure. An identical +higher-priority shadow leaves identities unchanged. A changed direct or +transitive dependency rekeys only the affected programs; those programs remain +eligible for normal exact-key fetch or source build instead of reusing +main-only bytes. Consumers do not synthesize or merge policy: they verify the +complete top projection. If the external root is absent, the main root's +committed suffix-context index becomes authoritative again. The first external package source using this pattern is [`brandonpayton/kandelo-software`](https://github.com/brandonpayton/kandelo-software): diff --git a/docs/package-sources.md b/docs/package-sources.md index 4be93eebb4..b4baab0518 100644 --- a/docs/package-sources.md +++ b/docs/package-sources.md @@ -27,6 +27,7 @@ README.md packages.txt gallery.json # optional browser-gallery metadata packages/ + program-packages.json # Rust-generated runtime projection / package.toml # portable package recipe build.toml # this source's publish/index state @@ -38,6 +39,45 @@ packages/ line. Blank lines and `#` comments are ignored by `scripts/publish-package-source.sh`. +If the source is used directly through `WASM_POSIX_DEPS_REGISTRY`, generate +`packages/program-packages.json` with Kandelo's authoritative parser and the +same ordered registry roots that consumers will use: + +```bash +WASM_POSIX_DEPS_REGISTRY="$PWD/packages:/path/to/kandelo/packages/registry" \ + cargo xtask build-deps program-index packages packages/program-packages.json +``` + +Validate all existing roots exactly as a source consumer will see them: + +```bash +WASM_POSIX_DEPS_REGISTRY="$PWD/packages:/path/to/kandelo/packages/registry" \ + cargo xtask build-deps program-index-context-check +``` + +Commit the result beside the package directories. Runtime consumers require it +to preserve exact first-hit output closures, per-architecture cache keys, and +fork policy without maintaining a second TOML parser. The projection also binds +each program to the identities of its complete transitive dependency closure +in that registry order. The reusable publication workflow checks this +projection before building, so a changed recipe or dependency cannot be +published with stale runtime identity. +Kandelo source checkouts run the same contextual Rust check before every public +program resolution. An existing configured root without an index is an error; +nonexistent optional roots are skipped. Installed host packages instead consume +the projection that Kandelo verified and copied at package-build time. + +The external index is a complete `external:main` projection. It contains +identities for every first-hit package and projections for every selected +program, including programs whose physical manifests remain in the main +registry. Identical shadows keep the same identity. A changed direct or +transitive external dependency gives each affected lower program a newly +computed combined-context cache key and closure, so normal resolution can fetch +or build that exact generation without copying the program recipe into the +external repository. Consumers use this highest-priority complete index; they +never merge it with lower policy. The main registry's own index remains a +self-contained fallback when the external root is not configured or present. + Use Kandelo's current package layout in new recipes: ```toml diff --git a/docs/plans/2026-07-21-homebrew-migration-execution-plan.md b/docs/plans/2026-07-21-homebrew-migration-execution-plan.md index cbbc95308e..47fbb65e83 100644 --- a/docs/plans/2026-07-21-homebrew-migration-execution-plan.md +++ b/docs/plans/2026-07-21-homebrew-migration-execution-plan.md @@ -1,7 +1,7 @@ # Homebrew Migration Living Execution Plan - Status: active -- Last reconciled: 2026-07-22 +- Last reconciled: 2026-07-23 - Primary repositories: `Automattic/kandelo` and `Kandelo-dev/homebrew-tap-core` - Purpose: preserve the complete Homebrew migration scope, record what has @@ -194,6 +194,8 @@ complete here only when its exact accepted artifact has been verified. | Language bottles | Public publication complete; immutable revision-18 candidate and exact Node.js/Chromium language acceptance green | Ruby is public and runtime-verified. Coordinated run `29886510272` built, publicly uploaded, anonymously verified, and atomically finalized Python `3.13.3_1` and Erlang `28.2_1` at tap commit `00ba350ffcee7df02fb9f329bb3c62873ae50831`. Perl is published. The exact 5,885,691-byte revision-18 shell candidate and its 39 deferred bottles are bound to immutable release `homebrew-shell-bottles-sha256-b51c071bc0f5eabf230f10d26f8e6c397999323dfbf18cb6da11cec822f8c21b`. Anonymous public Node.js and Chromium acceptance start Python, Perl, Erlang, and Ruby lazily in isolated first-use steps. PR #1062 landed the bounded per-asset browser evidence ledger at `dc5bb1210f1359be17d1d4078a7d56ec14903e4a`; the remaining activation gate is the shared product-VFS headroom fix and its final exact regression run. | | Third-party tap model | Live publisher proof complete; guest use remains | The stricter load-order-independent cross-tap runtime contract landed in Kandelo as PR #1046 at `bd2b090e3e6998350be24ed018bbb76d3eb5b012`, in the core tap as PR #82 at `caad125218a2e3c6f05d290151a32128ec6c54ac`, and in the canary as PR #13 at `25069ad2acb7f86746ec3d119a823e8210a7a1eb`. PR #1049 landed the active-repository tap-store correction at `466a685d9366d3b712c4fe998307e00157bd5d15`; core-tap PR #83 pinned it at `cbb439454adf2718b010d0fe2caffe7158340a0e`, and canary PR #14 pinned it at `ee4464b87b988b163608b6c3520c2260907bda61`. Independent run `29886510154` is completely green: public M4 package and index, anonymous exact-byte pour, dependency-bearing Node.js and Chromium image proof, transactional tap finalization, and immutable five-asset VFS release `homebrew-vfs-sha256-40a44df5c6f139a4e9105b5155040be757bc20596dc5dce2d7a64286447d9f3e`. Conventional third-party `brew tap` and `brew install` inside the guest remain Phase 5 work. | | Deferred bottle trees | Generic producer and Phase 3 public proof complete; Phase 4 mirror public and relocation locally validated | PR #1051 landed the generic first-use substrate at `122e62a77ffeb40039bee3f2b29cd5f82ed6b1fe`. PR #1054 landed the exact original-bottle producer at `c16a48c693c8a6dea4ca14e7886b735bf685d51d`: one independently lazy tree per Formula, complete source and guest inventories, exact compressed transport identity, hardlinks, and independent TypeScript/Python validation. PR #1055 composes the exact 38-Formula namespace, PR #1060's exact head proves its immutable public mirror and canonical revision-17 cutover, and PR #1056 landed the aggregate-budget correction. The Phase 4 worktree adds receipt-owned relocation before exposing language runtimes, because exact bottle bytes are transport truth while a correct pour may replace only the placeholders named by that bottle's `INSTALL_RECEIPT.json`; its complete 39-bottle browser mirror is public and immutable. | +| Browser deployment and exact bottle delivery | Complete for the bounded current contract | PR #1064 landed bounded, single-writer Pages publication. PR #1070 landed exact browser bottle-download delivery. Production verification reached GitHub Pages commit `418bd04` through successful Pages run `29994147876`; the app, guide, API, and service worker returned HTTP 200. This evidence closes the observed deployment failure, but does not remove later Phase 4/5 product activation work. | +| Atomic package-generation foundation | Ready for PR and landing on the exact fixture-ownership baseline | The packaging/build worktree makes Rust-generated program policy, scalar mirrors, and multi-member mirror directories publish as validated atomic generations. It aligns Rust, TypeScript, shell, Vite, external registries, and the standalone npm package on one complete highest-priority registry projection, with self-contained lower-root fallbacks. Fixture-ownership PR #802 landed as `427185cff21ed213de8b8b6573b4f1a3757aa80d`; this active foundation is rebased on that exact commit, its `program-packages.json` was regenerated there, and a repository source audit now rejects obsolete flat package paths while preserving inventory-owned and direct-source test fixtures. Independent High/Medium review found no remaining blocker. Exact-baseline validation is green across all 507 xtask tests, all 105 package-system tests, host typechecking, Chromium/Firefox/WebKit Vite boundaries, package-root and sealed/local-generation contracts, resolver-bundle freshness, Pages/CI/merge-workflow contracts, and the 17-case Homebrew shell closure. This foundation does not by itself activate the Phase 4 shell candidate, guest `brew`, registry retirement, or bottle-declared VFS packages. | | Guest upstream `brew` | Stock tap and bottle-pour proof complete in the opt-in image; product lifecycle incomplete | Draft PR #1059 pins upstream Homebrew, gives its unprivileged guest state the conventional writable layout, and passes exact Node.js/Chromium startup, config, operational doctor, first-party tap, and independent third-party tap discovery. An unmodified stock Bzip2 install pours and runs the public bottle once Homebrew can resolve the exact 19-Formula metadata closure for publisher-only native dependencies. Full `homebrew/core` is infeasible in the guest (about 1.3 GiB, including a 1.22 GiB Git pack); the product fix is a separately reviewed allowlist of custom Homebrew `Requirement` classes, not a partial core tap or unsupported dependency bypass. Main-shell activation, install/reinstall/uninstall, durable reboot state, and cross-tap M4 installation remain. | | Registry replacement | Incomplete | Formulae are increasingly authoritative, but `packages/registry` still owns recipes, platform artifacts, tests, and composite-image definitions. It cannot be deleted yet. | | Bottle-declared, mix-and-match VFS packages | Future retained scope | The current composer produces precomposed images. VFS Formulae/bottles and user-selectable composition remain a later product iteration. | diff --git a/examples/run-example.ts b/examples/run-example.ts index 5f089bab5c..113adababb 100644 --- a/examples/run-example.ts +++ b/examples/run-example.ts @@ -54,7 +54,7 @@ const grepWasm = tryResolveBinary("programs/grep.wasm"); const sedWasm = tryResolveBinary("programs/sed.wasm"); const gitWasm = tryResolveBinary("programs/git/git.wasm"); const bcWasm = tryResolveBinary("programs/bc.wasm"); -const fileWasm = tryResolveBinary("programs/file.wasm"); +const fileWasm = tryResolveBinary("programs/file/file.wasm"); const lessWasm = tryResolveBinary("programs/less.wasm"); const m4Wasm = tryResolveBinary("programs/m4.wasm"); const makeWasm = tryResolveBinary("programs/make.wasm"); @@ -71,7 +71,7 @@ const nodeWasm = tryResolveBinary("programs/node.wasm") ?? tryResolveBinary("programs/spidermonkey-node.wasm"); const lsofWasm = resolve(repoRoot, "examples/lsof.wasm"); -const rubyWasm = tryResolveBinary("programs/ruby.wasm"); +const rubyWasm = tryResolveBinary("programs/ruby/ruby.wasm"); const vimWasm = tryResolveBinary("programs/vim.zip"); const gawkWasm = tryResolveBinary("programs/gawk.wasm"); const findWasm = tryResolveBinary("programs/findutils/find.wasm"); @@ -83,7 +83,11 @@ const diff3Wasm = tryResolveBinary("programs/diffutils/diff3.wasm"); const perlWasm = tryResolveBinary("programs/perl.wasm"); const nanoWasm = tryResolveBinary("programs/nano.wasm"); const tclshWasm = tryResolveBinary("programs/tcl.wasm"); -const testfixtureWasm = tryResolveBinary("programs/sqlite/testfixture.wasm"); +const testfixtureBuild = resolve( + repoRoot, + "packages/registry/sqlite/bin/testfixture.wasm", +); +const testfixtureWasm = existsSync(testfixtureBuild) ? testfixtureBuild : null; const mysqltestWasm = tryResolveBinary("programs/mariadb/mysqltest.wasm"); const echoWasm = tryResolveBinary("programs/echo.wasm") ?? resolve(repoRoot, "examples/echo.wasm"); diff --git a/host/src/binary-resolver.ts b/host/src/binary-resolver.ts index a1e9435c9d..6367207036 100644 --- a/host/src/binary-resolver.ts +++ b/host/src/binary-resolver.ts @@ -14,8 +14,25 @@ * See `docs/binary-releases.md` for the layout. */ -import { existsSync, readdirSync, readFileSync } from "node:fs"; -import { dirname, join, resolve } from "node:path"; +import { + existsSync, + lstatSync, + readdirSync, + readFileSync, + realpathSync, + statSync, +} from "node:fs"; +import { createHash } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { + basename, + dirname, + isAbsolute, + join, + relative, + resolve, + sep, +} from "node:path"; import { fileURLToPath } from "node:url"; import { describeWasmArtifactPolicyFailures } from "./constants"; import { @@ -46,14 +63,18 @@ function currentModuleDir(): string { function isRepoRoot(dir: string): boolean { // Workspace Cargo.toml has a [workspace] table; nested crate - // Cargo.tomls do not. Cheap check that disambiguates without - // having to read+parse every Cargo.toml on the way up. + // Cargo.tomls do not. The package identity matters too: an installed host + // package may live below an unrelated consumer's Cargo/npm workspace, which + // must not be mistaken for a Kandelo source checkout. const cargo = join(dir, "Cargo.toml"); - if (!existsSync(cargo) || !existsSync(join(dir, "package.json"))) { + const packageJson = join(dir, "package.json"); + if (!existsSync(cargo) || !existsSync(packageJson)) { return false; } try { - return /^\s*\[workspace\]/m.test(readFileSync(cargo, "utf8")); + const packageIdentity = JSON.parse(readFileSync(packageJson, "utf8")); + return /^\s*\[workspace\]/m.test(readFileSync(cargo, "utf8")) + && packageIdentity?.name === "kandelo"; } catch { return false; } @@ -77,10 +98,77 @@ export function findRepoRoot(startFrom?: string): string { ); } +function resolverRepoRoot(): string { + const explicitStart = process.env.WASM_POSIX_BINARY_RESOLVER_REPO_ROOT; + return explicitStart ? findRepoRoot(explicitStart) : findRepoRoot(); +} + function packageRoot(): string { return resolve(currentModuleDir(), ".."); } +function hasSourceCheckout(): boolean { + try { + resolverRepoRoot(); + return true; + } catch { + return false; + } +} + +/** + * Cache root used by xtask for immutable package generations. + * + * `WASM_POSIX_BINARY_CACHE_ROOT` is the explicit cross-language override. It + * is required when an `archive-stage --cache-root` invocation also publishes + * symlinks into a binaries mirror that will be consumed in another process. + */ +export function binaryCacheRoot(): string { + const explicitCacheRoot = process.env.WASM_POSIX_BINARY_CACHE_ROOT; + if (explicitCacheRoot !== undefined) { + return isAbsolute(explicitCacheRoot) + ? resolve(explicitCacheRoot) + : resolve(resolverRepoRoot(), explicitCacheRoot); + } + const xdgCacheHome = process.env.XDG_CACHE_HOME; + if (xdgCacheHome !== undefined) { + return resolve(xdgCacheHome, "kandelo"); + } + const home = process.env.HOME; + if (home !== undefined) { + return resolve(home, ".cache", "kandelo"); + } + return "/tmp/kandelo"; +} + +/** Only program generations are valid external targets for browser assets. */ +export function binaryProgramCacheRoot(): string { + return join(binaryCacheRoot(), "programs"); +} + +/** + * Resolver paths are a portable, slash-separated namespace, not host paths. + * Reject aliases instead of normalizing them: closure discovery and tier + * lookup must receive exactly the same spelling or a path such as `pkg/../pkg` + * could bypass package-level identity checks before `node:path.join` collapses + * it back onto a declared member. + */ +function requirePortableResolverPath(relPath: string): string { + if ( + relPath.length === 0 + || relPath.startsWith("/") + || /^[A-Za-z]:/.test(relPath) + || relPath.includes("\\") + || relPath.includes("\0") + || relPath.split("/").some((part) => !part || part === "." || part === "..") + ) { + throw new Error( + `Binary resolver path must be a normalized portable relative path: ${JSON.stringify(relPath)}`, + ); + } + return relPath; +} + /** * Resolve an artifact relative to the binaries tree. * @@ -104,6 +192,7 @@ function packageRoot(): string { const ARCH_SEGMENTS = new Set(["wasm32", "wasm64"]); function applyDefaultArch(relPath: string): string { + requirePortableResolverPath(relPath); if (!relPath.startsWith("programs/")) return relPath; const tail = relPath.slice("programs/".length); const firstSeg = tail.split("/", 1)[0]; @@ -130,9 +219,23 @@ function packagedBinaryCandidates( interface BinaryCandidateTier { label: string; root: string; + identity: "local-generation" | "program-cache" | "installed-package"; + /** + * A genuine installed npm package is one versioned installation identity. + * A source checkout's host/wasm tree is mutable and never qualifies. + */ + allowRegularFileClosure: boolean; candidatesFor(relPath: string): string[]; } +/** A genuine absence, as distinct from present-but-invalid package state. */ +export class BinaryNotFoundError extends Error { + constructor(message: string) { + super(message); + this.name = "BinaryNotFoundError"; + } +} + /** * Ordered provenance roots used by both single-artifact and package-closure * resolution. Keeping the grouping explicit lets a closure fall back as a @@ -140,8 +243,10 @@ interface BinaryCandidateTier { */ function binaryCandidateTiers(): BinaryCandidateTier[] { const tiers: BinaryCandidateTier[] = []; + let sourceCheckout = false; try { - const repo = findRepoRoot(); + const repo = resolverRepoRoot(); + sourceCheckout = true; for (const [label, root] of [ ["local-binaries", join(repo, "local-binaries")], ["binaries", join(repo, "binaries")], @@ -149,6 +254,10 @@ function binaryCandidateTiers(): BinaryCandidateTier[] { tiers.push({ label, root, + identity: label === "local-binaries" + ? "local-generation" + : "program-cache", + allowRegularFileClosure: false, candidatesFor(relPath: string): string[] { return [join(root, applyDefaultArch(relPath))]; }, @@ -162,6 +271,8 @@ function binaryCandidateTiers(): BinaryCandidateTier[] { tiers.push({ label: "installed package", root, + identity: "installed-package", + allowRegularFileClosure: !sourceCheckout, candidatesFor(relPath: string): string[] { return packagedBinaryCandidates(relPath, root); }, @@ -169,92 +280,1359 @@ function binaryCandidateTiers(): BinaryCandidateTier[] { return tiers; } -let cachedForkInstrumentationDisabledOutputs: Set | null = null; +interface ProgramPackageClosureMember { + packageName: string; + relPath: string; + sourceArtifact: string; + cacheKey: string; + forkInstrumentation: "auto" | "disabled" | null; + projectionIdentity: string; +} -interface ProgramOutputPolicy { - name?: string; - wasm?: string; - forkInstrumentation?: string; +interface ProgramPackageClosure { + manifestPath: string; + packageName: string; + members: ProgramPackageClosureMember[]; } -function outputExtension(wasmPath: string): string { - const basename = wasmPath.split(/[\\/]/).pop() ?? wasmPath; - const dot = basename.indexOf("."); - return dot >= 0 ? basename.slice(dot) : ""; +function manifestError(manifestPath: string, detail: string): Error { + return new Error(`Invalid package manifest ${manifestPath}: ${detail}`); } -function outputRelForPackage( - packageName: string, - output: Required>, - outputCount: number, +/** Presence check that does not turn a dangling symlink into absence. */ +function pathEntryExists(path: string): boolean { + try { + lstatSync(path); + return true; + } catch (error) { + if ( + error instanceof Error + && "code" in error + && error.code === "ENOENT" + ) return false; + throw error; + } +} + +function portableArtifactPath( + value: string, + manifestPath: string, + field: string, +): string { + if ( + value.length === 0 + || value.startsWith("/") + || value.includes("\\") + || value.includes("\0") + || value.split("/").some((part) => !part || part === "." || part === "..") + ) { + throw manifestError( + manifestPath, + `${field} must be a normalized portable relative path`, + ); + } + return value; +} + +function safeSinglePathComponent( + value: string, + manifestPath: string, + field: string, + allowAt = true, ): string { - const destName = `${output.name}${outputExtension(output.wasm)}`; - return outputCount > 1 ? `${packageName}/${destName}` : destName; + if ( + value.length === 0 + || value === "." + || value === ".." + || value.includes("/") + || value.includes("\\") + || value.includes("\0") + || (!allowAt && value.includes("@")) + ) { + throw manifestError( + manifestPath, + `${field} must be a safe single path component`, + ); + } + return value; +} + +interface LegacyFlatOutputOwner { + scalarOwners: Set; + packagePaths: Map; + shadowedOwners: Set; } -function parseProgramOutputPolicies(packageToml: string): { - kind?: string; - name?: string; - outputs: ProgramOutputPolicy[]; -} { - const kind = packageToml.match(/^kind\s*=\s*"([^"]+)"/m)?.[1]; - const name = packageToml.match(/^name\s*=\s*"([^"]+)"/m)?.[1]; - const outputs: ProgramOutputPolicy[] = []; - let current: ProgramOutputPolicy | null = null; +interface ProgramRegistryIndex { + legacyFlatOutputs: Map; + forkInstrumentationDisabledOutputs: Map; + identities: Map; + unidentifiedPackages: Map; + packages: Map; + unprojectedPackages: Map; +} - for (const line of packageToml.split(/\r?\n/)) { - const trimmed = line.trim(); - if (trimmed === "[[outputs]]") { - if (current) outputs.push(current); - current = {}; - continue; +interface ProgramPackageProjectionMember { + kind: "output" | "runtime-file"; + sourceArtifact: string; + mirrorPath: string; + outputName?: string; + forkInstrumentation?: "auto" | "disabled"; + guestPath?: string; + mode?: number; +} + +interface ProgramPackageProjection { + manifestSha256: string; + arches: string[]; + cacheKeys: Record; + dependencyClosures: Record; + members: ProgramPackageProjectionMember[]; +} + +interface ProgramDependencyIdentity { + packageName: string; + manifestSha256: string; + cacheKey: string; +} + +interface PackageProjectionIdentity { + manifestSha256: string; + cacheKeys: Record; +} + +interface SelectedPackageProjectionIdentity extends PackageProjectionIdentity { + packageName: string; + policyPath: string; + manifestPath?: string; +} + +interface SelectedProgramPackageProjection extends ProgramPackageProjection { + packageName: string; + policyPath: string; + manifestPath?: string; +} + +interface LoadedProgramPackageProjection { + identities: Map; + packages: Map; + indexPath: string; +} + +interface PhysicalProgramProjectionClaim { + packageName: string; + projection: ProgramPackageProjection; + selected: boolean; +} + +interface SelectedProgramPackageState { + identities: Map; + unidentifiedPackages: Map; + packages: Map; + unprojectedPackages: Map; + physicalProgramClaims: PhysicalProgramProjectionClaim[]; +} + +const PROGRAM_PACKAGE_INDEX_FORMAT = "kandelo-program-packages-v2"; +const PROGRAM_PACKAGE_INDEX_FILE = "program-packages.json"; + +type ProgramIndexContextChecker = ( + sourceRepoRoot: string, + registryRoots: readonly string[], +) => void; + +let programIndexContextCheckerForTests: ProgramIndexContextChecker | null = null; +let preparedProgramIndexChecker: + | { sourceRepoRoot: string; xtaskPath: string } + | null = null; +let programIndexFreshnessBoundaryDepth = 0; + +/** + * @internal Compatibility hook for test fixtures. + * + * Registry policy is deliberately uncached. Package directories and generated + * projections can change while Vite or a long-lived Node process is running; + * stale negative cache entries would let a newly package-owned nested path + * fall through to scalar resolution. + */ +export function resetBinaryResolverManifestCacheForTests(): void { + // No-op by design. +} + +/** + * @internal Test-only substitution for the Rust source-freshness boundary. + * + * Tests that author deliberately synthetic projections can replace the + * external process while still asserting when and with which ordered roots + * the boundary runs. Production callers always execute xtask's canonical + * manifest/cache-key implementation. + */ +export function setProgramIndexContextCheckerForTests( + checker: ProgramIndexContextChecker | null, +): void { + programIndexContextCheckerForTests = checker; +} + +function configuredProgramRegistryRoots(): string[] | null { + if (Object.prototype.hasOwnProperty.call( + process.env, + "WASM_POSIX_DEPS_REGISTRY", + )) { + let sourceRepoRoot: string | null = null; + return (process.env.WASM_POSIX_DEPS_REGISTRY ?? "") + .split(":") + .filter(Boolean) + .map((entry) => { + if (entry.startsWith("~/") && process.env.HOME !== undefined) { + return join(process.env.HOME, entry.slice(2)); + } + if (isAbsolute(entry)) return resolve(entry); + sourceRepoRoot ??= resolverRepoRoot(); + return resolve(sourceRepoRoot, entry); + }); + } + try { + return [join(resolverRepoRoot(), "packages", "registry")]; + } catch { + return null; + } +} + +function completeSourceCheckoutRoot(): string | null { + let sourceRepoRoot: string; + try { + sourceRepoRoot = resolverRepoRoot(); + } catch { + return null; + } + if ( + !existsSync(join(sourceRepoRoot, "tools", "xtask", "Cargo.toml")) + || !existsSync(join(sourceRepoRoot, "scripts", "dev-shell.sh")) + ) return null; + + // An installed npm package can live below node_modules in an unrelated + // Kandelo source checkout. Only source-owned module locations may execute + // that checkout's policy checker: host/ for the shared TypeScript module, + // and scripts/ for the generated standalone resolver bundle. + try { + const sourceModuleDir = realpathSync(currentModuleDir()); + const realRepoRoot = realpathSync(sourceRepoRoot); + const sourceOwned = [ + join(realRepoRoot, "host"), + join(realRepoRoot, "scripts"), + ].some((ownedRoot) => { + return existsSync(ownedRoot) + && pathIsWithin(realpathSync(ownedRoot), sourceModuleDir) + }); + return sourceOwned ? realRepoRoot : null; + } catch { + return null; + } +} + +function commandFailure( + command: string, + args: readonly string[], + result: ReturnType, +): string { + const detail = [ + typeof result.stderr === "string" ? result.stderr.trim() : "", + typeof result.stdout === "string" ? result.stdout.trim() : "", + result.error?.message ?? "", + ].filter(Boolean).join("\n"); + return `${command} ${args.join(" ")} failed${ + result.status === null ? "" : ` with status ${result.status}` + }${detail ? `:\n${detail}` : ""}`; +} + +function rustHostTarget(sourceRepoRoot: string): string { + const inDevShell = process.env.KANDELO_DEV_SHELL_TOOL_PATH !== undefined; + const command = inDevShell ? "rustc" : "bash"; + const args = inDevShell + ? ["-vV"] + : [join(sourceRepoRoot, "scripts", "dev-shell.sh"), "rustc", "-vV"]; + const result = spawnSync(command, args, { + cwd: sourceRepoRoot, + encoding: "utf8", + }); + if (result.status !== 0) { + throw new Error(commandFailure(command, args, result)); + } + const host = result.stdout + .split(/\r?\n/) + .find((line) => line.startsWith("host: ")) + ?.slice("host: ".length) + .trim(); + if (!host) { + throw new Error( + `Could not determine the Rust host target for ${sourceRepoRoot}`, + ); + } + return host; +} + +function requireRegularXtask(path: string): string { + try { + if (lstatSync(path).isFile()) return realpathSync(path); + } catch { + // The caller below reports the complete preparation failure. + } + throw new Error(`Prepared xtask is not a regular file: ${path}`); +} + +function prepareProgramIndexChecker(sourceRepoRoot: string): string { + const explicit = process.env.WASM_POSIX_XTASK_BIN; + if (explicit !== undefined) { + const explicitPath = isAbsolute(explicit) + ? resolve(explicit) + : resolve(sourceRepoRoot, explicit); + return requireRegularXtask(explicitPath); + } + if (preparedProgramIndexChecker?.sourceRepoRoot === sourceRepoRoot) { + return requireRegularXtask(preparedProgramIndexChecker.xtaskPath); + } + + const host = rustHostTarget(sourceRepoRoot); + const xtaskPath = join( + sourceRepoRoot, + "target", + host, + "release", + process.platform === "win32" ? "xtask.exe" : "xtask", + ); + // A path left by an earlier checkout state is not evidence that it contains + // the current checker. Cargo's incremental no-op is the preparation + // contract; pay it once per long-lived resolver process, then execute the + // resulting binary at every public source-policy boundary. + const cargoArgs = [ + "build", + "--release", + "-p", + "xtask", + "--target", + host, + "--quiet", + ]; + const inDevShell = process.env.KANDELO_DEV_SHELL_TOOL_PATH !== undefined; + const command = inDevShell ? "cargo" : "bash"; + const args = inDevShell + ? cargoArgs + : [join(sourceRepoRoot, "scripts", "dev-shell.sh"), "cargo", ...cargoArgs]; + const result = spawnSync(command, args, { + cwd: sourceRepoRoot, + encoding: "utf8", + }); + if (result.status !== 0) { + throw new Error(commandFailure(command, args, result)); + } + preparedProgramIndexChecker = { + sourceRepoRoot, + xtaskPath: requireRegularXtask(xtaskPath), + }; + return preparedProgramIndexChecker.xtaskPath; +} + +function checkProgramIndexesInSourceContext(): void { + const sourceRepoRoot = completeSourceCheckoutRoot(); + if (sourceRepoRoot === null) return; + const registryRoots = configuredProgramRegistryRoots(); + if (registryRoots === null) return; + if (programIndexContextCheckerForTests) { + programIndexContextCheckerForTests(sourceRepoRoot, registryRoots); + return; + } + + const xtaskPath = prepareProgramIndexChecker(sourceRepoRoot); + const args = ["build-deps", "program-index-context-check"]; + const result = spawnSync(xtaskPath, args, { + cwd: sourceRepoRoot, + encoding: "utf8", + env: { + ...process.env, + WASM_POSIX_DEPS_REGISTRY: registryRoots.join(":"), + }, + }); + if (result.status !== 0) { + throw new Error( + `Program package source projection is not current:\n${ + commandFailure(xtaskPath, args, result) + }`, + ); + } +} + +function withFreshProgramIndexes( + relPaths: readonly string[], + operation: () => T, +): T { + if ( + programIndexFreshnessBoundaryDepth > 0 + || !relPaths.some((relPath) => relPath.startsWith("programs/")) + ) { + return operation(); + } + programIndexFreshnessBoundaryDepth += 1; + try { + checkProgramIndexesInSourceContext(); + return operation(); + } finally { + programIndexFreshnessBoundaryDepth -= 1; + } +} + +function hasExactObjectKeys( + value: object, + expectedKeys: readonly string[], +): boolean { + const actual = Object.keys(value).sort(); + const expected = [...expectedKeys].sort(); + return actual.length === expected.length + && actual.every((key, index) => key === expected[index]); +} + +function readProgramPackageProjection( + indexPath: string, +): LoadedProgramPackageProjection { + let raw: unknown; + try { + raw = JSON.parse(readFileSync(indexPath, "utf8")); + } catch (error) { + throw new Error( + `Invalid program package index ${indexPath}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + if ( + typeof raw !== "object" + || raw === null + || !hasExactObjectKeys(raw, ["format", "identities", "packages"]) + || (raw as { format?: unknown }).format !== PROGRAM_PACKAGE_INDEX_FORMAT + || typeof (raw as { identities?: unknown }).identities !== "object" + || (raw as { identities?: unknown }).identities === null + || Array.isArray((raw as { identities?: unknown }).identities) + || typeof (raw as { packages?: unknown }).packages !== "object" + || (raw as { packages?: unknown }).packages === null + || Array.isArray((raw as { packages?: unknown }).packages) + ) { + throw new Error( + `Invalid program package index ${indexPath}: expected ${PROGRAM_PACKAGE_INDEX_FORMAT}`, + ); + } + + const identities = new Map(); + const rawIdentities = (raw as { + identities: Record; + }).identities; + for (const [packageName, rawIdentity] of Object.entries(rawIdentities)) { + safeSinglePathComponent(packageName, indexPath, "identity package name", false); + if ( + typeof rawIdentity !== "object" + || rawIdentity === null + || !hasExactObjectKeys(rawIdentity, ["manifestSha256", "cacheKeys"]) + || typeof (rawIdentity as { manifestSha256?: unknown }).manifestSha256 + !== "string" + || !/^[a-f0-9]{64}$/.test( + (rawIdentity as { manifestSha256: string }).manifestSha256, + ) + || typeof (rawIdentity as { cacheKeys?: unknown }).cacheKeys !== "object" + || (rawIdentity as { cacheKeys?: unknown }).cacheKeys === null + || Array.isArray((rawIdentity as { cacheKeys?: unknown }).cacheKeys) + ) { + throw new Error( + `Invalid program package index ${indexPath}: malformed identity ${JSON.stringify(packageName)}`, + ); + } + const cacheKeys = (rawIdentity as { + cacheKeys: Record; + }).cacheKeys; + if ( + !hasExactObjectKeys(cacheKeys, ["wasm32", "wasm64"]) + || Object.values(cacheKeys).some( + (cacheKey) => + typeof cacheKey !== "string" || !/^[a-f0-9]{64}$/.test(cacheKey), + ) + ) { + throw new Error( + `Invalid program package index ${indexPath}: identity ${JSON.stringify(packageName)} has invalid contextual cache keys`, + ); + } + identities.set(packageName, { + manifestSha256: (rawIdentity as { manifestSha256: string }).manifestSha256, + cacheKeys: cacheKeys as Record, + }); + } + + const packages = new Map(); + const rawPackages = (raw as { packages: Record }).packages; + for (const [packageName, rawPackage] of Object.entries(rawPackages)) { + safeSinglePathComponent(packageName, indexPath, "package name", false); + if ( + typeof rawPackage !== "object" + || rawPackage === null + || !hasExactObjectKeys(rawPackage, [ + "manifestSha256", + "arches", + "cacheKeys", + "dependencyClosures", + "members", + ]) + || !Array.isArray((rawPackage as { arches?: unknown }).arches) + || typeof (rawPackage as { cacheKeys?: unknown }).cacheKeys !== "object" + || (rawPackage as { cacheKeys?: unknown }).cacheKeys === null + || Array.isArray((rawPackage as { cacheKeys?: unknown }).cacheKeys) + || typeof (rawPackage as { dependencyClosures?: unknown }) + .dependencyClosures !== "object" + || (rawPackage as { dependencyClosures?: unknown }) + .dependencyClosures === null + || Array.isArray( + (rawPackage as { dependencyClosures?: unknown }).dependencyClosures, + ) + || !Array.isArray((rawPackage as { members?: unknown }).members) + || typeof (rawPackage as { manifestSha256?: unknown }).manifestSha256 + !== "string" + || !/^[a-f0-9]{64}$/.test( + (rawPackage as { manifestSha256: string }).manifestSha256, + ) + ) { + throw new Error( + `Invalid program package index ${indexPath}: malformed package ${JSON.stringify(packageName)}`, + ); + } + const arches = (rawPackage as { arches: unknown[] }).arches; + if ( + arches.length === 0 + || new Set(arches).size !== arches.length + || arches.some((arch) => typeof arch !== "string" || !ARCH_SEGMENTS.has(arch)) + ) { + throw new Error( + `Invalid program package index ${indexPath}: package ${JSON.stringify(packageName)} has invalid arches`, + ); + } + const cacheKeys = (rawPackage as { + cacheKeys: Record; + }).cacheKeys; + if ( + !hasExactObjectKeys(cacheKeys, arches as string[]) + || Object.values(cacheKeys).some( + (cacheKey) => + typeof cacheKey !== "string" || !/^[a-f0-9]{64}$/.test(cacheKey), + ) + ) { + throw new Error( + `Invalid program package index ${indexPath}: package ${JSON.stringify(packageName)} has invalid cache keys`, + ); + } + const dependencyClosures = (rawPackage as { + dependencyClosures: Record; + }).dependencyClosures; + if (!hasExactObjectKeys(dependencyClosures, arches as string[])) { + throw new Error( + `Invalid program package index ${indexPath}: package ${JSON.stringify(packageName)} has invalid dependency closure arches`, + ); + } + const parsedDependencyClosures: Record = + {}; + for (const arch of arches as string[]) { + const rawClosure = dependencyClosures[arch]; + if (!Array.isArray(rawClosure)) { + throw new Error( + `Invalid program package index ${indexPath}: package ${JSON.stringify(packageName)} has a malformed dependency closure for ${arch}`, + ); + } + // Dependency order is deliberately non-semantic. Rust emits a stable + // order for reproducible checked-in JSON, while consumers require and + // compare a unique package-identity set. + const seenDependencies = new Set(); + parsedDependencyClosures[arch] = rawClosure.map( + (rawDependency, dependencyIndex): ProgramDependencyIdentity => { + if ( + typeof rawDependency !== "object" + || rawDependency === null + || !hasExactObjectKeys(rawDependency, [ + "packageName", + "manifestSha256", + "cacheKey", + ]) + || typeof (rawDependency as { packageName?: unknown }).packageName + !== "string" + || typeof (rawDependency as { manifestSha256?: unknown }) + .manifestSha256 !== "string" + || !/^[a-f0-9]{64}$/.test( + (rawDependency as { manifestSha256: string }).manifestSha256, + ) + || typeof (rawDependency as { cacheKey?: unknown }).cacheKey + !== "string" + || !/^[a-f0-9]{64}$/.test( + (rawDependency as { cacheKey: string }).cacheKey, + ) + ) { + throw new Error( + `Invalid program package index ${indexPath}: package ${JSON.stringify(packageName)} dependency ${dependencyIndex + 1} for ${arch} is malformed`, + ); + } + const dependency = rawDependency as unknown as ProgramDependencyIdentity; + safeSinglePathComponent( + dependency.packageName, + indexPath, + `${packageName} dependency packageName`, + false, + ); + if ( + dependency.packageName === packageName + || seenDependencies.has(dependency.packageName) + ) { + throw new Error( + `Invalid program package index ${indexPath}: package ${JSON.stringify(packageName)} dependency closure for ${arch} must contain unique dependencies other than itself`, + ); + } + seenDependencies.add(dependency.packageName); + const contextualIdentity = identities.get(dependency.packageName); + if ( + !contextualIdentity + || contextualIdentity.manifestSha256 + !== dependency.manifestSha256 + || contextualIdentity.cacheKeys[arch] !== dependency.cacheKey + ) { + throw new Error( + `Invalid program package index ${indexPath}: package ${JSON.stringify(packageName)} dependency ${JSON.stringify(dependency.packageName)} for ${arch} does not match the index's authoritative contextual identity`, + ); + } + return dependency; + }, + ); + } + const members = (rawPackage as { members: unknown[] }).members.map( + (rawMember, memberIndex): ProgramPackageProjectionMember => { + if ( + typeof rawMember !== "object" + || rawMember === null + || ( + (rawMember as { kind?: unknown }).kind !== "output" + && (rawMember as { kind?: unknown }).kind !== "runtime-file" + ) + || typeof (rawMember as { sourceArtifact?: unknown }).sourceArtifact + !== "string" + || typeof (rawMember as { mirrorPath?: unknown }).mirrorPath !== "string" + ) { + throw new Error( + `Invalid program package index ${indexPath}: package ${JSON.stringify(packageName)} member ${memberIndex + 1} is malformed`, + ); + } + const member = rawMember as Record; + const expectedMemberKeys = member.kind === "output" + ? [ + "kind", + "sourceArtifact", + "mirrorPath", + "outputName", + "forkInstrumentation", + ] + : [ + "kind", + "sourceArtifact", + "mirrorPath", + "guestPath", + "mode", + ]; + if (!hasExactObjectKeys(member, expectedMemberKeys)) { + throw new Error( + `Invalid program package index ${indexPath}: package ${JSON.stringify(packageName)} member ${memberIndex + 1} has unknown or missing fields`, + ); + } + portableArtifactPath( + member.sourceArtifact as string, + indexPath, + `${packageName} sourceArtifact`, + ); + portableArtifactPath( + member.mirrorPath as string, + indexPath, + `${packageName} mirrorPath`, + ); + if (member.kind === "output") { + if ( + typeof member.outputName !== "string" + || ( + member.forkInstrumentation !== "auto" + && member.forkInstrumentation !== "disabled" + ) + ) { + throw new Error( + `Invalid program package index ${indexPath}: package ${JSON.stringify(packageName)} output member lacks outputName or forkInstrumentation`, + ); + } + safeSinglePathComponent( + member.outputName, + indexPath, + `${packageName} outputName`, + ); + } else if ( + typeof member.guestPath !== "string" + || !member.guestPath.startsWith("/") + || !Number.isInteger(member.mode) + || (member.mode as number) < 0 + || (member.mode as number) > 0o777 + ) { + throw new Error( + `Invalid program package index ${indexPath}: package ${JSON.stringify(packageName)} runtime member lacks valid guestPath or mode`, + ); + } + return member as unknown as ProgramPackageProjectionMember; + }, + ); + if ( + members.length === 0 + || new Set(members.map((member) => member.sourceArtifact)).size + !== members.length + || new Set(members.map((member) => member.mirrorPath)).size + !== members.length + || ( + members.length === 1 + && members[0]!.mirrorPath.includes("/") + ) + || ( + members.length > 1 + && members.some( + (member) => !member.mirrorPath.startsWith(`${packageName}/`), + ) + ) + ) { + throw new Error( + `Invalid program package index ${indexPath}: package ${JSON.stringify(packageName)} members are empty, collide, or violate scalar/package-directory layout`, + ); } - if (!current) continue; - if (trimmed.startsWith("[") && trimmed !== "[[outputs]]") { - outputs.push(current); - current = null; - continue; + const manifestSha256 = + (rawPackage as { manifestSha256: string }).manifestSha256; + const identity = identities.get(packageName); + if ( + !identity + || identity.manifestSha256 !== manifestSha256 + || (arches as string[]).some( + (arch) => identity.cacheKeys[arch] !== cacheKeys[arch], + ) + ) { + throw new Error( + `Invalid program package index ${indexPath}: package ${JSON.stringify(packageName)} does not match its contextual package identity`, + ); } - const match = trimmed.match(/^([A-Za-z0-9_-]+)\s*=\s*"([^"]*)"/); - if (!match) continue; - if (match[1] === "name") current.name = match[2]; - if (match[1] === "wasm") current.wasm = match[2]; - if (match[1] === "fork_instrumentation") current.forkInstrumentation = match[2]; + packages.set(packageName, { + manifestSha256, + arches: arches as string[], + cacheKeys: cacheKeys as Record, + dependencyClosures: parsedDependencyClosures, + members, + }); } - if (current) outputs.push(current); + return { identities, packages, indexPath }; +} + +function programPackageProjectionIdentity( + projection: ProgramPackageProjection, +): string { + return JSON.stringify({ + manifestSha256: projection.manifestSha256, + arches: projection.arches, + cacheKeys: Object.fromEntries( + projection.arches.map((arch) => [arch, projection.cacheKeys[arch]]), + ), + dependencyClosures: Object.fromEntries( + projection.arches.map((arch) => [ + arch, + [...projection.dependencyClosures[arch]!].sort((left, right) => + left.packageName < right.packageName + ? -1 + : left.packageName > right.packageName + ? 1 + : 0 + ), + ]), + ), + members: projection.members.map((member) => + member.kind === "output" + ? { + kind: member.kind, + sourceArtifact: member.sourceArtifact, + mirrorPath: member.mirrorPath, + outputName: member.outputName, + forkInstrumentation: member.forkInstrumentation, + } + : { + kind: member.kind, + sourceArtifact: member.sourceArtifact, + mirrorPath: member.mirrorPath, + guestPath: member.guestPath, + mode: member.mode, + } + ), + }); +} - return { kind, name, outputs }; +function bundledProgramPackageProjection(): LoadedProgramPackageProjection | null { + const indexPath = join(packageRoot(), "wasm", PROGRAM_PACKAGE_INDEX_FILE); + return pathEntryExists(indexPath) + ? readProgramPackageProjection(indexPath) + : null; +} + +/** + * An explicit registry remains the authored policy source. The projection + * shipped beside installed bytes is independent identity evidence, but its + * namespaces must not fall through to generic scalar lookup merely because a + * custom registry omitted them. + */ +function bundledProgramClaimForPath(adjusted: string): string | null { + const loaded = bundledProgramPackageProjection(); + if (!loaded) return null; + const components = adjusted.split("/"); + if ( + components[0] !== "programs" + || !ARCH_SEGMENTS.has(components[1]!) + ) return null; + const arch = components[1]!; + if (components.length >= 4) { + const packageName = components[2]!; + const projection = loaded.packages.get(packageName); + return projection?.arches.includes(arch) ? packageName : null; + } + if (components.length !== 3) return null; + const flatName = components[2]!; + for (const [packageName, projection] of loaded.packages) { + if ( + projection.arches.includes(arch) + && projection.members.some( + (member) => + member.kind === "output" + && member.mirrorPath.split("/").at(-1) === flatName, + ) + ) return packageName; + } + return null; +} + +function rejectUnselectedBundledProgramClaim(adjusted: string): void { + const packageName = bundledProgramClaimForPath(adjusted); + if (packageName) { + throw new Error( + `Installed package resolver path ${JSON.stringify(adjusted)} is owned by ` + + `${JSON.stringify(packageName)}, but that package is not selected by ` + + `the configured program registry`, + ); + } +} + +function selectedProgramPackageState(): SelectedProgramPackageState { + const roots = configuredProgramRegistryRoots(); + const identities = new Map(); + const unidentifiedPackages = new Map(); + const packages = new Map(); + const unprojectedPackages = new Map(); + const physicalProgramClaims: PhysicalProgramProjectionClaim[] = []; + + if (roots === null) { + const indexPath = join(packageRoot(), "wasm", PROGRAM_PACKAGE_INDEX_FILE); + if (!pathEntryExists(indexPath)) { + return { + identities, + unidentifiedPackages, + packages, + unprojectedPackages, + physicalProgramClaims, + }; + } + const loaded = readProgramPackageProjection(indexPath); + for (const [packageName, identity] of loaded.identities) { + identities.set(packageName, { + ...identity, + packageName, + policyPath: `${loaded.indexPath}#identities.${packageName}`, + }); + } + for (const [packageName, projection] of loaded.packages) { + physicalProgramClaims.push({ + packageName, + projection, + selected: true, + }); + packages.set(packageName, { + ...projection, + packageName, + policyPath: `${loaded.indexPath}#${packageName}`, + }); + } + return { + identities, + unidentifiedPackages, + packages, + unprojectedPackages, + physicalProgramClaims, + }; + } + + const claimed = new Set(); + let authoritativeIdentities: + | Map + | null = null; + let authoritativePackages: + | Map + | null = null; + for (const root of roots) { + if (!pathEntryExists(root)) continue; + if (!statSync(root).isDirectory()) { + throw new Error(`Program registry root is not a directory: ${root}`); + } + const indexPath = join(root, PROGRAM_PACKAGE_INDEX_FILE); + if (!pathEntryExists(indexPath)) { + throw new Error( + `Program registry ${root} is missing ${PROGRAM_PACKAGE_INDEX_FILE}; generate it with xtask build-deps program-index`, + ); + } + const loaded = readProgramPackageProjection(indexPath); + // The highest-priority existing root was generated against the complete + // ordered registry path. It owns both contextual identities and program + // projections for every first-hit package, including lower-root programs + // rekeyed by a dependency-only override. Lower indexes retain suffix- + // context projections as fallbacks and namespace evidence only. + authoritativeIdentities ??= loaded.identities; + authoritativePackages ??= loaded.packages; + const entries = readdirSync(root, { withFileTypes: true }) + .filter((entry) => entry.isDirectory() || entry.isSymbolicLink()) + .sort((left, right) => left.name.localeCompare(right.name)); + for (const entry of entries) { + const packageName = entry.name; + const manifestPath = join(root, packageName, "package.toml"); + if (!pathEntryExists(manifestPath)) continue; + let isFile = false; + try { + isFile = statSync(manifestPath).isFile(); + } catch { + isFile = false; + } + if (!isFile) continue; + const physicalProjection = loaded.packages.get(packageName); + const selected = !claimed.has(packageName); + if (physicalProjection) { + physicalProgramClaims.push({ + packageName, + projection: physicalProjection, + selected, + }); + } + if (!selected) continue; + claimed.add(packageName); + const identity = authoritativeIdentities.get(packageName); + if (identity) { + identities.set(packageName, { + ...identity, + packageName, + manifestPath, + policyPath: manifestPath, + }); + } else { + unidentifiedPackages.set(packageName, manifestPath); + } + const projection = authoritativePackages.get(packageName); + if (!projection) { + unprojectedPackages.set(packageName, manifestPath); + continue; + } + packages.set(packageName, { + ...projection, + packageName, + manifestPath, + policyPath: manifestPath, + }); + } + } + return { + identities, + unidentifiedPackages, + packages, + unprojectedPackages, + physicalProgramClaims, + }; } -function forkInstrumentationDisabledOutputs(): Set { - if (cachedForkInstrumentationDisabledOutputs) { - return cachedForkInstrumentationDisabledOutputs; +function verifySelectedPackageIdentity( + identity: SelectedPackageProjectionIdentity, +): void { + if (!identity.manifestPath) return; + let bytes: Buffer; + try { + bytes = readFileSync(identity.manifestPath); + } catch (error) { + throw new Error( + `Program package identity cannot verify ${identity.manifestPath}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); } + const digest = createHash("sha256").update(bytes).digest("hex"); + if (digest !== identity.manifestSha256) { + throw new Error( + `Program package identity is stale for ${identity.manifestPath}; regenerate ${PROGRAM_PACKAGE_INDEX_FILE}`, + ); + } +} - const disabled = new Set(); +function verifySelectedProgramPackage( + projection: SelectedProgramPackageProjection, +): void { + if (!projection.manifestPath) return; + let bytes: Buffer; try { - const registry = join(findRepoRoot(), "packages", "registry"); - for (const entry of readdirSync(registry, { withFileTypes: true })) { - if (!entry.isDirectory()) continue; - const manifestPath = join(registry, entry.name, "package.toml"); - if (!existsSync(manifestPath)) continue; - const parsed = parseProgramOutputPolicies(readFileSync(manifestPath, "utf8")); - if (parsed.kind !== "program" || !parsed.name) continue; - const completeOutputs = parsed.outputs.filter( - (out): out is Required> & ProgramOutputPolicy => - Boolean(out.name && out.wasm), + bytes = readFileSync(projection.manifestPath); + } catch (error) { + throw new Error( + `Program package projection cannot verify ${projection.manifestPath}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + const digest = createHash("sha256").update(bytes).digest("hex"); + if (digest !== projection.manifestSha256) { + throw new Error( + `Program package projection is stale for ${projection.manifestPath}; regenerate ${PROGRAM_PACKAGE_INDEX_FILE}`, + ); + } +} + +function selectedProgramPackage( + packageName: string, +): SelectedProgramPackageProjection | null { + const index = programRegistryIndex(); + const projection = index.packages.get(packageName); + if (projection) { + verifySelectedProgramPackage(projection); + return projection; + } + const manifestPath = index.unprojectedPackages.get(packageName); + if (manifestPath) { + throw new Error( + `Package ${JSON.stringify(packageName)} is selected at ${manifestPath} but is absent from ${PROGRAM_PACKAGE_INDEX_FILE}; regenerate the registry projection`, + ); + } + return null; +} + +function verifyProgramDependencyContext( + projection: SelectedProgramPackageProjection, + arch: string, +): void { + const expectedDependencies = projection.dependencyClosures[arch]; + if (!expectedDependencies) { + throw manifestError( + projection.policyPath, + `package ${JSON.stringify(projection.packageName)} lacks a dependency identity closure for ${arch}`, + ); + } + const state = selectedProgramPackageState(); + const selectedProgramIdentity = state.identities.get(projection.packageName); + if (!selectedProgramIdentity) { + const unidentified = state.unidentifiedPackages.get(projection.packageName); + throw new Error( + `Program package ${JSON.stringify(projection.packageName)} has no authoritative ` + + `contextual identity for ${arch}${ + unidentified ? ` at ${unidentified}` : "" + }; regenerate ${PROGRAM_PACKAGE_INDEX_FILE} with the exact ordered registry roots`, + ); + } + verifySelectedPackageIdentity(selectedProgramIdentity); + const selectedProgramCacheKey = selectedProgramIdentity.cacheKeys[arch]; + if ( + selectedProgramIdentity.manifestSha256 !== projection.manifestSha256 + || selectedProgramCacheKey !== projection.cacheKeys[arch] + ) { + throw new Error( + `Program package ${JSON.stringify(projection.packageName)} was projected with ` + + `manifest ${projection.manifestSha256} and cache key ${projection.cacheKeys[arch]}` + + ` for ${arch}, but the authoritative first-hit registry context at ` + + `${selectedProgramIdentity.policyPath} requires manifest ` + + `${selectedProgramIdentity.manifestSha256} and cache key ` + + `${selectedProgramCacheKey ?? ""}. Regenerate this program projection ` + + `with the exact ordered registry roots; the highest-priority index must ` + + `carry the complete combined-context projection rather than relying on ` + + `a lower suffix-context build identity.`, + ); + } + for (const expected of expectedDependencies) { + const selected = state.identities.get(expected.packageName); + if (!selected) { + const unidentified = state.unidentifiedPackages.get(expected.packageName); + if (unidentified) { + throw new Error( + `Program package ${JSON.stringify(projection.packageName)} was generated against ` + + `dependency ${JSON.stringify(expected.packageName)}, but the first-hit package at ` + + `${unidentified} has no contextual identity in ${PROGRAM_PACKAGE_INDEX_FILE}`, + ); + } + throw new Error( + `Program package ${JSON.stringify(projection.packageName)} was generated against ` + + `dependency ${JSON.stringify(expected.packageName)}, but that dependency is absent ` + + `from the configured first-hit registry roots`, + ); + } + verifySelectedPackageIdentity(selected); + const selectedCacheKey = selected.cacheKeys[arch]; + if ( + selected.manifestSha256 !== expected.manifestSha256 + || selectedCacheKey !== expected.cacheKey + ) { + throw new Error( + `Program package ${JSON.stringify(projection.packageName)} has a contextual cache ` + + `identity mismatch for ${arch}: its projection expects dependency ` + + `${JSON.stringify(expected.packageName)} manifest ${expected.manifestSha256} ` + + `and cache key ${expected.cacheKey}, but first-hit selection at ` + + `${selected.policyPath} provides manifest ${selected.manifestSha256} and cache key ` + + `${selectedCacheKey ?? ""}. Regenerate the program projection with the ` + + `exact ordered registry roots; the complete highest-priority projection must ` + + `bind every selected program to the same combined dependency context.`, ); - for (const output of completeOutputs) { - if (output.forkInstrumentation !== "disabled") continue; - disabled.add(outputRelForPackage(parsed.name, output, completeOutputs.length)); + } + } +} + +function programRegistryIndex(): ProgramRegistryIndex { + const state = selectedProgramPackageState(); + const { physicalProgramClaims, ...selectedState } = state; + const index: ProgramRegistryIndex = { + ...selectedState, + legacyFlatOutputs: new Map(), + forkInstrumentationDisabledOutputs: new Map(), + }; + const resolverPaths: Array<{ + arch: string; + path: string; + packageName: string; + }> = []; + for (const projection of state.packages.values()) { + const packageOwned = projection.members.length > 1; + for (const arch of projection.arches) { + for (const member of projection.members) { + const conflict = resolverPaths.find( + (previous) => + previous.arch === arch + && ( + previous.path === member.mirrorPath + || previous.path.startsWith(`${member.mirrorPath}/`) + || member.mirrorPath.startsWith(`${previous.path}/`) + ), + ); + if (conflict) { + throw new Error( + `Program resolver paths programs/${arch}/${conflict.path} and programs/${arch}/${member.mirrorPath} conflict between selected packages ${JSON.stringify(conflict.packageName)} and ${JSON.stringify(projection.packageName)}`, + ); + } + resolverPaths.push({ + arch, + path: member.mirrorPath, + packageName: projection.packageName, + }); + if (member.kind !== "output") continue; + const flatPath = member.mirrorPath.split("/").at(-1)!; + const key = `${arch}/${flatPath}`; + let owner = index.legacyFlatOutputs.get(key); + if (!owner) { + owner = { + scalarOwners: new Set(), + packagePaths: new Map(), + shadowedOwners: new Set(), + }; + index.legacyFlatOutputs.set(key, owner); + } + if (packageOwned) { + owner.packagePaths.set( + `programs/${arch}/${member.mirrorPath}`, + projection.packageName, + ); + } else { + owner.scalarOwners.add(projection.packageName); + } + if (member.forkInstrumentation === "disabled") { + index.forkInstrumentationDisabledOutputs.set( + `${arch}/${member.mirrorPath}`, + projection.packageName, + ); + } + } + } + } + // A lower physical program can be shadowed by a higher package of another + // kind (or by a different program layout). Its old flat mirror path must + // remain a fail-closed namespace claim; otherwise generic scalar lookup + // could serve stale bytes that bypass the actual first-hit package. A + // selected lower program reprojected by the authoritative top index already + // has active claims for the same manifest-owned members. + for ( + const { packageName, projection, selected } of physicalProgramClaims + ) { + if (selected && state.packages.has(packageName)) continue; + for (const arch of projection.arches) { + for (const member of projection.members) { + if (member.kind !== "output") continue; + const flatPath = member.mirrorPath.split("/").at(-1)!; + const key = `${arch}/${flatPath}`; + let owner = index.legacyFlatOutputs.get(key); + if (!owner) { + owner = { + scalarOwners: new Set(), + packagePaths: new Map(), + shadowedOwners: new Set(), + }; + index.legacyFlatOutputs.set(key, owner); + } + owner.shadowedOwners.add(packageName); } } - } catch { - // Installed package consumers do not carry registry manifests. } - cachedForkInstrumentationDisabledOutputs = disabled; - return disabled; + return index; +} + +/** + * Reject the former flat spelling of an output that now belongs to a + * multi-member package directory. Without this migration guard, a stale + * `programs//` symlink can enter scalar lookup and bypass the + * package closure. A flat spelling remains valid when a true single-member + * package owns the same output name. + */ +function selectedFlatProgramPackage( + adjusted: string, +): SelectedProgramPackageProjection | null { + const components = adjusted.split("/"); + if ( + components.length !== 3 + || components[0] !== "programs" + || !ARCH_SEGMENTS.has(components[1]!) + ) return null; + const owner = programRegistryIndex().legacyFlatOutputs.get( + `${components[1]}/${components[2]}`, + ); + if (!owner) return null; + for (const packageName of owner.scalarOwners) { + const projection = selectedProgramPackage(packageName); + if (projection) return projection; + } + for (const packageName of owner.packagePaths.values()) { + selectedProgramPackage(packageName); + } + if (owner.packagePaths.size > 0) { + throw new Error( + `Legacy flat resolver path ${JSON.stringify(adjusted)} belongs to a multi-member package; use ${[...owner.packagePaths.keys()].sort().map((path) => JSON.stringify(path)).join(" or ")}`, + ); + } + for (const packageName of owner.shadowedOwners) { + const projection = selectedProgramPackage(packageName); + if (projection) return projection; + throw new Error( + `Legacy flat resolver path ${JSON.stringify(adjusted)} is claimed by ` + + `a lower-root program package ${JSON.stringify(packageName)}, but its ` + + `first-hit selected package does not project that program; stale scalar ` + + `mirror fallback is forbidden`, + ); + } + return null; +} + +function closureForProjection( + projection: SelectedProgramPackageProjection, + arch: string, + adjusted: string, +): ProgramPackageClosure { + if (!projection.arches.includes(arch)) { + throw manifestError( + projection.policyPath, + `package ${JSON.stringify(projection.packageName)} does not declare resolver artifacts for ${arch}`, + ); + } + const cacheKey = projection.cacheKeys[arch]; + if (!cacheKey) { + throw manifestError( + projection.policyPath, + `package ${JSON.stringify(projection.packageName)} lacks a cache identity for ${arch}`, + ); + } + verifyProgramDependencyContext(projection, arch); + const projectionIdentity = programPackageProjectionIdentity(projection); + const members: ProgramPackageClosureMember[] = projection.members.map( + (member) => ({ + packageName: projection.packageName, + relPath: `programs/${arch}/${member.mirrorPath}`, + sourceArtifact: member.sourceArtifact, + cacheKey, + forkInstrumentation: member.kind === "output" + ? member.forkInstrumentation ?? null + : null, + projectionIdentity, + }), + ); + if (!members.some((member) => member.relPath === adjusted)) { + throw manifestError( + projection.policyPath, + `resolver path ${JSON.stringify(adjusted)} is not a declared member of package ${JSON.stringify(projection.packageName)}`, + ); + } + return { + manifestPath: projection.policyPath, + packageName: projection.packageName, + members, + }; +} + +function discoverProgramPackageClosure( + relPath: string, +): ProgramPackageClosure | null { + const adjusted = applyDefaultArch(relPath); + const components = adjusted.split("/"); + if ( + components[0] === "programs" + && !hasSourceCheckout() + && bundledProgramPackageProjection() === null + ) { + throw new Error( + `Installed host package is missing wasm/${PROGRAM_PACKAGE_INDEX_FILE}; ` + + `program artifacts cannot be resolved without packaged policy`, + ); + } + if (components.length === 3) { + const projection = selectedFlatProgramPackage(adjusted); + if (projection) { + return closureForProjection(projection, components[1]!, adjusted); + } + rejectUnselectedBundledProgramClaim(adjusted); + return null; + } + if ( + components.length < 4 + || components[0] !== "programs" + || !ARCH_SEGMENTS.has(components[1]!) + ) return null; + const arch = components[1]!; + const packageDirectory = components[2]!; + const projection = selectedProgramPackage(packageDirectory); + if (!projection) { + rejectUnselectedBundledProgramClaim(adjusted); + return null; + } + return closureForProjection(projection, arch, adjusted); +} + +/** + * Return every output and runtime file when `relPath` names a projected + * program package. Even a one-member package carries an exact selected cache + * identity; packages with multiple members additionally resolve as one + * all-or-nothing closure. + * + * An absent registry directory means the path is not package-owned. Once the + * directory exists, a missing, unreadable, or incomplete manifest is an error + * rather than permission to fall back to single-output resolution. + */ +export function programOutputClosureRelPaths(relPath: string): string[] | null { + return withFreshProgramIndexes([relPath], () => + discoverProgramPackageClosure(relPath)?.members.map( + (member) => member.relPath, + ) ?? null + ); } function stripProgramArch(relPath: string): string | null { @@ -266,8 +1644,19 @@ function stripProgramArch(relPath: string): string | null { } function disablesForkInstrumentation(relPath: string): boolean { - const programRel = stripProgramArch(relPath); - return programRel !== null && forkInstrumentationDisabledOutputs().has(programRel); + const adjusted = applyDefaultArch(relPath); + for (const arch of ARCH_SEGMENTS) { + const prefix = `programs/${arch}/`; + if (adjusted.startsWith(prefix)) { + const packageName = programRegistryIndex() + .forkInstrumentationDisabledOutputs.get( + `${arch}/${adjusted.slice(prefix.length)}`, + ); + if (!packageName) return false; + return selectedProgramPackage(packageName) !== null; + } + } + return false; } function requiredExportsForRelPath(relPath: string): readonly string[] | undefined { @@ -284,12 +1673,18 @@ function requiredExportsForRelPath(relPath: string): readonly string[] | undefin return undefined; } -function hasWasmArtifactPolicyFailures(path: string, relPath: string): boolean { +function hasWasmArtifactPolicyFailures( + path: string, + relPath: string, + capturedForkInstrumentation?: "auto" | "disabled" | null, +): boolean { if (!path.endsWith(".wasm")) return false; try { const bytes = readFileSync(path); const programBytes = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength); - const forkDisabled = disablesForkInstrumentation(relPath); + const forkDisabled = capturedForkInstrumentation === undefined + ? disablesForkInstrumentation(relPath) + : capturedForkInstrumentation === "disabled"; return describeWasmArtifactPolicyFailures(programBytes, { expectedAbi: ABI_VERSION, requiredExports: requiredExportsForRelPath(relPath), @@ -317,20 +1712,395 @@ function hasVfsArtifactPolicyFailures(path: string): boolean { } } -function hasBinaryArtifactPolicyFailures(path: string, relPath: string): boolean { - return hasWasmArtifactPolicyFailures(path, relPath) || +function hasBinaryArtifactPolicyFailures( + path: string, + relPath: string, + capturedForkInstrumentation?: "auto" | "disabled" | null, +): boolean { + return hasWasmArtifactPolicyFailures( + path, + relPath, + capturedForkInstrumentation, + ) || hasVfsArtifactPolicyFailures(path); } -function chooseBinaryCandidate(candidates: string[], relPath: string): string | null { - const existing = candidates.filter((candidate) => existsSync(candidate)); +function chooseBinaryCandidate( + candidates: string[], + relPath: string, + capturedForkInstrumentation?: "auto" | "disabled" | null, +): string | null { + const existing = candidates.filter(pathEntryExists); if (existing.length === 0) return null; - return existing.find((candidate) => !hasBinaryArtifactPolicyFailures(candidate, relPath)) ?? null; + return existing.find((candidate) => { + try { + return statSync(candidate).isFile() + && !hasBinaryArtifactPolicyFailures( + candidate, + relPath, + capturedForkInstrumentation, + ); + } catch { + return false; + } + }) ?? null; +} + +function pinScalarCandidate( + candidate: string, + relPath: string, + capturedForkInstrumentation?: "auto" | "disabled" | null, +): string { + try { + const metadata = lstatSync(candidate); + if (!metadata.isSymbolicLink()) return candidate; + const pinned = realpathSync(candidate); + if ( + !statSync(pinned).isFile() + || hasBinaryArtifactPolicyFailures( + pinned, + relPath, + capturedForkInstrumentation, + ) + ) { + throw new Error("canonical target is not an accepted regular file"); + } + if ( + applyDefaultArch(relPath).startsWith("programs/") + && isResolverOwnedProgramGenerationTarget(pinned) + ) { + throw new Error( + "resolver-owned program generation has no matching selected package projection", + ); + } + return pinned; + } catch (error) { + throw new Error( + `Binary changed or became invalid while pinning ${relPath}: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } +} + +function isResolverOwnedProgramGenerationTarget(path: string): boolean { + const roots = [binaryProgramCacheRoot()]; + try { + roots.push( + join( + resolverRepoRoot(), + "local-binaries", + ".kandelo-local-generations", + ), + ); + } catch { + // Installed consumers have no local source-build generation namespace. + } + return roots.some((root) => { + try { + return pathEntryExists(root) + && pathIsWithin(realpathSync(root), path); + } catch { + return false; + } + }); +} + +function pathIsWithin(root: string, path: string): boolean { + const pathFromRoot = relative(root, path); + return pathFromRoot === "" + || ( + pathFromRoot !== ".." + && !pathFromRoot.startsWith(`..${sep}`) + && !isAbsolute(pathFromRoot) + ); +} + +function canonicalRootForArtifact( + resolvedTarget: string, + sourceArtifact: string, +): string | null { + const parts = sourceArtifact.split("/"); + let root = resolvedTarget; + for (let index = 0; index < parts.length; index++) root = dirname(root); + return resolve(root, ...parts) === resolvedTarget ? root : null; +} + +function mutableGenerationIdentityFailure( + tier: BinaryCandidateTier, + sharedRoot: string, + members: readonly ProgramPackageClosureMember[], +): string | null { + const [programs, arch] = members[0]!.relPath.split("/"); + const packageName = members[0]!.packageName; + if ( + programs !== "programs" + || !ARCH_SEGMENTS.has(arch!) + || !packageName + || members.some((member) => member.packageName !== packageName) + ) return "declared package members do not share a valid program namespace"; + if (!statSync(sharedRoot).isDirectory()) { + return "shared package generation root is not a directory"; + } + const cacheKey = members[0]!.cacheKey; + if ( + !/^[a-f0-9]{64}$/.test(cacheKey) + || members.some((member) => member.cacheKey !== cacheKey) + ) { + return "declared package members do not share one valid cache identity"; + } + + if (tier.identity === "local-generation") { + const expectedParentPath = join( + tier.root, + ".kandelo-local-generations", + arch!, + packageName, + cacheKey, + ); + if (!pathEntryExists(expectedParentPath)) { + return "local mirror targets are not one direct immutable local generation"; + } + const expectedParent = realpathSync(expectedParentPath); + return dirname(sharedRoot) === expectedParent + ? null + : "local mirror targets are not one direct immutable local generation"; + } + if (tier.identity === "program-cache") { + const expectedParentPath = binaryProgramCacheRoot(); + if (!pathEntryExists(expectedParentPath)) { + return "fetched mirror targets are not one canonical program-cache generation"; + } + const expectedParent = realpathSync(expectedParentPath); + const generationName = basename(sharedRoot); + const hasCanonicalName = generationName.startsWith(`${packageName}-`) + && new RegExp(`-rev[0-9]+-${arch}-${cacheKey}$`).test(generationName); + return dirname(sharedRoot) === expectedParent && hasCanonicalName + ? null + : "fetched mirror targets are not one canonical program-cache generation"; + } + return "installed-package symlink closures are not an immutable installed identity"; +} + +interface PinnedPackageClosure { + paths: string[]; +} + +interface RejectedPackageClosure { + failure: string; +} + +/** + * Verify that a selected multi-member package is one generation and return + * canonical member paths inside that generation. Returning the mirror paths + * would reopen a time-of-check/time-of-use race: the live package directory + * can be atomically replaced after validation, changing what those strings + * name before a caller reads them. + */ +function pinPackageClosureIdentity( + tier: BinaryCandidateTier, + selected: readonly string[], + members: readonly ProgramPackageClosureMember[], +): PinnedPackageClosure | RejectedPackageClosure { + if (selected.length !== members.length) { + return { failure: "internal member/path count mismatch" }; + } + + try { + const linkKinds = selected.map((candidate) => { + const metadata = lstatSync(candidate); + if (metadata.isSymbolicLink()) return "symlink" as const; + if (metadata.isFile()) return "file" as const; + return "other" as const; + }); + if (linkKinds.includes("other")) { + return { + failure: "a selected mirror member is neither a regular file nor a symlink", + }; + } + const allSymlinks = linkKinds.every((kind) => kind === "symlink"); + const allFiles = linkKinds.every((kind) => kind === "file"); + if (!allSymlinks && !allFiles) { + return { + failure: "regular files and symlinks cannot share one package identity", + }; + } + + if (allFiles) { + if (!tier.allowRegularFileClosure) { + return { + failure: "a mutable source-checkout wasm tree is not an installed package identity", + }; + } + const packageName = members[0]!.packageName; + const projectionIdentity = members[0]!.projectionIdentity; + if ( + members.some( + (member) => + member.packageName !== packageName + || member.projectionIdentity !== projectionIdentity, + ) + ) { + return { + failure: "declared members do not share one selected package projection", + }; + } + const bundled = bundledProgramPackageProjection(); + const bundledProjection = bundled?.packages.get(packageName); + if ( + !bundledProjection + || programPackageProjectionIdentity(bundledProjection) + !== projectionIdentity + ) { + return { + failure: "installed bytes do not match the selected package projection", + }; + } + const installedRoot = realpathSync(tier.root); + const pinnedPaths: string[] = []; + for (const candidate of selected) { + const resolvedCandidate = realpathSync(candidate); + if ( + !pathIsWithin(installedRoot, resolvedCandidate) + || !statSync(resolvedCandidate).isFile() + ) { + return { + failure: "an installed-package member escapes its immutable wasm tree", + }; + } + pinnedPaths.push(resolvedCandidate); + } + return { paths: pinnedPaths }; + } + + let sharedRoot: string | null = null; + const pinnedPaths: string[] = []; + for (let index = 0; index < selected.length; index++) { + const resolvedTarget = realpathSync(selected[index]!); + if (!statSync(resolvedTarget).isFile()) { + return { + failure: `${members[index]!.relPath} does not resolve to a regular file`, + }; + } + const canonicalRoot = canonicalRootForArtifact( + resolvedTarget, + members[index]!.sourceArtifact, + ); + if (!canonicalRoot) { + return { + failure: `${members[index]!.relPath} does not target its declared source artifact ${members[index]!.sourceArtifact}`, + }; + } + if (sharedRoot === null) { + sharedRoot = canonicalRoot; + } else if (canonicalRoot !== sharedRoot) { + return { + failure: "member symlinks target different canonical package generations", + }; + } + pinnedPaths.push(resolvedTarget); + } + const generationFailure = mutableGenerationIdentityFailure( + tier, + sharedRoot!, + members, + ); + if (generationFailure) return { failure: generationFailure }; + return { paths: pinnedPaths }; + } catch (error) { + return { + failure: `cannot inspect package identity: ${ + error instanceof Error ? error.message : String(error) + }`, + }; + } +} + +function samePackageClosure( + left: ProgramPackageClosure, + right: ProgramPackageClosure, +): boolean { + if ( + left.manifestPath !== right.manifestPath + || left.members.length !== right.members.length + ) { + return false; + } + const rightByPath = new Map( + right.members.map((member) => [ + member.relPath, + `${member.packageName}\0${member.sourceArtifact}\0${member.cacheKey}\0${member.forkInstrumentation ?? ""}\0${member.projectionIdentity}`, + ]), + ); + return left.members.every( + (member) => + rightByPath.get(member.relPath) + === `${member.packageName}\0${member.sourceArtifact}\0${member.cacheKey}\0${member.forkInstrumentation ?? ""}\0${member.projectionIdentity}`, + ); +} + +function closureMembersForRequestedSet( + relPaths: readonly string[], +): ProgramPackageClosureMember[] | null { + let closure: ProgramPackageClosure | null = null; + for (const relPath of relPaths) { + const discovered = discoverProgramPackageClosure(relPath); + if (!discovered) continue; + if (closure && !samePackageClosure(closure, discovered)) { + throw new Error( + "A binary set cannot combine members from different package closures", + ); + } + closure = discovered; + } + if (!closure) return null; + + const requested = new Set(relPaths.map(applyDefaultArch)); + const declared = new Set(closure.members.map((member) => member.relPath)); + if ( + requested.size !== relPaths.length + || requested.size !== declared.size + || [...declared].some((relPath) => !requested.has(relPath)) + ) { + throw new Error( + `Package ${closure.packageName} must resolve its complete declared ` + + `closure: ${[...declared].join(", ")}`, + ); + } + const memberByPath = new Map( + closure.members.map((member) => [member.relPath, member]), + ); + return relPaths.map((relPath) => memberByPath.get(applyDefaultArch(relPath))!); } export function resolveBinary(relPath: string): string { + return withFreshProgramIndexes([relPath], () => + resolveBinaryInFreshProgramContext(relPath) + ); +} + +function resolveBinaryInFreshProgramContext(relPath: string): string { const adjusted = applyDefaultArch(relPath); + const packageClosure = discoverProgramPackageClosure(adjusted); + if (packageClosure) { + const selected = tryResolveBinarySetFromTiers( + packageClosure.members.map((member) => member.relPath), + packageClosure.members, + ); + if (selected) { + return selected[ + packageClosure.members.findIndex((member) => member.relPath === adjusted) + ]!; + } + // The package scan already checked every member in every tier. Never + // re-enter scalar lookup for a package-owned path: a concurrent publisher + // could otherwise create one member between the two scans and bypass the + // closure/identity contract. + throw new BinaryNotFoundError( + `Package artifacts not found for ${packageClosure.packageName}: ${adjusted}`, + ); + } const checked: string[] = []; const candidates: string[] = []; for (const tier of binaryCandidateTiers()) { @@ -340,8 +2110,14 @@ export function resolveBinary(relPath: string): string { } } const candidate = chooseBinaryCandidate(candidates, relPath); - if (candidate) return candidate; - throw new Error( + if (candidate) return pinScalarCandidate(candidate, relPath); + if (candidates.some(pathEntryExists)) { + throw new Error( + `Binary exists but was rejected by artifact policy: ${relPath}\n` + + checked.map((p) => ` checked: ${p}`).join("\n"), + ); + } + throw new BinaryNotFoundError( `Binary not found: ${relPath}\n` + checked.map((p) => ` checked: ${p}`).join("\n") + `\n Run scripts/fetch-binaries.sh, place a file at local-binaries/${adjusted}, or install a package that includes wasm/${relPath}.` @@ -355,8 +2131,9 @@ export function resolveBinary(relPath: string): string { export function tryResolveBinary(relPath: string): string | null { try { return resolveBinary(relPath); - } catch { - return null; + } catch (error) { + if (error instanceof BinaryNotFoundError) return null; + throw error; } } @@ -369,10 +2146,22 @@ export function tryResolveBinary(relPath: string): string | null { * silently composing a package from unrelated builds. It returns `null` only * when none of the requested artifacts exists in any tier. * - * Returned paths preserve `relPaths` order and are guaranteed to share the - * same local, fetched, or installed-package root. + * Returned paths preserve `relPaths` order and share one verified provenance + * identity. For symlink-backed package closures they are canonical generation + * member paths, not mutable live-mirror paths. The fetched cache still assumes + * no concurrent force-rebuild or stale-entry repair of the same cache key. */ export function tryResolveBinarySet(relPaths: readonly string[]): string[] | null { + return withFreshProgramIndexes(relPaths, () => { + const closureMembers = closureMembersForRequestedSet(relPaths); + return tryResolveBinarySetFromTiers(relPaths, closureMembers); + }); +} + +function tryResolveBinarySetFromTiers( + relPaths: readonly string[], + closureMembers: readonly ProgramPackageClosureMember[] | null, +): string[] | null { if (relPaths.length === 0) return []; let anyExisting = false; @@ -380,11 +2169,21 @@ export function tryResolveBinarySet(relPaths: readonly string[]): string[] | nul for (const tier of binaryCandidateTiers()) { const selected: string[] = []; const unavailable: string[] = []; - for (const relPath of relPaths) { + if (closureMembers) { + const [programs, arch, packageName] = closureMembers[0]!.relPath.split("/"); + if (programs === "programs" && arch && packageName) { + anyExisting ||= pathEntryExists(join(tier.root, programs, arch, packageName)); + } + } + for (const [index, relPath] of relPaths.entries()) { const candidates = tier.candidatesFor(relPath); - const existing = candidates.filter((candidate) => existsSync(candidate)); + const existing = candidates.filter(pathEntryExists); anyExisting ||= existing.length > 0; - const candidate = chooseBinaryCandidate(candidates, relPath); + const candidate = chooseBinaryCandidate( + candidates, + relPath, + closureMembers?.[index]?.forkInstrumentation, + ); if (candidate) { selected.push(candidate); } else if (existing.length > 0) { @@ -393,7 +2192,42 @@ export function tryResolveBinarySet(relPaths: readonly string[]): string[] | nul unavailable.push(`${relPath} (missing)`); } } - if (unavailable.length === 0) return selected; + if (unavailable.length === 0 && closureMembers) { + const identity = pinPackageClosureIdentity( + tier, + selected, + closureMembers, + ); + if ("failure" in identity) { + unavailable.push(`shared package identity rejected: ${identity.failure}`); + } else { + const rejectedPinnedMembers = identity.paths.flatMap((path, index) => + hasBinaryArtifactPolicyFailures( + path, + relPaths[index]!, + closureMembers[index]!.forkInstrumentation, + ) + ? [relPaths[index]!] + : [] + ); + if (rejectedPinnedMembers.length > 0) { + unavailable.push( + `pinned package generation rejected by artifact policy: ${rejectedPinnedMembers.join(", ")}`, + ); + } else { + return identity.paths; + } + } + } + if (unavailable.length === 0) { + return selected.map((path, index) => + pinScalarCandidate( + path, + relPaths[index]!, + closureMembers?.[index]?.forkInstrumentation, + ) + ); + } incomplete.push( ` ${tier.label} (${tier.root}): ${unavailable.join(", ")}`, ); @@ -409,10 +2243,10 @@ export function tryResolveBinarySet(relPaths: readonly string[]): string[] | nul /** Returns the absolute path of binaries/ whether or not it exists. */ export function binariesDir(): string { - return join(findRepoRoot(), "binaries"); + return join(resolverRepoRoot(), "binaries"); } /** Returns the absolute path of local-binaries/ whether or not it exists. */ export function localBinariesDir(): string { - return join(findRepoRoot(), "local-binaries"); + return join(resolverRepoRoot(), "local-binaries"); } diff --git a/host/src/index.ts b/host/src/index.ts index 0e75f59c8b..a0c755aae1 100644 --- a/host/src/index.ts +++ b/host/src/index.ts @@ -43,8 +43,10 @@ export type { } from "./worker-protocol"; export * from "./vfs/index"; export { + BinaryNotFoundError, resolveBinary, tryResolveBinary, + tryResolveBinarySet, findRepoRoot, binariesDir, localBinariesDir, diff --git a/host/test/binary-resolver.test.ts b/host/test/binary-resolver.test.ts index edbb0f113a..15c8f955d5 100644 --- a/host/test/binary-resolver.test.ts +++ b/host/test/binary-resolver.test.ts @@ -1,12 +1,30 @@ -import { afterEach, describe, expect, it } from "vitest"; -import { randomUUID } from "node:crypto"; -import { mkdirSync, rmdirSync, rmSync, writeFileSync } from "node:fs"; -import { dirname, join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { createHash, randomUUID } from "node:crypto"; +import { + chmodSync, + mkdirSync, + mkdtempSync, + readFileSync, + realpathSync, + renameSync, + rmdirSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { dirname, join, relative } from "node:path"; +import { tmpdir } from "node:os"; import { zstdCompressSync } from "node:zlib"; import { binariesDir, + binaryProgramCacheRoot, + findRepoRoot, localBinariesDir, + programOutputClosureRelPaths, + resetBinaryResolverManifestCacheForTests, resolveBinary, + setProgramIndexContextCheckerForTests, + tryResolveBinary, tryResolveBinarySet, } from "../src/binary-resolver"; import { ABI_VERSION } from "../src/generated/abi"; @@ -17,6 +35,42 @@ import { const cleanupDirs = new Set(); const cleanupEmptyDirs = new Set(); +let savedXdgCacheHome: string | undefined; +let savedBinaryCacheRoot: string | undefined; +let hadSavedBinaryCacheRoot = false; +let savedRegistry: string | undefined; +let hadSavedRegistry = false; +let fixtureRegistryRoot = ""; +let fixtureRegistryIdentities: Record = {}; +let fixtureRegistryPackages: Record = {}; + +beforeEach(() => { + hadSavedBinaryCacheRoot = Object.prototype.hasOwnProperty.call( + process.env, + "WASM_POSIX_BINARY_CACHE_ROOT", + ); + savedBinaryCacheRoot = process.env.WASM_POSIX_BINARY_CACHE_ROOT; + delete process.env.WASM_POSIX_BINARY_CACHE_ROOT; + savedXdgCacheHome = process.env.XDG_CACHE_HOME; + const cacheHome = mkdtempSync(join(tmpdir(), "kandelo-resolver-xdg-cache-")); + cleanupDirs.add(cacheHome); + process.env.XDG_CACHE_HOME = cacheHome; + hadSavedRegistry = Object.prototype.hasOwnProperty.call( + process.env, + "WASM_POSIX_DEPS_REGISTRY", + ); + savedRegistry = process.env.WASM_POSIX_DEPS_REGISTRY; + fixtureRegistryRoot = mkdtempSync( + join(tmpdir(), "kandelo-resolver-registry-"), + ); + cleanupDirs.add(fixtureRegistryRoot); + fixtureRegistryIdentities = {}; + fixtureRegistryPackages = {}; + writeFixtureRegistryIndex(); + process.env.WASM_POSIX_DEPS_REGISTRY = fixtureRegistryRoot; + setProgramIndexContextCheckerForTests(() => {}); + resetBinaryResolverManifestCacheForTests(); +}); afterEach(() => { for (const dir of cleanupDirs) { @@ -31,6 +85,23 @@ afterEach(() => { } cleanupDirs.clear(); cleanupEmptyDirs.clear(); + setProgramIndexContextCheckerForTests(null); + resetBinaryResolverManifestCacheForTests(); + if (savedXdgCacheHome === undefined) { + delete process.env.XDG_CACHE_HOME; + } else { + process.env.XDG_CACHE_HOME = savedXdgCacheHome; + } + if (hadSavedBinaryCacheRoot) { + process.env.WASM_POSIX_BINARY_CACHE_ROOT = savedBinaryCacheRoot ?? ""; + } else { + delete process.env.WASM_POSIX_BINARY_CACHE_ROOT; + } + if (hadSavedRegistry) { + process.env.WASM_POSIX_DEPS_REGISTRY = savedRegistry ?? ""; + } else { + delete process.env.WASM_POSIX_DEPS_REGISTRY; + } }); function uleb128(n: number): number[] { @@ -134,7 +205,503 @@ function writeCandidate(root: string, relPath: string, bytes: Uint8Array): strin return path; } +interface MultiOutputFixture { + name: string; + members: Array<{ + relPath: string; + sourceArtifact: string; + }>; +} + +function fixturePackageName(): string { + return `binary-resolver-test-${randomUUID()}`; +} + +function fixturePackageDirectory(name: string): string { + const directory = join(fixtureRegistryRoot, name); + cleanupDirs.add(directory); + return directory; +} + +function writeFixturePackageManifest(name: string, manifest: string): string { + const directory = fixturePackageDirectory(name); + mkdirSync(directory, { recursive: true }); + const manifestPath = join(directory, "package.toml"); + writeFileSync(manifestPath, manifest); + resetBinaryResolverManifestCacheForTests(); + return manifestPath; +} + +interface FixtureProjectionMember { + kind: "output" | "runtime-file"; + sourceArtifact: string; + mirrorPath: string; + outputName?: string; + forkInstrumentation?: "auto" | "disabled"; + guestPath?: string; + mode?: number; +} + +interface FixtureDependencyIdentity { + packageName: string; + manifestSha256: string; + cacheKey: string; +} + +function fixtureCacheKey(packageName: string, arch = "wasm32"): string { + return createHash("sha256") + .update(`binary-resolver fixture:${packageName}:${arch}`) + .digest("hex"); +} + +function writeFixtureRegistryIndex(): void { + writeFileSync( + join(fixtureRegistryRoot, "program-packages.json"), + `${JSON.stringify({ + format: "kandelo-program-packages-v2", + identities: fixtureRegistryIdentities, + packages: fixtureRegistryPackages, + }, null, 2)}\n`, + ); + resetBinaryResolverManifestCacheForTests(); +} + +function writeFixturePackageProjection( + name: string, + members: FixtureProjectionMember[], + arches = ["wasm32"], + cacheKeys = Object.fromEntries( + arches.map((arch) => [arch, fixtureCacheKey(name, arch)]), + ), + dependencyClosures: Record = + Object.fromEntries(arches.map((arch) => [arch, []])), +): void { + const manifestPath = join(fixturePackageDirectory(name), "package.toml"); + const manifestSha256 = createHash("sha256") + .update(readFileSync(manifestPath)) + .digest("hex"); + fixtureRegistryIdentities[name] = { + manifestSha256, + cacheKeys: { + wasm32: cacheKeys.wasm32 ?? fixtureCacheKey(name, "wasm32"), + wasm64: cacheKeys.wasm64 ?? fixtureCacheKey(name, "wasm64"), + }, + }; + fixtureRegistryPackages[name] = { + manifestSha256, + arches, + cacheKeys, + dependencyClosures, + members, + }; + writeFixtureRegistryIndex(); +} + +function writeFixturePackageIdentity( + name: string, + cacheKeys: Record = { + wasm32: fixtureCacheKey(name, "wasm32"), + wasm64: fixtureCacheKey(name, "wasm64"), + }, +): FixtureDependencyIdentity { + const manifestPath = join(fixturePackageDirectory(name), "package.toml"); + const manifestSha256 = createHash("sha256") + .update(readFileSync(manifestPath)) + .digest("hex"); + fixtureRegistryIdentities[name] = { manifestSha256, cacheKeys }; + writeFixtureRegistryIndex(); + return { + packageName: name, + manifestSha256, + cacheKey: cacheKeys.wasm32!, + }; +} + +describe("program package source freshness boundary", () => { + it("checks every public program-resolution boundary without duplicate nested checks", () => { + const relPath = fixtureRelPath(".dat"); + const path = writeCandidate( + localBinariesDir(), + relPath, + new TextEncoder().encode("program data"), + ); + const calls: Array<{ sourceRepoRoot: string; registryRoots: string[] }> = []; + setProgramIndexContextCheckerForTests((sourceRepoRoot, registryRoots) => { + calls.push({ sourceRepoRoot, registryRoots: [...registryRoots] }); + }); + + expect(programOutputClosureRelPaths(relPath)).toBeNull(); + expect(resolveBinary(relPath)).toBe(path); + expect(tryResolveBinary(relPath)).toBe(path); + expect(tryResolveBinarySet([relPath])).toEqual([path]); + + expect(calls).toHaveLength(4); + for (const call of calls) { + expect(call.sourceRepoRoot).toBe(findRepoRoot()); + expect(call.registryRoots).toEqual([fixtureRegistryRoot]); + } + }); + + it("passes the exact configured registry order to the Rust checker", () => { + const upperRoot = mkdtempSync( + join(tmpdir(), "kandelo-resolver-upper-registry-"), + ); + cleanupDirs.add(upperRoot); + writeFileSync( + join(upperRoot, "program-packages.json"), + '{"format":"kandelo-program-packages-v2","identities":{},"packages":{}}\n', + ); + process.env.WASM_POSIX_DEPS_REGISTRY = + `${upperRoot}:${fixtureRegistryRoot}`; + const calls: string[][] = []; + setProgramIndexContextCheckerForTests((_sourceRepoRoot, registryRoots) => { + calls.push([...registryRoots]); + }); + + expect( + programOutputClosureRelPaths( + "programs/wasm32/not-selected/not-selected.wasm", + ), + ).toBeNull(); + + expect(calls).toEqual([[upperRoot, fixtureRegistryRoot]]); + }); + + it("does not consume source projection policy after the exact checker fails", () => { + setProgramIndexContextCheckerForTests(() => { + throw new Error("injected stale program projection"); + }); + + expect(() => + programOutputClosureRelPaths( + "programs/wasm32/stale-package/stale.wasm", + ) + ).toThrow("injected stale program projection"); + }); + + it("executes the production checker command and fails closed on its error", () => { + const checkerRoot = mkdtempSync( + join(tmpdir(), "kandelo-resolver-checker-command-"), + ); + cleanupDirs.add(checkerRoot); + const checkerPath = join(checkerRoot, "xtask"); + writeFileSync( + checkerPath, + `#!/bin/sh +printf 'checker args: %s %s\\nregistry: %s\\n' "$1" "$2" "$WASM_POSIX_DEPS_REGISTRY" >&2 +exit 23 +`, + ); + chmodSync(checkerPath, 0o755); + const savedXtask = process.env.WASM_POSIX_XTASK_BIN; + const hadSavedXtask = Object.prototype.hasOwnProperty.call( + process.env, + "WASM_POSIX_XTASK_BIN", + ); + process.env.WASM_POSIX_XTASK_BIN = checkerPath; + setProgramIndexContextCheckerForTests(null); + try { + expect(() => + programOutputClosureRelPaths( + "programs/wasm32/production-check/production-check.wasm", + ) + ).toThrow( + /program-index-context-check failed with status 23[\s\S]*checker args: build-deps program-index-context-check/, + ); + } finally { + if (hadSavedXtask) { + process.env.WASM_POSIX_XTASK_BIN = savedXtask ?? ""; + } else { + delete process.env.WASM_POSIX_XTASK_BIN; + } + setProgramIndexContextCheckerForTests(() => {}); + } + }); +}); + +interface StandaloneRegistryEntry { + manifest: string; + cacheKeys: Record<"wasm32" | "wasm64", string>; + projection?: { + arches: Array<"wasm32" | "wasm64">; + dependencyClosures: Record; + members: FixtureProjectionMember[]; + }; +} + +function writeStandaloneRegistry( + root: string, + entries: Record, + contextualEntries: Record = {}, +): void { + const identities: Record = {}; + const packages: Record = {}; + const addProjection = ( + packageName: string, + entry: StandaloneRegistryEntry, + ) => { + const manifestSha256 = createHash("sha256") + .update(entry.manifest) + .digest("hex"); + identities[packageName] = { + manifestSha256, + cacheKeys: entry.cacheKeys, + }; + if (entry.projection) { + packages[packageName] = { + manifestSha256, + arches: entry.projection.arches, + cacheKeys: Object.fromEntries( + entry.projection.arches.map( + (arch) => [arch, entry.cacheKeys[arch]], + ), + ), + dependencyClosures: entry.projection.dependencyClosures, + members: entry.projection.members, + }; + } + }; + for (const [packageName, entry] of Object.entries(contextualEntries)) { + addProjection(packageName, entry); + } + for (const [packageName, entry] of Object.entries(entries)) { + const packageRoot = join(root, packageName); + mkdirSync(packageRoot, { recursive: true }); + writeFileSync(join(packageRoot, "package.toml"), entry.manifest); + addProjection(packageName, entry); + } + writeFileSync( + join(root, "program-packages.json"), + `${JSON.stringify({ + format: "kandelo-program-packages-v2", + identities, + packages, + }, null, 2)}\n`, + ); +} + +function standaloneDependencyIdentity( + packageName: string, + manifest: string, + cacheKey: string, +): FixtureDependencyIdentity { + return { + packageName, + manifestSha256: createHash("sha256").update(manifest).digest("hex"), + cacheKey, + }; +} + +function createMultiOutputFixture(): MultiOutputFixture { + const name = fixturePackageName(); + writeFixturePackageManifest(name, `kind = "program" +name = "${name}" +version = "1.0.0" +kernel_abi = ${ABI_VERSION} +depends_on = [] + +[source] +url = "https://example.invalid/source.tar.gz" +sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + +[license] +spdx = "MIT" + +[[outputs]] +name = "image" +wasm = "artifacts/image.zip" + +[[outputs]] +name = "bootstrap" +wasm = "support/bootstrap.zip" + +[[runtime_files]] +artifact = "share/runtime.dat" +guest_path = "/usr/share/runtime.dat" +`); + const members = [ + { + relPath: `programs/wasm32/${name}/image.zip`, + sourceArtifact: "artifacts/image.zip", + }, + { + relPath: `programs/wasm32/${name}/bootstrap.zip`, + sourceArtifact: "support/bootstrap.zip", + }, + { + relPath: `programs/wasm32/${name}/share/runtime.dat`, + sourceArtifact: "share/runtime.dat", + }, + ]; + writeFixturePackageProjection(name, [ + { + kind: "output", + sourceArtifact: "artifacts/image.zip", + mirrorPath: `${name}/image.zip`, + outputName: "image", + forkInstrumentation: "auto", + }, + { + kind: "output", + sourceArtifact: "support/bootstrap.zip", + mirrorPath: `${name}/bootstrap.zip`, + outputName: "bootstrap", + forkInstrumentation: "auto", + }, + { + kind: "runtime-file", + sourceArtifact: "share/runtime.dat", + mirrorPath: `${name}/share/runtime.dat`, + guestPath: "/usr/share/runtime.dat", + mode: 0o644, + }, + ]); + for (const root of [ + localBinariesDir(), + binariesDir(), + join(findRepoRoot(), "host", "wasm"), + ]) { + cleanupDirs.add(join(root, "programs", "wasm32", name)); + } + return { name, members }; +} + +function createScalarOutputFixture(): { + name: string; + relPath: string; + sourceArtifact: string; +} { + const name = fixturePackageName(); + const outputName = `command-${randomUUID()}`; + const sourceArtifact = `bin/${outputName}.wasm`; + const relPath = `programs/wasm32/${outputName}.wasm`; + writeFixturePackageManifest(name, `kind = "program" +name = "${name}" +version = "1.0.0" +kernel_abi = ${ABI_VERSION} +depends_on = [] + +[source] +url = "https://example.invalid/${name}.tar.gz" +sha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + +[license] +spdx = "MIT" + +[[outputs]] +name = "${outputName}" +wasm = "${sourceArtifact}" +`); + writeFixturePackageProjection(name, [{ + kind: "output", + sourceArtifact, + mirrorPath: `${outputName}.wasm`, + outputName, + forkInstrumentation: "auto", + }]); + for (const root of [localBinariesDir(), binariesDir()]) { + cleanupDirs.add(join(root, relPath)); + } + return { name, relPath, sourceArtifact }; +} + +function fixtureCanonicalRoot( + packageName: string, + arch = "wasm32", + cacheKey = fixtureCacheKey(packageName, arch), +): string { + const root = join( + binaryProgramCacheRoot(), + `${packageName}-1.0.0-rev1-${arch}-${cacheKey}`, + ); + mkdirSync(root, { recursive: true }); + cleanupDirs.add(root); + return root; +} + +function fixtureLocalCanonicalRoot( + packageName: string, + arch = "wasm32", + cacheKey = fixtureCacheKey(packageName, arch), +): string { + const packageGenerations = join( + localBinariesDir(), + ".kandelo-local-generations", + arch, + packageName, + cacheKey, + ); + const root = join(packageGenerations, randomUUID()); + mkdirSync(root, { recursive: true }); + cleanupDirs.add(packageGenerations); + cleanupEmptyDirs.add(dirname(packageGenerations)); + cleanupEmptyDirs.add(dirname(dirname(packageGenerations))); + cleanupEmptyDirs.add(dirname(dirname(dirname(packageGenerations)))); + return root; +} + +function fixtureArbitraryRoot(): string { + const root = mkdtempSync(join(tmpdir(), "kandelo-binary-resolver-arbitrary-")); + cleanupDirs.add(root); + return root; +} + +function writeCanonicalMember( + canonicalRoot: string, + sourceArtifact: string, + contents: string | Uint8Array, +): string { + const target = join(canonicalRoot, ...sourceArtifact.split("/")); + mkdirSync(dirname(target), { recursive: true }); + writeFileSync(target, contents); + return target; +} + +function linkClosureMember( + mirrorRoot: string, + member: MultiOutputFixture["members"][number], + canonicalRoot: string, + contents: string | Uint8Array = member.relPath, +): string { + const target = writeCanonicalMember( + canonicalRoot, + member.sourceArtifact, + contents, + ); + const mirror = join(mirrorRoot, member.relPath); + mkdirSync(dirname(mirror), { recursive: true }); + symlinkSync(target, mirror); + return mirror; +} + describe("binary resolver artifact policy", () => { + it("does not mistake an installed package consumer workspace for Kandelo", () => { + const consumer = mkdtempSync(join(tmpdir(), "kandelo-consumer-root-")); + cleanupDirs.add(consumer); + const installedModule = join( + consumer, + "node_modules", + "@automattic", + "wasm-posix-host", + "dist", + ); + mkdirSync(installedModule, { recursive: true }); + writeFileSync(join(consumer, "Cargo.toml"), "[workspace]\nmembers = []\n"); + writeFileSync( + join(consumer, "package.json"), + '{"name":"unrelated-consumer","private":true}\n', + ); + + expect(() => findRepoRoot(installedModule)).toThrow( + /Could not find repo root/, + ); + writeFileSync( + join(consumer, "package.json"), + '{"name":"kandelo","private":true}\n', + ); + expect(findRepoRoot(installedModule)).toBe(consumer); + }); + it("skips a stale local .vfs.zst when a fetched ABI-matching candidate exists", async () => { const relPath = fixtureRelPath(".vfs.zst"); const staleLocal = await vfsImage( @@ -239,9 +806,1673 @@ describe("binary resolver artifact policy", () => { expect(resolveBinary(relPath)).toBe(localPath); }); + + it("returns null only for a genuinely absent scalar artifact", () => { + const missing = fixtureRelPath(".dat"); + expect(tryResolveBinary(missing)).toBeNull(); + + const rejected = fixtureRelPath(".wasm"); + writeCandidate( + localBinariesDir(), + rejected, + new TextEncoder().encode("not a Wasm module"), + ); + expect(() => tryResolveBinary(rejected)).toThrow( + /exists but was rejected by artifact policy/, + ); + + const dangling = fixtureRelPath(".dat"); + const danglingPath = candidatePath(localBinariesDir(), dangling); + mkdirSync(dirname(danglingPath), { recursive: true }); + symlinkSync(`${danglingPath}.missing-target`, danglingPath); + expect(() => tryResolveBinary(dangling)).toThrow( + /exists but was rejected by artifact policy/, + ); + }); }); describe("binary resolver package closures", () => { + it("rejects noncanonical path spellings at every public resolver entry", () => { + const fixture = createMultiOutputFixture(); + const canonical = fixture.members[0]!.relPath; + const packagePrefix = `programs/wasm32/${fixture.name}`; + const aliases = [ + `${packagePrefix}/./image.zip`, + `${packagePrefix}//image.zip`, + `${packagePrefix}/../${fixture.name}/image.zip`, + canonical.replaceAll("/", "\\"), + `/absolute/${canonical}`, + `C:/${canonical}`, + ]; + + for (const alias of aliases) { + expect(() => programOutputClosureRelPaths(alias)).toThrow( + /normalized portable relative path/, + ); + expect(() => resolveBinary(alias)).toThrow( + /normalized portable relative path/, + ); + expect(() => tryResolveBinary(alias)).toThrow( + /normalized portable relative path/, + ); + expect(() => tryResolveBinarySet([alias])).toThrow( + /normalized portable relative path/, + ); + } + }); + + it("leaves a nested path with no registry package directory on the single-artifact path", () => { + const name = fixturePackageName(); + const relPath = `programs/wasm32/${name}/standalone.dat`; + cleanupDirs.add(join(localBinariesDir(), "programs", "wasm32", name)); + const localPath = writeCandidate( + localBinariesDir(), + relPath, + new TextEncoder().encode("standalone"), + ); + + expect(programOutputClosureRelPaths(relPath)).toBeNull(); + expect(resolveBinary(relPath)).toBe(localPath); + }); + + it("matches Rust first-hit semantics by ignoring a directory without package.toml", () => { + const name = fixturePackageName(); + mkdirSync(fixturePackageDirectory(name), { recursive: true }); + const relPath = `programs/wasm32/${name}/image.zip`; + const localPath = writeCandidate( + localBinariesDir(), + relPath, + new TextEncoder().encode("not-a-package"), + ); + + expect(programOutputClosureRelPaths(relPath)).toBeNull(); + expect(resolveBinary(relPath)).toBe(localPath); + }); + + it("matches Rust first-hit semantics when package.toml is not a file", () => { + const name = fixturePackageName(); + const directory = fixturePackageDirectory(name); + mkdirSync(join(directory, "package.toml"), { recursive: true }); + const relPath = `programs/wasm32/${name}/image.zip`; + const localPath = writeCandidate( + localBinariesDir(), + relPath, + new TextEncoder().encode("not-a-package"), + ); + + expect(programOutputClosureRelPaths(relPath)).toBeNull(); + expect(resolveBinary(relPath)).toBe(localPath); + }); + + it("does not parse an unrelated malformed manifest at runtime", () => { + const malformedName = fixturePackageName(); + writeFixturePackageManifest(malformedName, `kind = "program" +name = "${malformedName}" +[[outputs] +name = "one" +wasm = "one.zip" +[[outputs]] +name = "two" +wasm = "two.zip" +`); + const fixture = createMultiOutputFixture(); + expect(programOutputClosureRelPaths(fixture.members[0]!.relPath)).toEqual( + fixture.members.map((member) => member.relPath), + ); + expect(() => programOutputClosureRelPaths( + `programs/wasm32/${malformedName}/one.zip`, + )).toThrow(/absent from program-packages\.json/); + + const incompleteName = fixturePackageName(); + writeFixturePackageManifest(incompleteName, `kind = "program" +name = "${incompleteName}" +[[outputs]] +name = "one" +wasm = "one.zip" +[[outputs]] +name = "two" +`); + expect(() => programOutputClosureRelPaths( + `programs/wasm32/${incompleteName}/one.zip`, + )).toThrow(/absent from program-packages\.json/); + }); + + it("discovers every output and runtime file in a multi-member package", () => { + const fixture = createMultiOutputFixture(); + const expected = fixture.members.map((member) => member.relPath); + + for (const member of fixture.members) { + expect(programOutputClosureRelPaths(member.relPath)).toEqual(expected); + } + expect(() => programOutputClosureRelPaths( + `programs/wasm32/${fixture.name}/not-declared.zip`, + )).toThrow(/is not a declared member of package/); + }); + + it("discovers Rust-valid package and output path components", () => { + const name = `.binary resolver ${randomUUID()}`; + writeFixturePackageManifest(name, `kind = "program" +name = "${name}" +[[outputs]] +name = "image one" +wasm = "artifacts/image.zip" +[[outputs]] +name = ".bootstrap" +wasm = "support/bootstrap.zip" +`); + writeFixturePackageProjection(name, [ + { + kind: "output", + sourceArtifact: "artifacts/image.zip", + mirrorPath: `${name}/image one.zip`, + outputName: "image one", + forkInstrumentation: "auto", + }, + { + kind: "output", + sourceArtifact: "support/bootstrap.zip", + mirrorPath: `${name}/.bootstrap.zip`, + outputName: ".bootstrap", + forkInstrumentation: "auto", + }, + ]); + const members = [ + { + relPath: `programs/wasm32/${name}/image one.zip`, + sourceArtifact: "artifacts/image.zip", + }, + { + relPath: `programs/wasm32/${name}/.bootstrap.zip`, + sourceArtifact: "support/bootstrap.zip", + }, + ]; + for (const root of [localBinariesDir(), binariesDir()]) { + cleanupDirs.add(join(root, "programs", "wasm32", name)); + } + const canonicalRoot = fixtureCanonicalRoot(name); + const targets = members.map((member) => realpathSync( + linkClosureMember(binariesDir(), member, canonicalRoot), + )); + + expect(programOutputClosureRelPaths(members[0]!.relPath)).toEqual( + members.map((member) => member.relPath), + ); + expect(resolveBinary(members[0]!.relPath)).toBe(targets[0]); + }); + + it("treats one output plus a runtime file as one package generation", () => { + const name = fixturePackageName(); + writeFixturePackageManifest(name, `kind = "program" +name = "${name}" +[[outputs]] +name = "${name}" +wasm = "${name}.wasm" +[[runtime_files]] +artifact = "share/runtime.dat" +guest_path = "/usr/share/runtime.dat" +`); + writeFixturePackageProjection(name, [ + { + kind: "output", + sourceArtifact: `${name}.wasm`, + mirrorPath: `${name}/${name}.wasm`, + outputName: name, + forkInstrumentation: "auto", + }, + { + kind: "runtime-file", + sourceArtifact: "share/runtime.dat", + mirrorPath: `${name}/share/runtime.dat`, + guestPath: "/usr/share/runtime.dat", + mode: 0o644, + }, + ]); + const members = [ + { + relPath: `programs/wasm32/${name}/${name}.wasm`, + sourceArtifact: `${name}.wasm`, + }, + { + relPath: `programs/wasm32/${name}/share/runtime.dat`, + sourceArtifact: "share/runtime.dat", + }, + ]; + for (const root of [localBinariesDir(), binariesDir()]) { + cleanupDirs.add(join(root, "programs", "wasm32", name)); + } + const canonicalRoot = fixtureCanonicalRoot(name); + const targets = members.map((member, index) => { + const mirror = linkClosureMember( + binariesDir(), + member, + canonicalRoot, + index === 0 ? executableWasmWithAbi(ABI_VERSION) : "runtime", + ); + return realpathSync(mirror); + }); + + for (const member of members) { + expect(programOutputClosureRelPaths(member.relPath)).toEqual( + members.map((entry) => entry.relPath), + ); + } + expect(resolveBinary(members[0]!.relPath)).toBe(targets[0]); + expect(tryResolveBinarySet(members.map((member) => member.relPath))).toEqual( + targets, + ); + + const legacyFlatPath = `programs/wasm32/${name}.wasm`; + expect(() => programOutputClosureRelPaths(legacyFlatPath)).toThrow( + new RegExp(`Legacy flat resolver path.*${name}/${name}\\.wasm`), + ); + expect(() => resolveBinary(legacyFlatPath)).toThrow( + /Legacy flat resolver path/, + ); + expect(() => tryResolveBinary(legacyFlatPath)).toThrow( + /Legacy flat resolver path/, + ); + expect(() => tryResolveBinarySet([legacyFlatPath])).toThrow( + /Legacy flat resolver path/, + ); + expect(() => programOutputClosureRelPaths( + `programs/wasm64/${name}/${name}.wasm`, + )).toThrow(/does not declare resolver artifacts for wasm64/); + }); + + it("rejects a stale nested directory after a package becomes scalar", () => { + const name = fixturePackageName(); + writeFixturePackageManifest(name, `kind = "program" +name = "${name}" +[[outputs]] +name = "${name}" +wasm = "${name}.wasm" +`); + writeFixturePackageProjection(name, [ + { + kind: "output", + sourceArtifact: `${name}.wasm`, + mirrorPath: `${name}.wasm`, + outputName: name, + forkInstrumentation: "auto", + }, + ]); + const staleNested = `programs/wasm32/${name}/${name}.wasm`; + writeCandidate( + localBinariesDir(), + staleNested, + executableWasmWithAbi(ABI_VERSION), + ); + + expect(() => programOutputClosureRelPaths(staleNested)).toThrow( + /is not a declared member of package/, + ); + expect(() => resolveBinary(staleNested)).toThrow( + /is not a declared member of package/, + ); + }); + + it("fails a selected package when its generated projection is stale", () => { + const fixture = createMultiOutputFixture(); + const manifestPath = join( + fixtureRegistryRoot, + fixture.name, + "package.toml", + ); + writeFileSync( + manifestPath, + `${readFileSync(manifestPath, "utf8")}\n# changed after projection\n`, + ); + + expect(() => programOutputClosureRelPaths( + fixture.members[0]!.relPath, + )).toThrow(/projection is stale/); + }); + + it("uses the requested arch when a scalar owner shares a legacy flat name", () => { + const sharedOutput = `shared-${randomUUID()}`; + const packageOwnedName = fixturePackageName(); + writeFixturePackageManifest(packageOwnedName, `kind = "program" +name = "${packageOwnedName}" +arches = ["wasm32", "wasm64"] +[[outputs]] +name = "${sharedOutput}" +wasm = "${sharedOutput}.wasm" +[[runtime_files]] +artifact = "share/runtime.dat" +guest_path = "/usr/share/runtime.dat" +`); + writeFixturePackageProjection( + packageOwnedName, + [ + { + kind: "output", + sourceArtifact: `${sharedOutput}.wasm`, + mirrorPath: `${packageOwnedName}/${sharedOutput}.wasm`, + outputName: sharedOutput, + forkInstrumentation: "auto", + }, + { + kind: "runtime-file", + sourceArtifact: "share/runtime.dat", + mirrorPath: `${packageOwnedName}/share/runtime.dat`, + guestPath: "/usr/share/runtime.dat", + mode: 0o644, + }, + ], + ["wasm32", "wasm64"], + ); + const scalarName = fixturePackageName(); + writeFixturePackageManifest(scalarName, `kind = "program" +name = "${scalarName}" +arches = ["wasm32"] +[[outputs]] +name = "${sharedOutput}" +wasm = "${sharedOutput}.wasm" +`); + writeFixturePackageProjection(scalarName, [ + { + kind: "output", + sourceArtifact: `${sharedOutput}.wasm`, + mirrorPath: `${sharedOutput}.wasm`, + outputName: sharedOutput, + forkInstrumentation: "auto", + }, + ]); + + expect(programOutputClosureRelPaths( + `programs/wasm32/${sharedOutput}.wasm`, + )).toEqual([`programs/wasm32/${sharedOutput}.wasm`]); + expect(() => programOutputClosureRelPaths( + `programs/wasm64/${sharedOutput}.wasm`, + )).toThrow(/Legacy flat resolver path/); + }); + + it("does not merge a lower external registry over a first-hit package", () => { + const name = fixturePackageName(); + writeFixturePackageManifest(name, `kind = "program" +name = "${name}" +[[outputs] +name = "broken" +wasm = "broken.wasm" +`); + + const lowerRoot = mkdtempSync(join(tmpdir(), "kandelo-lower-registry-")); + cleanupDirs.add(lowerRoot); + const lowerDirectory = join(lowerRoot, name); + mkdirSync(lowerDirectory, { recursive: true }); + const lowerManifest = `kind = "program" +name = "${name}" +[[outputs]] +name = "lower" +wasm = "lower.wasm" +[[runtime_files]] +artifact = "share/lower.dat" +guest_path = "/share/lower.dat" +`; + writeFileSync(join(lowerDirectory, "package.toml"), lowerManifest); + writeFileSync( + join(lowerRoot, "program-packages.json"), + `${JSON.stringify({ + format: "kandelo-program-packages-v2", + identities: { + [name]: { + manifestSha256: createHash("sha256") + .update(lowerManifest) + .digest("hex"), + cacheKeys: { + wasm32: fixtureCacheKey(name), + wasm64: fixtureCacheKey(name, "wasm64"), + }, + }, + }, + packages: { + [name]: { + manifestSha256: createHash("sha256") + .update(lowerManifest) + .digest("hex"), + arches: ["wasm32"], + cacheKeys: { + wasm32: fixtureCacheKey(name), + }, + dependencyClosures: { wasm32: [] }, + members: [ + { + kind: "output", + sourceArtifact: "lower.wasm", + mirrorPath: `${name}/lower.wasm`, + outputName: "lower", + forkInstrumentation: "auto", + }, + { + kind: "runtime-file", + sourceArtifact: "share/lower.dat", + mirrorPath: `${name}/share/lower.dat`, + guestPath: "/share/lower.dat", + mode: 0o644, + }, + ], + }, + }, + }, null, 2)}\n`, + ); + process.env.WASM_POSIX_DEPS_REGISTRY = + `${fixtureRegistryRoot}:${lowerRoot}`; + resetBinaryResolverManifestCacheForTests(); + + expect(() => programOutputClosureRelPaths( + `programs/wasm32/${name}/lower.wasm`, + )).toThrow(/absent from program-packages\.json/); + }); + + it("keeps a shadowed lower scalar program path fail-closed", () => { + const upperRoot = mkdtempSync(join(tmpdir(), "kandelo-upper-library-shadow-")); + const lowerRoot = mkdtempSync(join(tmpdir(), "kandelo-lower-program-shadow-")); + cleanupDirs.add(upperRoot); + cleanupDirs.add(lowerRoot); + const name = fixturePackageName(); + const outputName = `shadowed-${randomUUID()}`; + const upperManifest = `kind = "library" +name = "${name}" +version = "1.0.0" +`; + const lowerManifest = `kind = "program" +name = "${name}" +version = "1.0.0" +`; + writeStandaloneRegistry(upperRoot, { + [name]: { + manifest: upperManifest, + cacheKeys: { + wasm32: "1".repeat(64), + wasm64: "2".repeat(64), + }, + }, + }); + writeStandaloneRegistry(lowerRoot, { + [name]: { + manifest: lowerManifest, + cacheKeys: { + wasm32: "3".repeat(64), + wasm64: "4".repeat(64), + }, + projection: { + arches: ["wasm32"], + dependencyClosures: { wasm32: [] }, + members: [{ + kind: "output", + sourceArtifact: `${outputName}.wasm`, + mirrorPath: `${outputName}.wasm`, + outputName, + forkInstrumentation: "auto", + }], + }, + }, + }); + process.env.WASM_POSIX_DEPS_REGISTRY = `${upperRoot}:${lowerRoot}`; + const relPath = `programs/wasm32/${outputName}.wasm`; + writeCandidate( + binariesDir(), + relPath, + executableWasmWithAbi(ABI_VERSION), + ); + + expect(() => programOutputClosureRelPaths(relPath)).toThrow( + /selected at .* but is absent from program-packages\.json/, + ); + expect(() => resolveBinary(relPath)).toThrow( + /selected at .* but is absent from program-packages\.json/, + ); + }); + + it("accepts identical first-hit shadows and an external program generated against external:main", () => { + const upperRoot = mkdtempSync(join(tmpdir(), "kandelo-upper-context-")); + const lowerRoot = mkdtempSync(join(tmpdir(), "kandelo-lower-context-")); + cleanupDirs.add(upperRoot); + cleanupDirs.add(lowerRoot); + const contextId = randomUUID(); + const dependencyName = `z-context-dependency-${contextId}`; + const auxiliaryName = `a-context-dependency-${contextId}`; + const externalName = fixturePackageName(); + const lowerProgramName = fixturePackageName(); + const dependencyManifest = `kind = "library" +name = "${dependencyName}" +version = "1.0.0" +depends_on = [] +`; + const auxiliaryManifest = `kind = "source" +name = "${auxiliaryName}" +version = "1.0.0" +depends_on = [] +`; + const externalManifest = `kind = "program" +name = "${externalName}" +version = "1.0.0" +depends_on = ["${dependencyName}@1.0.0"] +`; + const lowerProgramManifest = `kind = "program" +name = "${lowerProgramName}" +version = "1.0.0" +depends_on = ["${dependencyName}@1.0.0"] +`; + const dependencyKeys = { + wasm32: "1".repeat(64), + wasm64: "2".repeat(64), + }; + const expectedDependency = standaloneDependencyIdentity( + dependencyName, + dependencyManifest, + dependencyKeys.wasm32, + ); + const auxiliaryKeys = { + wasm32: "7".repeat(64), + wasm64: "8".repeat(64), + }; + const expectedAuxiliary = standaloneDependencyIdentity( + auxiliaryName, + auxiliaryManifest, + auxiliaryKeys.wasm32, + ); + const scalarProjection = ( + packageName: string, + dependencyClosures: Record, + ) => ({ + arches: ["wasm32"] as Array<"wasm32">, + dependencyClosures, + members: [{ + kind: "output" as const, + sourceArtifact: `${packageName}.wasm`, + mirrorPath: `${packageName}.wasm`, + outputName: packageName, + forkInstrumentation: "auto" as const, + }], + }); + + writeStandaloneRegistry( + upperRoot, + { + [dependencyName]: { + manifest: dependencyManifest, + cacheKeys: dependencyKeys, + }, + [auxiliaryName]: { + manifest: auxiliaryManifest, + cacheKeys: auxiliaryKeys, + }, + [externalName]: { + manifest: externalManifest, + cacheKeys: { + wasm32: "3".repeat(64), + wasm64: "4".repeat(64), + }, + projection: scalarProjection(externalName, { + // Deliberately z-before-a: dependency order is non-semantic. + wasm32: [expectedDependency, expectedAuxiliary], + }), + }, + }, + { + [lowerProgramName]: { + manifest: lowerProgramManifest, + cacheKeys: { + wasm32: "5".repeat(64), + wasm64: "6".repeat(64), + }, + projection: scalarProjection(lowerProgramName, { + wasm32: [expectedDependency], + }), + }, + }, + ); + writeStandaloneRegistry(lowerRoot, { + [dependencyName]: { + manifest: dependencyManifest, + cacheKeys: dependencyKeys, + }, + [lowerProgramName]: { + manifest: lowerProgramManifest, + cacheKeys: { + wasm32: "5".repeat(64), + wasm64: "6".repeat(64), + }, + projection: scalarProjection(lowerProgramName, { + wasm32: [expectedDependency], + }), + }, + }); + process.env.WASM_POSIX_DEPS_REGISTRY = `${upperRoot}:${lowerRoot}`; + + expect(programOutputClosureRelPaths( + `programs/wasm32/${externalName}.wasm`, + )).toEqual([`programs/wasm32/${externalName}.wasm`]); + expect(programOutputClosureRelPaths( + `programs/wasm32/${lowerProgramName}.wasm`, + )).toEqual([`programs/wasm32/${lowerProgramName}.wasm`]); + }); + + it("accepts an external program generated against a changed transitive external:main context", () => { + const upperRoot = mkdtempSync(join(tmpdir(), "kandelo-upper-external-context-")); + const lowerRoot = mkdtempSync(join(tmpdir(), "kandelo-lower-external-context-")); + cleanupDirs.add(upperRoot); + cleanupDirs.add(lowerRoot); + const leafName = fixturePackageName(); + const middleName = fixturePackageName(); + const externalName = fixturePackageName(); + const lowerLeafManifest = `kind = "source" +name = "${leafName}" +version = "1.0.0" +depends_on = [] +[source] +url = "https://example.test/${leafName}.tar.gz" +sha256 = "${"0".repeat(64)}" +[license] +spdx = "MIT" +`; + const upperLeafManifest = lowerLeafManifest.replace( + "0".repeat(64), + "1".repeat(64), + ); + const middleManifest = `kind = "library" +name = "${middleName}" +version = "1.0.0" +depends_on = ["${leafName}@1.0.0"] +`; + const externalManifest = `kind = "program" +name = "${externalName}" +version = "1.0.0" +depends_on = ["${middleName}@1.0.0"] +`; + const upperLeafKeys = { + wasm32: "1".repeat(64), + wasm64: "2".repeat(64), + }; + const combinedMiddleKeys = { + wasm32: "3".repeat(64), + wasm64: "4".repeat(64), + }; + const externalKeys = { + wasm32: "5".repeat(64), + wasm64: "6".repeat(64), + }; + const expectedLeaf = standaloneDependencyIdentity( + leafName, + upperLeafManifest, + upperLeafKeys.wasm32, + ); + const expectedMiddle = standaloneDependencyIdentity( + middleName, + middleManifest, + combinedMiddleKeys.wasm32, + ); + writeStandaloneRegistry( + upperRoot, + { + [leafName]: { + manifest: upperLeafManifest, + cacheKeys: upperLeafKeys, + }, + [externalName]: { + manifest: externalManifest, + cacheKeys: externalKeys, + projection: { + arches: ["wasm32"], + dependencyClosures: { + wasm32: [expectedLeaf, expectedMiddle], + }, + members: [{ + kind: "output", + sourceArtifact: `${externalName}.wasm`, + mirrorPath: `${externalName}.wasm`, + outputName: externalName, + forkInstrumentation: "auto", + }], + }, + }, + }, + { + [middleName]: { + manifest: middleManifest, + cacheKeys: combinedMiddleKeys, + }, + }, + ); + writeStandaloneRegistry(lowerRoot, { + [leafName]: { + manifest: lowerLeafManifest, + cacheKeys: { + wasm32: "7".repeat(64), + wasm64: "8".repeat(64), + }, + }, + [middleName]: { + manifest: middleManifest, + cacheKeys: { + wasm32: "9".repeat(64), + wasm64: "a".repeat(64), + }, + }, + }); + process.env.WASM_POSIX_DEPS_REGISTRY = `${upperRoot}:${lowerRoot}`; + + expect(programOutputClosureRelPaths( + `programs/wasm32/${externalName}.wasm`, + )).toEqual([`programs/wasm32/${externalName}.wasm`]); + + writeFileSync( + join(lowerRoot, middleName, "package.toml"), + `${middleManifest}build_input = "changed-after-projection"\n`, + ); + expect(() => + programOutputClosureRelPaths(`programs/wasm32/${externalName}.wasm`) + ).toThrow(new RegExp(`identity is stale.*${middleName}`)); + }); + + it("uses the complete top projection for a lower program whose direct dependency is overridden", () => { + const upperRoot = mkdtempSync(join(tmpdir(), "kandelo-upper-direct-shadow-")); + const lowerRoot = mkdtempSync(join(tmpdir(), "kandelo-lower-direct-shadow-")); + cleanupDirs.add(upperRoot); + cleanupDirs.add(lowerRoot); + const dependencyName = fixturePackageName(); + const programName = fixturePackageName(); + const lowerDependencyManifest = `kind = "library" +name = "${dependencyName}" +version = "1.0.0" +depends_on = [] +[source] +url = "https://example.test/${dependencyName}.tar.gz" +sha256 = "${"0".repeat(64)}" +[license] +spdx = "MIT" +`; + const upperDependencyManifest = lowerDependencyManifest.replace( + "0".repeat(64), + "1".repeat(64), + ); + const programManifest = `kind = "program" +name = "${programName}" +version = "1.0.0" +depends_on = ["${dependencyName}@1.0.0"] +`; + const upperDependencyIdentity = standaloneDependencyIdentity( + dependencyName, + upperDependencyManifest, + "7".repeat(64), + ); + const programProjection = ( + dependency: FixtureDependencyIdentity, + ): StandaloneRegistryEntry["projection"] => ({ + arches: ["wasm32"], + dependencyClosures: { wasm32: [dependency] }, + members: [{ + kind: "output", + sourceArtifact: `${programName}.wasm`, + mirrorPath: `${programName}.wasm`, + outputName: programName, + forkInstrumentation: "auto", + }], + }); + writeStandaloneRegistry( + upperRoot, + { + [dependencyName]: { + manifest: upperDependencyManifest, + cacheKeys: { + wasm32: "7".repeat(64), + wasm64: "8".repeat(64), + }, + }, + }, + { + [programName]: { + manifest: programManifest, + cacheKeys: { + wasm32: "d".repeat(64), + wasm64: "e".repeat(64), + }, + projection: programProjection(upperDependencyIdentity), + }, + }, + ); + writeStandaloneRegistry(lowerRoot, { + [dependencyName]: { + manifest: lowerDependencyManifest, + cacheKeys: { + wasm32: "9".repeat(64), + wasm64: "a".repeat(64), + }, + }, + [programName]: { + manifest: programManifest, + cacheKeys: { + wasm32: "b".repeat(64), + wasm64: "c".repeat(64), + }, + projection: programProjection(standaloneDependencyIdentity( + dependencyName, + lowerDependencyManifest, + "9".repeat(64), + )), + }, + }); + process.env.WASM_POSIX_DEPS_REGISTRY = `${upperRoot}:${lowerRoot}`; + + expect(programOutputClosureRelPaths( + `programs/wasm32/${programName}.wasm`, + )).toEqual([`programs/wasm32/${programName}.wasm`]); + + process.env.WASM_POSIX_DEPS_REGISTRY = lowerRoot; + expect(programOutputClosureRelPaths( + `programs/wasm32/${programName}.wasm`, + )).toEqual([`programs/wasm32/${programName}.wasm`]); + }); + + it("uses the complete top projection when a lower program's transitive dependency is overridden", () => { + const upperRoot = mkdtempSync(join(tmpdir(), "kandelo-upper-transitive-shadow-")); + const lowerRoot = mkdtempSync(join(tmpdir(), "kandelo-lower-transitive-shadow-")); + cleanupDirs.add(upperRoot); + cleanupDirs.add(lowerRoot); + const leafName = fixturePackageName(); + const intermediateName = fixturePackageName(); + const programName = fixturePackageName(); + const lowerLeafManifest = `kind = "source" +name = "${leafName}" +version = "1.0.0" +depends_on = [] +[source] +url = "https://example.test/${leafName}.tar.gz" +sha256 = "${"0".repeat(64)}" +[license] +spdx = "MIT" +`; + const upperLeafManifest = lowerLeafManifest.replace( + "0".repeat(64), + "1".repeat(64), + ); + const intermediateManifest = `kind = "library" +name = "${intermediateName}" +version = "1.0.0" +depends_on = ["${leafName}@1.0.0"] +`; + const programManifest = `kind = "program" +name = "${programName}" +version = "1.0.0" +depends_on = ["${intermediateName}@1.0.0"] +`; + const lowerLeafIdentity = standaloneDependencyIdentity( + leafName, + lowerLeafManifest, + "d".repeat(64), + ); + const intermediateIdentity = standaloneDependencyIdentity( + intermediateName, + intermediateManifest, + "e".repeat(64), + ); + const combinedLeafIdentity = standaloneDependencyIdentity( + leafName, + upperLeafManifest, + "f".repeat(64), + ); + const combinedIntermediateIdentity = standaloneDependencyIdentity( + intermediateName, + intermediateManifest, + "4".repeat(64), + ); + const programMembers: FixtureProjectionMember[] = [{ + kind: "output", + sourceArtifact: `${programName}.wasm`, + mirrorPath: `${programName}.wasm`, + outputName: programName, + forkInstrumentation: "auto", + }]; + writeStandaloneRegistry( + upperRoot, + { + [leafName]: { + manifest: upperLeafManifest, + cacheKeys: { + wasm32: "f".repeat(64), + wasm64: "f".repeat(64), + }, + }, + }, + { + [intermediateName]: { + manifest: intermediateManifest, + cacheKeys: { + wasm32: "4".repeat(64), + wasm64: "5".repeat(64), + }, + }, + [programName]: { + manifest: programManifest, + cacheKeys: { + wasm32: "6".repeat(64), + wasm64: "7".repeat(64), + }, + projection: { + arches: ["wasm32"], + dependencyClosures: { + wasm32: [ + combinedIntermediateIdentity, + combinedLeafIdentity, + ].sort( + (left, right) => + left.packageName.localeCompare(right.packageName), + ), + }, + members: programMembers, + }, + }, + }, + ); + writeStandaloneRegistry(lowerRoot, { + [leafName]: { + manifest: lowerLeafManifest, + cacheKeys: { + wasm32: lowerLeafIdentity.cacheKey, + wasm64: lowerLeafIdentity.cacheKey, + }, + }, + [intermediateName]: { + manifest: intermediateManifest, + cacheKeys: { + wasm32: intermediateIdentity.cacheKey, + wasm64: "1".repeat(64), + }, + }, + [programName]: { + manifest: programManifest, + cacheKeys: { + wasm32: "2".repeat(64), + wasm64: "3".repeat(64), + }, + projection: { + arches: ["wasm32"], + dependencyClosures: { + wasm32: [intermediateIdentity, lowerLeafIdentity].sort( + (left, right) => left.packageName.localeCompare(right.packageName), + ), + }, + members: programMembers, + }, + }, + }); + process.env.WASM_POSIX_DEPS_REGISTRY = `${upperRoot}:${lowerRoot}`; + + expect(programOutputClosureRelPaths( + `programs/wasm32/${programName}.wasm`, + )).toEqual([`programs/wasm32/${programName}.wasm`]); + }); + + it("rejects unknown projection fields instead of widening the policy schema", () => { + const name = fixturePackageName(); + writeFixturePackageManifest(name, `kind = "program" +name = "${name}" +[[outputs]] +name = "${name}" +wasm = "${name}.wasm" +`); + writeFixturePackageProjection(name, [{ + kind: "output", + sourceArtifact: `${name}.wasm`, + mirrorPath: `${name}.wasm`, + outputName: name, + forkInstrumentation: "auto", + }]); + const projected = fixtureRegistryPackages[name] as Record; + projected.unreviewedPolicy = true; + writeFixtureRegistryIndex(); + + expect(() => programOutputClosureRelPaths( + `programs/wasm32/${name}.wasm`, + )).toThrow(/malformed package/); + }); + + it("rejects a projection that invents a noncanonical scalar mirror layout", () => { + const name = fixturePackageName(); + writeFixturePackageManifest(name, `kind = "program" +name = "${name}" +[[outputs]] +name = "${name}" +wasm = "${name}.wasm" +`); + writeFixturePackageProjection(name, [{ + kind: "output", + sourceArtifact: `${name}.wasm`, + mirrorPath: `${name}/${name}.wasm`, + outputName: name, + forkInstrumentation: "auto", + }]); + + expect(() => programOutputClosureRelPaths( + `programs/wasm32/${name}/${name}.wasm`, + )).toThrow(/violate scalar\/package-directory layout/); + }); + + it("observes package additions and shape changes without a process restart", () => { + const name = fixturePackageName(); + const nested = `programs/wasm32/${name}/first.wasm`; + expect(programOutputClosureRelPaths(nested)).toBeNull(); + + writeFixturePackageManifest(name, `kind = "program" +name = "${name}" +[[outputs]] +name = "first" +wasm = "first.wasm" +[[outputs]] +name = "second" +wasm = "second.wasm" +`); + writeFixturePackageProjection(name, [ + { + kind: "output", + sourceArtifact: "first.wasm", + mirrorPath: `${name}/first.wasm`, + outputName: "first", + forkInstrumentation: "auto", + }, + { + kind: "output", + sourceArtifact: "second.wasm", + mirrorPath: `${name}/second.wasm`, + outputName: "second", + forkInstrumentation: "auto", + }, + ]); + expect(programOutputClosureRelPaths(nested)).toEqual([ + nested, + `programs/wasm32/${name}/second.wasm`, + ]); + + writeFixturePackageManifest(name, `kind = "program" +name = "${name}" +[[outputs]] +name = "first" +wasm = "first.wasm" +`); + writeFixturePackageProjection(name, [{ + kind: "output", + sourceArtifact: "first.wasm", + mirrorPath: "first.wasm", + outputName: "first", + forkInstrumentation: "auto", + }]); + expect(() => programOutputClosureRelPaths(nested)).toThrow( + /not a declared member/, + ); + expect(programOutputClosureRelPaths( + "programs/wasm32/first.wasm", + )).toEqual(["programs/wasm32/first.wasm"]); + }); + + it("rejects cross-package file and package-directory mirror collisions", () => { + const scalarName = fixturePackageName(); + const directoryName = fixturePackageName(); + writeFixturePackageManifest(scalarName, `kind = "program" +name = "${scalarName}" +[[outputs]] +name = "${directoryName}" +wasm = "artifact" +`); + writeFixturePackageProjection(scalarName, [{ + kind: "output", + sourceArtifact: "artifact", + mirrorPath: directoryName, + outputName: directoryName, + forkInstrumentation: "auto", + }]); + writeFixturePackageManifest(directoryName, `kind = "program" +name = "${directoryName}" +[[outputs]] +name = "first" +wasm = "first.wasm" +[[outputs]] +name = "second" +wasm = "second.wasm" +`); + writeFixturePackageProjection(directoryName, [ + { + kind: "output", + sourceArtifact: "first.wasm", + mirrorPath: `${directoryName}/first.wasm`, + outputName: "first", + forkInstrumentation: "auto", + }, + { + kind: "output", + sourceArtifact: "second.wasm", + mirrorPath: `${directoryName}/second.wasm`, + outputName: "second", + forkInstrumentation: "auto", + }, + ]); + + expect(() => programOutputClosureRelPaths( + `programs/wasm32/${directoryName}/first.wasm`, + )).toThrow(/conflict between selected packages/); + }); + + it("accepts Rust-valid literal-string metadata through the generated projection", () => { + const name = fixturePackageName(); + writeFixturePackageManifest(name, `kind = 'program' +name = '${name}' +[[outputs]] +name = '${name}' +wasm = '${name}.wasm' +[[runtime_files]] +artifact = 'share/runtime.dat' +guest_path = '/usr/share/runtime.dat' +`); + writeFixturePackageProjection(name, [ + { + kind: "output", + sourceArtifact: `${name}.wasm`, + mirrorPath: `${name}/${name}.wasm`, + outputName: name, + forkInstrumentation: "auto", + }, + { + kind: "runtime-file", + sourceArtifact: "share/runtime.dat", + mirrorPath: `${name}/share/runtime.dat`, + guestPath: "/usr/share/runtime.dat", + mode: 0o644, + }, + ]); + + const nested = `programs/wasm32/${name}/${name}.wasm`; + expect(programOutputClosureRelPaths(nested)).toEqual([ + nested, + `programs/wasm32/${name}/share/runtime.dat`, + ]); + expect(() => programOutputClosureRelPaths( + `programs/wasm32/${name}.wasm`, + )).toThrow(/Legacy flat resolver path/); + }); + + it("accepts mirror symlinks that all target one canonical generation", () => { + const fixture = createMultiOutputFixture(); + const canonicalRoot = fixtureCanonicalRoot(fixture.name); + const mirrors = fixture.members.map((member) => + linkClosureMember(binariesDir(), member, canonicalRoot) + ); + + const targets = mirrors.map((mirror) => realpathSync(mirror)); + expect(resolveBinary(fixture.members[0]!.relPath)).toBe(targets[0]); + expect(tryResolveBinarySet( + fixture.members.map((member) => member.relPath), + )).toEqual(targets); + }); + + it("shares an explicit package cache root with archive-stage consumers", () => { + const explicitCacheRoot = mkdtempSync( + join(tmpdir(), "kandelo-explicit-package-cache-"), + ); + cleanupDirs.add(explicitCacheRoot); + process.env.WASM_POSIX_BINARY_CACHE_ROOT = explicitCacheRoot; + const fixture = createMultiOutputFixture(); + const canonicalRoot = fixtureCanonicalRoot(fixture.name); + const mirrors = fixture.members.map((member) => + linkClosureMember(binariesDir(), member, canonicalRoot) + ); + const targets = mirrors.map((mirror) => realpathSync(mirror)); + + expect(binaryProgramCacheRoot()).toBe(join(explicitCacheRoot, "programs")); + expect(resolveBinary(fixture.members[0]!.relPath)).toBe(targets[0]); + expect(tryResolveBinarySet( + fixture.members.map((member) => member.relPath), + )).toEqual(targets); + }); + + it("anchors a relative explicit package cache at the Kandelo repository", () => { + const relativeRoot = `.test-package-cache-${randomUUID()}`; + process.env.WASM_POSIX_BINARY_CACHE_ROOT = relativeRoot; + expect(binaryProgramCacheRoot()).toBe( + join(findRepoRoot(), relativeRoot, "programs"), + ); + }); + + it("anchors a relative registry root at Kandelo even from an app cwd", () => { + const repo = findRepoRoot(); + const relativeRoot = `.test-program-registry-${randomUUID()}`; + const registryRoot = join(repo, relativeRoot); + cleanupDirs.add(registryRoot); + const savedFixtureRoot = fixtureRegistryRoot; + const savedFixturePackages = fixtureRegistryPackages; + const savedCwd = process.cwd(); + try { + fixtureRegistryRoot = registryRoot; + fixtureRegistryPackages = {}; + mkdirSync(registryRoot, { recursive: true }); + writeFixtureRegistryIndex(); + const fixture = createScalarOutputFixture(); + process.env.WASM_POSIX_DEPS_REGISTRY = relativeRoot; + process.chdir(join(repo, "apps", "browser-demos")); + const canonicalRoot = fixtureCanonicalRoot(fixture.name); + const mirror = linkClosureMember( + binariesDir(), + fixture, + canonicalRoot, + executableWasmWithAbi(ABI_VERSION), + ); + + expect(relative(repo, mirror)).toContain("binaries/programs/wasm32/"); + expect(resolveBinary(fixture.relPath)).toBe(realpathSync(mirror)); + } finally { + process.chdir(savedCwd); + fixtureRegistryRoot = savedFixtureRoot; + fixtureRegistryPackages = savedFixturePackages; + } + }); + + it("binds a fetched scalar output to its projected package and cache identity", () => { + const fixture = createScalarOutputFixture(); + const canonicalRoot = fixtureCanonicalRoot(fixture.name); + const mirror = linkClosureMember( + binariesDir(), + fixture, + canonicalRoot, + executableWasmWithAbi(ABI_VERSION), + ); + + expect(programOutputClosureRelPaths(fixture.relPath)).toEqual([ + fixture.relPath, + ]); + expect(resolveBinary(fixture.relPath)).toBe(realpathSync(mirror)); + }); + + it("binds a local scalar output to its projected package and cache identity", () => { + const fixture = createScalarOutputFixture(); + const canonicalRoot = fixtureLocalCanonicalRoot(fixture.name); + const mirror = linkClosureMember( + localBinariesDir(), + fixture, + canonicalRoot, + executableWasmWithAbi(ABI_VERSION), + ); + + expect(resolveBinary(fixture.relPath)).toBe(realpathSync(mirror)); + }); + + it("rejects stale fetched and local scalar generations after a recipe switch", () => { + const fixture = createScalarOutputFixture(); + const oldCacheKey = fixtureCacheKey(fixture.name); + const fetchedRoot = fixtureCanonicalRoot( + fixture.name, + "wasm32", + oldCacheKey, + ); + const localRoot = fixtureLocalCanonicalRoot( + fixture.name, + "wasm32", + oldCacheKey, + ); + linkClosureMember( + binariesDir(), + fixture, + fetchedRoot, + executableWasmWithAbi(ABI_VERSION), + ); + linkClosureMember( + localBinariesDir(), + fixture, + localRoot, + executableWasmWithAbi(ABI_VERSION), + ); + + const outputName = fixture.relPath.split("/").at(-1)!.replace(/\.wasm$/, ""); + writeFixturePackageManifest(fixture.name, `kind = "program" +name = "${fixture.name}" +version = "2.0.0" +kernel_abi = ${ABI_VERSION} +depends_on = [] +[source] +url = "https://example.invalid/${fixture.name}-v2.tar.gz" +sha256 = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +[license] +spdx = "MIT" +[[outputs]] +name = "${outputName}" +wasm = "${fixture.sourceArtifact}" +`); + const newCacheKey = "b".repeat(64); + writeFixturePackageProjection( + fixture.name, + [{ + kind: "output", + sourceArtifact: fixture.sourceArtifact, + mirrorPath: `${outputName}.wasm`, + outputName, + forkInstrumentation: "auto", + }], + ["wasm32"], + { wasm32: newCacheKey }, + ); + + expect(() => resolveBinary(fixture.relPath)).toThrow( + /shared package identity rejected/, + ); + expect(() => tryResolveBinarySet([fixture.relPath])).toThrow( + /shared package identity rejected/, + ); + }); + + it("rejects resolver-owned scalar links after their output is renamed", () => { + const fixture = createScalarOutputFixture(); + const fetchedRoot = fixtureCanonicalRoot(fixture.name); + const localRoot = fixtureLocalCanonicalRoot(fixture.name); + const localMirror = linkClosureMember( + localBinariesDir(), + fixture, + localRoot, + executableWasmWithAbi(ABI_VERSION), + ); + linkClosureMember( + binariesDir(), + fixture, + fetchedRoot, + executableWasmWithAbi(ABI_VERSION), + ); + + const renamedOutput = `renamed-${randomUUID()}`; + writeFixturePackageManifest(fixture.name, `kind = "program" +name = "${fixture.name}" +version = "2.0.0" +depends_on = [] +[source] +url = "https://example.invalid/${fixture.name}-v2.tar.gz" +sha256 = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +[license] +spdx = "MIT" +[[outputs]] +name = "${renamedOutput}" +wasm = "bin/${renamedOutput}.zip" +`); + writeFixturePackageProjection(fixture.name, [{ + kind: "output", + sourceArtifact: `bin/${renamedOutput}.zip`, + mirrorPath: `${renamedOutput}.zip`, + outputName: renamedOutput, + forkInstrumentation: "auto", + }]); + + expect(() => resolveBinary(fixture.relPath)).toThrow( + /resolver-owned program generation has no matching selected package projection/, + ); + rmSync(localMirror); + expect(() => resolveBinary(fixture.relPath)).toThrow( + /resolver-owned program generation has no matching selected package projection/, + ); + }); + + it("accepts local mirrors only from one direct immutable local generation", () => { + const fixture = createMultiOutputFixture(); + const canonicalRoot = fixtureLocalCanonicalRoot(fixture.name); + const mirrors = fixture.members.map((member) => + linkClosureMember(localBinariesDir(), member, canonicalRoot) + ); + const targets = mirrors.map((mirror) => realpathSync(mirror)); + + expect(resolveBinary(fixture.members[0]!.relPath)).toBe(targets[0]); + expect(tryResolveBinarySet( + fixture.members.map((member) => member.relPath), + )).toEqual(targets); + }); + + it("rejects stale fetched and local multi-member cache identities", () => { + const fixture = createMultiOutputFixture(); + const wrongCacheKey = "c".repeat(64); + const fetchedRoot = fixtureCanonicalRoot( + fixture.name, + "wasm32", + wrongCacheKey, + ); + const localRoot = fixtureLocalCanonicalRoot( + fixture.name, + "wasm32", + wrongCacheKey, + ); + for (const member of fixture.members) { + linkClosureMember(binariesDir(), member, fetchedRoot); + linkClosureMember(localBinariesDir(), member, localRoot); + } + + expect(() => resolveBinary(fixture.members[0]!.relPath)).toThrow( + /shared package identity rejected/, + ); + }); + + it("rejects fetched mirrors whose target is outside the canonical program cache", () => { + const fixture = createMultiOutputFixture(); + const arbitraryRoot = fixtureArbitraryRoot(); + for (const member of fixture.members) { + linkClosureMember(binariesDir(), member, arbitraryRoot); + } + + expect(() => resolveBinary(fixture.members[0]!.relPath)).toThrow( + /fetched mirror targets are not one canonical program-cache generation/, + ); + }); + + it("pins canonical member paths across a concurrent live-directory swap", () => { + const fixture = createMultiOutputFixture(); + const oldCanonicalRoot = fixtureLocalCanonicalRoot(fixture.name); + const oldMirrors = fixture.members.map((member) => + linkClosureMember( + localBinariesDir(), + member, + oldCanonicalRoot, + "generation", + ) + ); + const pinned = tryResolveBinarySet( + fixture.members.map((member) => member.relPath), + ); + expect(pinned).toEqual(oldMirrors.map((mirror) => realpathSync(mirror))); + + const newCanonicalRoot = fixtureLocalCanonicalRoot(fixture.name); + const liveDirectory = join( + localBinariesDir(), + "programs", + "wasm32", + fixture.name, + ); + const stagedDirectory = `${liveDirectory}.test-stage-${randomUUID()}`; + cleanupDirs.add(stagedDirectory); + const newTargets: string[] = []; + for (const member of fixture.members) { + const target = writeCanonicalMember( + newCanonicalRoot, + member.sourceArtifact, + "generation", + ); + newTargets.push(target); + const packageRelative = member.relPath.split("/").slice(3).join("/"); + const mirror = join(stagedDirectory, packageRelative); + mkdirSync(dirname(mirror), { recursive: true }); + symlinkSync(target, mirror); + } + const backupDirectory = `${liveDirectory}.test-backup-${randomUUID()}`; + cleanupDirs.add(backupDirectory); + renameSync(liveDirectory, backupDirectory); + renameSync(stagedDirectory, liveDirectory); + + expect(pinned).toEqual( + fixture.members.map((member) => + join(oldCanonicalRoot, ...member.sourceArtifact.split("/")) + ), + ); + expect(tryResolveBinarySet( + fixture.members.map((member) => member.relPath), + )).toEqual(newTargets); + }); + + it("uses the whole fetched closure when a local runtime member is absent", () => { + const fixture = createMultiOutputFixture(); + writeCandidate( + localBinariesDir(), + fixture.members[0]!.relPath, + new TextEncoder().encode("partial-local-output"), + ); + writeCandidate( + localBinariesDir(), + fixture.members[1]!.relPath, + new TextEncoder().encode("partial-local-output"), + ); + const canonicalRoot = fixtureCanonicalRoot(fixture.name); + const fetched = fixture.members.map((member) => + linkClosureMember(binariesDir(), member, canonicalRoot) + ); + + expect(resolveBinary(fixture.members[0]!.relPath)).toBe( + realpathSync(fetched[0]!), + ); + }); + + it("rejects preexisting same-tier symlinks into different canonical generations", () => { + const fixture = createMultiOutputFixture(); + const firstCanonicalRoot = fixtureLocalCanonicalRoot(fixture.name); + const secondCanonicalRoot = fixtureLocalCanonicalRoot(fixture.name); + linkClosureMember( + localBinariesDir(), + fixture.members[0]!, + firstCanonicalRoot, + ); + for (const member of fixture.members.slice(1)) { + linkClosureMember(localBinariesDir(), member, secondCanonicalRoot); + } + + expect(() => resolveBinary(fixture.members[0]!.relPath)).toThrow( + /shared package identity rejected: member symlinks target different canonical package generations/, + ); + }); + + it("requires each cache symlink to end in its declared source artifact path", () => { + const fixture = createMultiOutputFixture(); + const canonicalRoot = fixtureCanonicalRoot(fixture.name); + const firstMirror = linkClosureMember( + binariesDir(), + fixture.members[0]!, + canonicalRoot, + ); + rmSync(firstMirror); + symlinkSync( + writeCanonicalMember( + canonicalRoot, + "wrong/image.zip", + "wrong-source-path", + ), + firstMirror, + ); + for (const member of fixture.members.slice(1)) { + linkClosureMember(binariesDir(), member, canonicalRoot); + } + + expect(() => resolveBinary(fixture.members[0]!.relPath)).toThrow( + /does not target its declared source artifact artifacts\/image\.zip/, + ); + }); + + it("skips mutable real-file closures because they have no shared cache identity", () => { + const fixture = createMultiOutputFixture(); + for (const member of fixture.members) { + writeCandidate( + localBinariesDir(), + member.relPath, + new TextEncoder().encode("unidentified-local-copy"), + ); + } + const canonicalRoot = fixtureCanonicalRoot(fixture.name); + const fetched = fixture.members.map((member) => + linkClosureMember(binariesDir(), member, canonicalRoot) + ); + + expect(resolveBinary(fixture.members[0]!.relPath)).toBe( + realpathSync(fetched[0]!), + ); + }); + + it("does not treat mutable source-checkout wasm files as an installed package identity", () => { + const fixture = createMultiOutputFixture(); + const installedRoot = join(findRepoRoot(), "host", "wasm"); + fixture.members.map((member) => + writeCandidate( + installedRoot, + member.relPath, + new TextEncoder().encode("installed-package-member"), + ) + ); + + expect(() => resolveBinary(fixture.members[1]!.relPath)).toThrow( + /mutable source-checkout wasm tree/, + ); + }); + + it("rejects mixed files and symlinks in the installed package identity", () => { + const fixture = createMultiOutputFixture(); + const installedRoot = join(findRepoRoot(), "host", "wasm"); + writeCandidate( + installedRoot, + fixture.members[0]!.relPath, + new TextEncoder().encode("installed-package-member"), + ); + const canonicalRoot = fixtureCanonicalRoot(fixture.name); + for (const member of fixture.members.slice(1)) { + linkClosureMember(installedRoot, member, canonicalRoot); + } + + expect(() => resolveBinary(fixture.members[0]!.relPath)).toThrow( + /regular files and symlinks cannot share one package identity/, + ); + }); + + it("requires explicit set callers to request the complete package closure", () => { + const fixture = createMultiOutputFixture(); + expect(() => tryResolveBinarySet([ + fixture.members[0]!.relPath, + fixture.members[1]!.relPath, + ])).toThrow(/must resolve its complete declared closure/); + }); + + it("keeps an absent package-owned member on the package lookup path", () => { + const fixture = createMultiOutputFixture(); + expect(tryResolveBinary(fixture.members[0]!.relPath)).toBeNull(); + expect(() => resolveBinary(fixture.members[0]!.relPath)).toThrow( + new RegExp(`Package artifacts not found for ${fixture.name}`), + ); + expect(tryResolveBinarySet( + fixture.members.map((member) => member.relPath), + )).toBeNull(); + }); + + it("does not report an empty or dangling package mirror as absent", () => { + const fixture = createMultiOutputFixture(); + const liveDirectory = join( + localBinariesDir(), + "programs", + "wasm32", + fixture.name, + ); + mkdirSync(liveDirectory, { recursive: true }); + + expect(() => tryResolveBinary(fixture.members[0]!.relPath)).toThrow( + /Package artifact closure is incomplete/, + ); + expect(() => tryResolveBinarySet( + fixture.members.map((member) => member.relPath), + )).toThrow(/Package artifact closure is incomplete/); + + rmSync(liveDirectory, { recursive: true }); + for (const member of fixture.members) { + const mirror = candidatePath(localBinariesDir(), member.relPath); + mkdirSync(dirname(mirror), { recursive: true }); + symlinkSync(`${mirror}.missing-target`, mirror); + } + expect(() => tryResolveBinary(fixture.members[0]!.relPath)).toThrow( + /Package artifact closure is incomplete/, + ); + }); + it("returns a complete local closure from one provenance root", () => { const [wasmRel, dataRel] = fixtureClosureRelPaths([ "program.wasm", diff --git a/host/test/homebrew-vfs-image-save.test.ts b/host/test/homebrew-vfs-image-save.test.ts new file mode 100644 index 0000000000..e159a2701c --- /dev/null +++ b/host/test/homebrew-vfs-image-save.test.ts @@ -0,0 +1,76 @@ +import { + existsSync, + mkdtempSync, + rmSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { + saveVerifiedHomebrewVfsImage, +} from "../../images/vfs/scripts/build-homebrew-vfs-image"; +import { MemoryFileSystem } from "../src/vfs/memory-fs"; + +const MiB = 1024 * 1024; + +describe("Homebrew VFS image publication boundary", () => { + it("writes an image whose encoded ceiling matches its consumer contract", async () => { + const maxByteLength = 8 * MiB; + const fs = MemoryFileSystem.create( + new SharedArrayBuffer(1 * MiB, { maxByteLength }), + maxByteLength, + ); + const dir = mkdtempSync(join(tmpdir(), "homebrew-vfs-capacity-")); + const outFile = join(dir, "homebrew.vfs.zst"); + try { + const image = await saveVerifiedHomebrewVfsImage( + fs, + outFile, + { skipWasmArtifactCheck: true }, + maxByteLength, + ); + + expect( + MemoryFileSystem.readImageCapacity(image).maxByteLength, + ).toBe(maxByteLength); + expect(existsSync(outFile)).toBe(true); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("rejects a masked encoded ceiling before creating an output artifact", async () => { + const encodedMaxByteLength = 8 * MiB; + const consumerMaxByteLength = 4 * MiB; + const source = MemoryFileSystem.create( + new SharedArrayBuffer(1 * MiB, { + maxByteLength: encodedMaxByteLength, + }), + encodedMaxByteLength, + ); + const restored = MemoryFileSystem.fromImage(await source.saveImage(), { + maxByteLength: consumerMaxByteLength, + }); + expect(restored.statfs("/").blocks * restored.statfs("/").bsize).toBe( + consumerMaxByteLength, + ); + + const dir = mkdtempSync(join(tmpdir(), "homebrew-vfs-capacity-drift-")); + const outFile = join(dir, "homebrew.vfs.zst"); + try { + await expect( + saveVerifiedHomebrewVfsImage( + restored, + outFile, + { skipWasmArtifactCheck: true }, + consumerMaxByteLength, + ), + ).rejects.toThrow( + /has a 8388608-byte VFS capacity; 4194304 bytes are required/, + ); + expect(existsSync(outFile)).toBe(false); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/host/test/mariadb-test-source-copy.test.ts b/host/test/mariadb-test-source-copy.test.ts new file mode 100644 index 0000000000..17c4def54c --- /dev/null +++ b/host/test/mariadb-test-source-copy.test.ts @@ -0,0 +1,165 @@ +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { describe, expect, it } from "vitest"; +import { + copyMariaDbTestSources, +} from "../../images/vfs/scripts/mariadb-test-source-copy"; +import { MemoryFileSystem } from "../src/vfs/memory-fs"; + +const O_RDONLY = 0; + +function createFs(): MemoryFileSystem { + return MemoryFileSystem.create(new SharedArrayBuffer(4 * 1024 * 1024)); +} + +function readVfsText(fs: MemoryFileSystem, path: string): string { + const bytes = new Uint8Array(fs.stat(path).size); + const fd = fs.open(path, O_RDONLY, 0); + try { + const count = fs.read(fd, bytes, null, bytes.byteLength); + if (count !== bytes.byteLength) { + throw new Error(`short test read for ${path}`); + } + } finally { + fs.close(fd); + } + return new TextDecoder().decode(bytes); +} + +function withMariaDbSource(run: (root: string) => void): void { + const root = mkdtempSync(join(tmpdir(), "mariadb-test-source-")); + try { + mkdirSync(join(root, "main")); + mkdirSync(join(root, "include")); + mkdirSync(join(root, "std_data")); + writeFileSync(join(root, "main", "selected.test"), "selected"); + writeFileSync(join(root, "main", "other.test"), "other"); + writeFileSync(join(root, "main", "README"), "not a test"); + writeFileSync(join(root, "include", "helper.inc"), "include"); + writeFileSync(join(root, "std_data", "fixture.dat"), "fixture"); + run(root); + } finally { + rmSync(root, { recursive: true, force: true }); + } +} + +describe("MariaDB test source closure", () => { + it("keeps the artifact and browser runner on one curated selection", () => { + const repoRoot = resolve(import.meta.dirname, "../.."); + const builder = readFileSync( + join(repoRoot, "images/vfs/scripts/build-mariadb-test-vfs-image.ts"), + "utf8", + ); + const runner = readFileSync( + join(repoRoot, "scripts/run-browser-mariadb-tests.sh"), + "utf8", + ); + const builderBody = builder.match( + /const CURATED_TESTS = \[([\s\S]*?)\];/, + )?.[1]; + const runnerBody = runner.match( + /CURATED_TESTS=\(([\s\S]*?)\n\)/, + )?.[1]; + expect(builderBody, "builder curated test list").toBeDefined(); + expect(runnerBody, "runner curated test list").toBeDefined(); + + const builderTests = Array.from( + builderBody!.matchAll(/"([^"]+)"/g), + (match) => match[1], + ); + const runnerTests = runnerBody!.trim().split(/\s+/); + expect(builderTests).toEqual(runnerTests); + expect(new Set(builderTests).size).toBe(builderTests.length); + }); + + it("copies every curated test and both required fixture trees", () => { + withMariaDbSource((root) => { + const fs = createFs(); + + expect(copyMariaDbTestSources(fs, root, { + includeAll: false, + curatedTests: ["selected"], + })).toBe(1); + + expect(readVfsText(fs, "/mysql-test/main/selected.test")).toBe("selected"); + expect(() => fs.stat("/mysql-test/main/other.test")).toThrow(); + expect(readVfsText(fs, "/mysql-test/include/helper.inc")).toBe("include"); + expect(readVfsText(fs, "/mysql-test/std_data/fixture.dat")).toBe("fixture"); + }); + }); + + it("copies every .test entry in all-tests mode and ignores unrelated files", () => { + withMariaDbSource((root) => { + const fs = createFs(); + + expect(copyMariaDbTestSources(fs, root, { + includeAll: true, + curatedTests: [], + })).toBe(2); + + expect(readVfsText(fs, "/mysql-test/main/selected.test")).toBe("selected"); + expect(readVfsText(fs, "/mysql-test/main/other.test")).toBe("other"); + expect(() => fs.stat("/mysql-test/main/README")).toThrow(); + }); + }); + + it("rejects a missing declared curated test", () => { + withMariaDbSource((root) => { + unlinkSync(join(root, "main", "selected.test")); + + expect(() => copyMariaDbTestSources(createFs(), root, { + includeAll: false, + curatedTests: ["selected"], + })).toThrow(/selected\.test/); + }); + }); + + it("rejects a non-regular .test source", () => { + withMariaDbSource((root) => { + unlinkSync(join(root, "main", "selected.test")); + mkdirSync(join(root, "main", "selected.test")); + + expect(() => copyMariaDbTestSources(createFs(), root, { + includeAll: false, + curatedTests: ["selected"], + })).toThrow(/MariaDB test source entry is not a regular file/); + }); + }); + + it.each(["include", "std_data"] as const)( + "rejects a missing required %s fixture tree", + (fixtureName) => { + withMariaDbSource((root) => { + rmSync(join(root, fixtureName), { recursive: true }); + + expect(() => copyMariaDbTestSources(createFs(), root, { + includeAll: false, + curatedTests: ["selected"], + })).toThrow(); + }); + }, + ); + + it.each(["include", "std_data"] as const)( + "rejects an empty required %s fixture tree", + (fixtureName) => { + withMariaDbSource((root) => { + rmSync(join(root, fixtureName), { recursive: true }); + mkdirSync(join(root, fixtureName)); + + expect(() => copyMariaDbTestSources(createFs(), root, { + includeAll: false, + curatedTests: ["selected"], + })).toThrow(/Required MariaDB test fixture tree is empty/); + }); + }, + ); +}); diff --git a/host/test/run-example-resolver.test.ts b/host/test/run-example-resolver.test.ts index 0e38a66188..90b7753fe5 100644 --- a/host/test/run-example-resolver.test.ts +++ b/host/test/run-example-resolver.test.ts @@ -1,5 +1,12 @@ import { describe, expect, it } from "vitest"; -import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join, dirname } from "node:path"; import { fileURLToPath } from "node:url"; @@ -12,6 +19,62 @@ const runExample = join(repoRoot, "examples", "run-example.ts"); const spawnSmokeWasm = join(repoRoot, "examples", "spawn-smoke.wasm"); describe("run-example exec resolver", () => { + it("loads unrelated examples without probing legacy flat paths for multi-member packages", () => { + const source = readFileSync(runExample, "utf8"); + const projection = JSON.parse( + readFileSync( + join(repoRoot, "packages", "registry", "program-packages.json"), + "utf8", + ), + ) as { + packages: Record; + }>; + }; + const probes = new Set( + [...source.matchAll(/tryResolveBinary\("([^"]+)"\)/g)] + .map((match) => match[1]), + ); + const legacyMultiMemberProbes: string[] = []; + for (const packageProjection of Object.values(projection.packages)) { + if (packageProjection.members.length < 2) continue; + for (const member of packageProjection.members) { + if (member.kind !== "output") continue; + const basename = member.mirrorPath.split("/").at(-1); + if (basename && probes.has(`programs/${basename}`)) { + legacyMultiMemberProbes.push(`programs/${basename}`); + } + } + } + expect(legacyMultiMemberProbes).toEqual([]); + + const cacheRoot = mkdtempSync(join(tmpdir(), "kandelo-run-example-load-")); + try { + const result = spawnSync( + process.execPath, + ["--import", "tsx/esm", runExample], + { + cwd: repoRoot, + env: { + ...process.env, + WASM_POSIX_BINARY_CACHE_ROOT: cacheRoot, + WASM_POSIX_DEPS_REGISTRY: "packages/registry", + }, + encoding: "utf8", + timeout: 30_000, + }, + ); + + expect(result.status).toBe(1); + expect(result.stderr).toContain( + "Usage: npx tsx examples/run-example.ts ", + ); + expect(result.stderr).not.toContain("Legacy flat resolver path"); + } finally { + rmSync(cacheRoot, { recursive: true, force: true }); + } + }); + it("compares canonical workdir paths without allowing symlink escapes", () => { const tempDir = mkdtempSync(join(tmpdir(), "kandelo-workdir-boundary-")); const realWorkdir = join(tempDir, "real-workdir"); diff --git a/host/test/shell-vfs-build.test.ts b/host/test/shell-vfs-build.test.ts index 61c3bb4c58..f01cc4023b 100644 --- a/host/test/shell-vfs-build.test.ts +++ b/host/test/shell-vfs-build.test.ts @@ -1,10 +1,17 @@ import { zstdCompressSync } from "node:zlib"; -import { readdirSync, readFileSync } from "node:fs"; +import { mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; -import { loadShellBaseFileSystemFromImage } from "../../images/vfs/scripts/shell-vfs-build"; +import { + loadShellBaseFileSystemFromImage, + saveShellDerivedVfsImage, +} from "../../images/vfs/scripts/shell-vfs-build"; import { MemoryFileSystem } from "../src/vfs/memory-fs"; import type { ZipEntry } from "../src/vfs/zip"; +import { + SHELL_DERIVED_VFS_PROFILE_MAX_BYTES, +} from "../../web-libs/kandelo-session/src/vfs-capacity"; const MiB = 1024 * 1024; const O_RDONLY = 0x0000; @@ -151,4 +158,71 @@ describe("shell VFS base composition", () => { expect(restored.sharedBuffer.maxByteLength).toBe(8 * MiB); expectContentsPreserved(restored); }); + + it("rejects an image that drifts from the standard product capacity", async () => { + const largerProfile = 1024 * MiB; + const fs = MemoryFileSystem.create( + new SharedArrayBuffer(16 * MiB, { maxByteLength: largerProfile }), + largerProfile, + ); + + await expect( + saveShellDerivedVfsImage(fs, "/tmp/not-written.vfs.zst"), + ).rejects.toThrow( + new RegExp( + `${largerProfile}-byte VFS capacity.*` + + `${SHELL_DERIVED_VFS_PROFILE_MAX_BYTES} bytes are required`, + ), + ); + }); + + it("rejects an explicit product profile below the standard capacity", () => { + const smallerProfile = 512 * MiB; + const fs = MemoryFileSystem.create( + new SharedArrayBuffer(16 * MiB, { maxByteLength: smallerProfile }), + smallerProfile, + ); + + expect(() => + saveShellDerivedVfsImage(fs, "/tmp/not-written.vfs.zst", { + expectedMaxByteLength: smallerProfile, + }) + ).toThrow( + new RegExp( + `must use the standard ${SHELL_DERIVED_VFS_PROFILE_MAX_BYTES}-byte ` + + "product profile or an explicitly reviewed, strictly larger profile", + ), + ); + }); + + const capacityProfiles: Array<[string, number, number | undefined]> = [ + ["the standard profile", SHELL_DERIVED_VFS_PROFILE_MAX_BYTES, undefined], + ["an explicit larger product profile", 1024 * MiB, 1024 * MiB], + ]; + + it.each(capacityProfiles)("saves %s only under its exact declared capacity", async ( + _label, + profileMaxBytes, + expectedMaxByteLength, + ) => { + const fs = MemoryFileSystem.create( + new SharedArrayBuffer(16 * MiB, { maxByteLength: profileMaxBytes }), + profileMaxBytes, + ); + writeFile(fs, "/product.txt", "complete product"); + const dir = mkdtempSync(join(tmpdir(), "shell-derived-capacity-")); + try { + const image = await saveShellDerivedVfsImage( + fs, + join(dir, "product.vfs.zst"), + expectedMaxByteLength === undefined ? {} : { expectedMaxByteLength }, + ); + + expect(MemoryFileSystem.readImageCapacity(image).maxByteLength).toBe( + profileMaxBytes, + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); }); diff --git a/host/test/vfs-image-helpers.test.ts b/host/test/vfs-image-helpers.test.ts new file mode 100644 index 0000000000..690049e88a --- /dev/null +++ b/host/test/vfs-image-helpers.test.ts @@ -0,0 +1,183 @@ +import { + chmodSync, + mkdirSync, + mkdtempSync, + rmSync, + symlinkSync, + unlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { createServer } from "node:net"; +import { join } from "node:path"; +import { describe, expect, it, vi } from "vitest"; +import { walkAndWrite } from "../../images/vfs/scripts/vfs-image-helpers"; +import { MemoryFileSystem } from "../src/vfs/memory-fs"; + +const O_RDONLY = 0; + +function readFile(fs: MemoryFileSystem, path: string): Uint8Array { + const size = fs.stat(path).size; + const bytes = new Uint8Array(size); + const fd = fs.open(path, O_RDONLY, 0); + try { + const count = fs.read(fd, bytes, null, bytes.byteLength); + if (count !== bytes.byteLength) { + throw new Error(`short test read: ${count} of ${bytes.byteLength}`); + } + } finally { + fs.close(fd); + } + return bytes; +} + +function withSourceTree(run: (root: string) => void): void { + const root = mkdtempSync(join(tmpdir(), "vfs-walk-source-")); + try { + mkdirSync(join(root, "nested")); + writeFileSync(join(root, "keep.txt"), "kept"); + writeFileSync(join(root, "skip.txt"), "skipped"); + writeFileSync(join(root, "nested", "tool"), "tool"); + chmodSync(join(root, "nested"), 0o710); + chmodSync(join(root, "nested", "tool"), 0o751); + symlinkSync("keep.txt", join(root, "alias")); + run(root); + } finally { + rmSync(root, { recursive: true, force: true }); + } +} + +describe("walkAndWrite", () => { + it("copies files, directories, modes, and requested symlinks while honoring exclusions", () => { + withSourceTree((root) => { + const fs = MemoryFileSystem.create(new SharedArrayBuffer(4 * 1024 * 1024)); + + const count = walkAndWrite(fs, root, "/payload", { + exclude: (path) => path === "skip.txt", + preserveMode: true, + preserveSymlinks: true, + }); + + expect(count).toBe(3); + expect( + new TextDecoder().decode(readFile(fs, "/payload/keep.txt")), + ).toBe("kept"); + expect( + new TextDecoder().decode(readFile(fs, "/payload/nested/tool")), + ).toBe("tool"); + expect(fs.stat("/payload/nested").mode & 0o7777).toBe(0o710); + expect(fs.stat("/payload/nested/tool").mode & 0o7777).toBe(0o751); + expect(fs.readlink("/payload/alias")).toBe("keep.txt"); + expect(() => fs.lstat("/payload/skip.txt")).toThrow(); + }); + }); + + it("rejects an unexcluded symlink unless preservation is requested", () => { + withSourceTree((root) => { + const fs = MemoryFileSystem.create(new SharedArrayBuffer(4 * 1024 * 1024)); + + expect(() => walkAndWrite(fs, root, "/payload")).toThrow( + new RegExp( + `VFS image source symlink requires preserveSymlinks or an explicit exclude: ` + + `${join(root, "alias")}`, + ), + ); + }); + }); + + it("omits a symlink only through an explicit exclusion", () => { + withSourceTree((root) => { + const fs = MemoryFileSystem.create(new SharedArrayBuffer(4 * 1024 * 1024)); + + const count = walkAndWrite(fs, root, "/payload", { + exclude: (path) => path === "alias", + }); + + expect(count).toBe(3); + expect(() => fs.lstat("/payload/alias")).toThrow(); + expect(fs.stat("/payload/nested/tool").mode & 0o7777).toBe(0o644); + }); + }); + + it("propagates a host file read failure", () => { + withSourceTree((root) => { + const fs = MemoryFileSystem.create(new SharedArrayBuffer(4 * 1024 * 1024)); + + expect(() => + walkAndWrite(fs, root, "/payload", { + exclude: (path) => { + if (path === "alias") return true; + if (path === "keep.txt") unlinkSync(join(root, path)); + return false; + }, + }) + ).toThrow(); + }); + }); + + it("propagates a host symlink read failure", () => { + withSourceTree((root) => { + const fs = MemoryFileSystem.create(new SharedArrayBuffer(4 * 1024 * 1024)); + + expect(() => + walkAndWrite(fs, root, "/payload", { + preserveSymlinks: true, + exclude: (path) => { + if (path === "alias") unlinkSync(join(root, path)); + return false; + }, + }) + ).toThrow(); + }); + }); + + it("propagates a VFS write failure instead of silently omitting the file", () => { + const root = mkdtempSync(join(tmpdir(), "vfs-walk-error-")); + try { + writeFileSync(join(root, "payload.bin"), new Uint8Array([1, 2, 3])); + const failure = new Error("synthetic VFS write failure"); + const fs = { + mkdir: vi.fn(), + open: vi.fn(() => { throw failure; }), + } as unknown as MemoryFileSystem; + + expect(() => walkAndWrite(fs, root, "/payload")).toThrow(failure); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it("propagates terminal ENOSPC after a partial product-tree write", () => { + const root = mkdtempSync(join(tmpdir(), "vfs-walk-enospc-")); + try { + writeFileSync(join(root, "payload.bin"), new Uint8Array(1024 * 1024)); + const fs = MemoryFileSystem.create(new SharedArrayBuffer(128 * 1024)); + + expect(() => walkAndWrite(fs, root, "/payload")).toThrow(); + expect(fs.stat("/payload/payload.bin").size).toBeGreaterThan(0); + expect(fs.stat("/payload/payload.bin").size).toBeLessThan(1024 * 1024); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it("rejects a source entry type that a VFS image cannot represent", async () => { + const root = mkdtempSync(join(tmpdir(), "vfs-walk-socket-")); + const socketPath = join(root, "runtime.sock"); + const server = createServer(); + try { + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(socketPath, resolve); + }); + const fs = MemoryFileSystem.create(new SharedArrayBuffer(4 * 1024 * 1024)); + + expect(() => walkAndWrite(fs, root, "/payload")).toThrow( + new RegExp(`Unsupported VFS image source entry: ${socketPath}`), + ); + } finally { + await new Promise((resolve) => server.close(() => resolve())); + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/host/test/vfs-image.test.ts b/host/test/vfs-image.test.ts index e52738c735..433e107897 100644 --- a/host/test/vfs-image.test.ts +++ b/host/test/vfs-image.test.ts @@ -1,11 +1,18 @@ import { describe, it, expect, vi } from "vitest"; import { zstdCompressSync } from "node:zlib"; -import { mkdtempSync, readFileSync, rmSync, statSync } from "node:fs"; +import { + existsSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { ABI_VERSION } from "../src/generated/abi"; import { MemoryFileSystem } from "../src/vfs/memory-fs"; import { + assertVfsImageCapacity, assertVfsImageHeadroom, saveImage, sourceDateEpochMilliseconds, @@ -94,6 +101,81 @@ function stripStandaloneLazyIdentity(image: Uint8Array): Uint8Array { describe("VFS image save/restore", () => { describe("product image runtime headroom", () => { + it("validates the serialized product capacity contract and reports drift", async () => { + const mfs = createMemfs(); + const image = await mfs.saveImage(); + const maxByteLength = + MemoryFileSystem.readImageCapacity(image).maxByteLength; + + expect(() => + assertVfsImageCapacity(image, maxByteLength, "test image") + ).not.toThrow(); + expect(() => + assertVfsImageCapacity(image, maxByteLength + 4096, "test image") + ).toThrow(/test image has a .* VFS capacity; .* required/); + }); + + it.each([-1, 0, 1.5, Number.NaN, Number.MAX_SAFE_INTEGER + 1])( + "rejects invalid product capacity %s", + async (maxByteLength) => { + const image = await createMemfs().saveImage(); + expect(() => + assertVfsImageCapacity(image, maxByteLength, "test image") + ).toThrow(/expectedMaxByteLength must be a positive safe integer/); + }, + ); + + it("rejects malformed serialized capacity state", () => { + expect(() => + assertVfsImageCapacity(new Uint8Array(0), 1, "test image") + ).toThrow(/VFS image too small/); + }); + + it("rejects an encoded ceiling hidden by a smaller runtime buffer before writing", async () => { + const MiB = 1024 * 1024; + const encodedMaxBytes = 8 * MiB; + const runtimeMaxBytes = 4 * MiB; + const source = MemoryFileSystem.create( + new SharedArrayBuffer(1 * MiB, { maxByteLength: encodedMaxBytes }), + encodedMaxBytes, + ); + const sourceImage = await source.saveImage(); + const restored = MemoryFileSystem.fromImage(sourceImage, { + maxByteLength: runtimeMaxBytes, + }); + expect(restored.statfs("/").blocks * restored.statfs("/").bsize).toBe( + runtimeMaxBytes, + ); + + const maskedImage = await restored.saveImage(); + expect(MemoryFileSystem.readImageCapacity(maskedImage).maxByteLength).toBe( + encodedMaxBytes, + ); + expect(() => + assertVfsImageFitsProfile( + MemoryFileSystem.readImageCapacity(maskedImage), + runtimeMaxBytes, + undefined, + "masked.vfs.zst", + ) + ).toThrow(/requires 8388608 VFS bytes, but its profile permits 4194304/); + + const dir = mkdtempSync(join(tmpdir(), "vfs-masked-capacity-")); + const outFile = join(dir, "masked.vfs.zst"); + try { + await expect( + saveImage(restored, outFile, { + expectedMaxByteLength: runtimeMaxBytes, + }), + ).rejects.toThrow( + /has a 8388608-byte VFS capacity; 4194304 bytes are required/, + ); + expect(existsSync(outFile)).toBe(false); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + it("checks free blocks and free inodes as independent resources", () => { const mfs = createMemfs(); const stats = mfs.statfs("/"); diff --git a/images/vfs/scripts/build-erlang-vfs-image.ts b/images/vfs/scripts/build-erlang-vfs-image.ts index 7f8a151f38..f5947da012 100644 --- a/images/vfs/scripts/build-erlang-vfs-image.ts +++ b/images/vfs/scripts/build-erlang-vfs-image.ts @@ -80,7 +80,6 @@ async function main() { const otpRoot = "/usr/local/lib/erlang"; ensureDirRecursive(fs, otpRoot); const totalFiles = walkAndWrite(fs, INSTALL_DIR, otpRoot, { - failOnError: true, preserveMode: true, preserveSymlinks: true, }); diff --git a/images/vfs/scripts/build-homebrew-vfs-image.ts b/images/vfs/scripts/build-homebrew-vfs-image.ts index db8daf14b7..508f36bafb 100644 --- a/images/vfs/scripts/build-homebrew-vfs-image.ts +++ b/images/vfs/scripts/build-homebrew-vfs-image.ts @@ -68,6 +68,7 @@ import { ensureDirRecursive, saveImage, sourceDateEpochMilliseconds, + type SaveImageOptions, writeVfsBinary, } from "./vfs-image-helpers"; @@ -127,6 +128,23 @@ export type HomebrewVfsImageMaterializer = ( options: HomebrewVfsImageMaterializationOptions, ) => Promise; +/** + * Serialize a Homebrew product image only when its encoded SharedFS ceiling + * matches the consumer contract before compression, directory creation, or + * output writes. + */ +export async function saveVerifiedHomebrewVfsImage( + fs: MemoryFileSystem, + outFile: string, + options: Omit, + expectedMaxByteLength: number, +): Promise { + return saveImage(fs, outFile, { + ...options, + expectedMaxByteLength, + }); +} + const DEFAULT_MAX_BYTES = 128 * 1024 * 1024; const SHARED_FS_BLOCK_BYTES = 4096; const HOMEBREW_COMPOSITION_PATH = "/etc/kandelo/homebrew-vfs.json"; @@ -340,7 +358,7 @@ export async function runHomebrewVfsImageBuilder( writeVfsBinary(fs, KANDELO_DEMO_CONFIG_PATH, demoConfig.source, 0o644); } - const imageBytes = await saveImage(fs, options.out, { + const imageBytes = await saveVerifiedHomebrewVfsImage(fs, options.out, { normalizeTimestampsMs: sourceDateEpochMilliseconds( process.env.SOURCE_DATE_EPOCH, ), @@ -432,7 +450,7 @@ export async function runHomebrewVfsImageBuilder( })), }, }, - }); + }, maxByteLength); const imageCapacity = MemoryFileSystem.readImageCapacity(imageBytes); if (imageCapacity.maxByteLength !== maxByteLength) { throw new Error( diff --git a/images/vfs/scripts/build-mariadb-test-vfs-image.ts b/images/vfs/scripts/build-mariadb-test-vfs-image.ts index edfcd659ff..d788b71409 100644 --- a/images/vfs/scripts/build-mariadb-test-vfs-image.ts +++ b/images/vfs/scripts/build-mariadb-test-vfs-image.ts @@ -15,8 +15,8 @@ * npx tsx images/vfs/scripts/build-mariadb-test-vfs-image.ts # curated tests * npx tsx images/vfs/scripts/build-mariadb-test-vfs-image.ts --all # ALL tests */ -import { readFileSync, readdirSync, lstatSync, existsSync } from "node:fs"; -import { join, resolve } from "node:path"; +import { readFileSync, existsSync } from "node:fs"; +import { join } from "node:path"; import { MemoryFileSystem } from "../../../host/src/vfs/memory-fs"; import { ensureDir, @@ -26,9 +26,10 @@ import { symlink, } from "../../../host/src/vfs/image-helpers"; import { resolveBinary, findRepoRoot } from "../../../host/src/binary-resolver"; -import { saveImage, walkAndWrite } from "./vfs-image-helpers"; +import { saveImage } from "./vfs-image-helpers"; import { addDinitInit, type DinitService } from "./dinit-image-helpers"; import { prepareMariadbWritableDirectories } from "./mariadb-image-helpers"; +import { copyMariaDbTestSources } from "./mariadb-test-source-copy"; import { ensureSourceExtract } from "./source-extract-helper"; const REPO_ROOT = findRepoRoot(); @@ -65,7 +66,7 @@ const COREUTILS_SYMLINK_NAMES = [ "md5sum", "seq", "test", "[", ]; -// 185 tests verified to pass in headless Chromium with MariaDB on kandelo. +// 184 tests verified to pass in headless Chromium with MariaDB on kandelo. const CURATED_TESTS = [ "1st", "adddate_454", "almost_full", "alter_table_combinations", "alter_table_lock", "alter_table_mdev539_maria", @@ -113,7 +114,7 @@ const CURATED_TESTS = [ "set_statement_notembedded", "show_create_user", "show_function_with_pad_char_to_full_length", "show_row_order-9226", "signal_demo1", "signal_demo2", "signal_demo3", - "signal_sqlmode", "simple_select", "single_delete_update", + "signal_sqlmode", "single_delete_update", "skip_log_bin", "sp-bugs2", "sp-condition-handler", "sp-destruct", "sp-memory-leak", "sp-no-code", "sp-no-valgrind", "sp-ucs2", "sp-vars", "sp_gis", "sp_missing_4665", "sql_mode_pad_char_to_full_length", @@ -268,53 +269,21 @@ kill -KILL $PID 2>/dev/null exit 0 `); - // Test files - ensureDirRecursive(fs, "/mysql-test/main"); - let testCount = 0; - - if (includeAll) { - console.log(" Writing ALL .test files from main/..."); - const mainDir = resolve(MYSQL_TEST_DIR, "main"); - for (const name of readdirSync(mainDir).sort()) { - if (!name.endsWith(".test")) continue; - const full = join(mainDir, name); - try { - const stat = lstatSync(full); - if (!stat.isFile()) continue; - const data = readFileSync(full); - writeVfsBinary(fs, `/mysql-test/main/${name}`, new Uint8Array(data), 0o644); - testCount++; - } catch { /* skip */ } - } - } else { - console.log(" Writing curated test files..."); - for (const name of CURATED_TESTS) { - const testFile = resolve(MYSQL_TEST_DIR, "main", `${name}.test`); - if (existsSync(testFile)) { - const data = readFileSync(testFile); - writeVfsBinary(fs, `/mysql-test/main/${name}.test`, new Uint8Array(data), 0o644); - testCount++; - } - } - } + console.log( + includeAll + ? " Writing ALL .test files and required fixtures..." + : " Writing curated .test files and required fixtures...", + ); + const testCount = copyMariaDbTestSources(fs, MYSQL_TEST_DIR, { + includeAll, + curatedTests: CURATED_TESTS, + }); console.log(` ${testCount} test files`); // Setup and reset SQL test files (run by the page after server-ready) writeVfsFile(fs, "/mysql-test/main/__setup.test", SETUP_SQL); writeVfsFile(fs, "/mysql-test/main/__reset.test", RESET_SQL); - // Include + std_data directories - const includeDir = resolve(MYSQL_TEST_DIR, "include"); - if (existsSync(includeDir)) { - console.log(" Writing include/ directory..."); - walkAndWrite(fs, includeDir, "/mysql-test/include"); - } - const stdDataDir = resolve(MYSQL_TEST_DIR, "std_data"); - if (existsSync(stdDataDir)) { - console.log(" Writing std_data/ directory..."); - walkAndWrite(fs, stdDataDir, "/mysql-test/std_data"); - } - // dinit service tree (no auto-boot — page passes target service as argv). // We use the default boot:true here because the page only ever wants // the mariadb tree up; no engine selection like the mariadb demo. diff --git a/images/vfs/scripts/build-php-test-vfs-image.ts b/images/vfs/scripts/build-php-test-vfs-image.ts index 68f66178d2..b01ec9b2bb 100644 --- a/images/vfs/scripts/build-php-test-vfs-image.ts +++ b/images/vfs/scripts/build-php-test-vfs-image.ts @@ -406,7 +406,6 @@ async function main() { exclude: (childRel) => shouldExclude(phpSrc, rel ? `${rel}/${childRel}` : childRel), preserveMode: true, preserveSymlinks: true, - failOnError: true, }); } if (supportDirs.length > 0) { diff --git a/images/vfs/scripts/mariadb-test-source-copy.ts b/images/vfs/scripts/mariadb-test-source-copy.ts new file mode 100644 index 0000000000..5dc975a79b --- /dev/null +++ b/images/vfs/scripts/mariadb-test-source-copy.ts @@ -0,0 +1,75 @@ +import { + lstatSync, + readFileSync, + readdirSync, +} from "node:fs"; +import { join } from "node:path"; +import { type MemoryFileSystem } from "../../../host/src/vfs/memory-fs"; +import { + ensureDirRecursive, + walkAndWrite, + writeVfsBinary, +} from "./vfs-image-helpers"; + +export interface MariaDbTestSourceCopyOptions { + includeAll: boolean; + curatedTests: readonly string[]; +} + +function requireRegularTestSource(path: string): Uint8Array { + const stat = lstatSync(path); + if (!stat.isFile()) { + throw new Error(`MariaDB test source entry is not a regular file: ${path}`); + } + return new Uint8Array(readFileSync(path)); +} + +function copyRequiredFixtureTree( + fs: MemoryFileSystem, + mysqlTestDir: string, + name: "include" | "std_data", +): void { + const source = join(mysqlTestDir, name); + const count = walkAndWrite(fs, source, `/mysql-test/${name}`); + if (count === 0) { + throw new Error(`Required MariaDB test fixture tree is empty: ${source}`); + } +} + +/** + * Copy the declared MariaDB test closure without best-effort omissions. + * + * The upstream source is pinned, so a missing curated test or fixture tree is + * a broken build input rather than an optional feature. Any host read, source + * type, or VFS write failure must abort the artifact build. + */ +export function copyMariaDbTestSources( + fs: MemoryFileSystem, + mysqlTestDir: string, + options: MariaDbTestSourceCopyOptions, +): number { + const mainDir = join(mysqlTestDir, "main"); + const testFiles = options.includeAll + ? readdirSync(mainDir).filter((name) => name.endsWith(".test")).sort() + : options.curatedTests.map((name) => `${name}.test`); + if (testFiles.length === 0) { + throw new Error(`No MariaDB test sources were selected from ${mainDir}`); + } + if (new Set(testFiles).size !== testFiles.length) { + throw new Error("MariaDB test source selection contains duplicate entries"); + } + + ensureDirRecursive(fs, "/mysql-test/main"); + for (const fileName of testFiles) { + writeVfsBinary( + fs, + `/mysql-test/main/${fileName}`, + requireRegularTestSource(join(mainDir, fileName)), + 0o644, + ); + } + + copyRequiredFixtureTree(fs, mysqlTestDir, "include"); + copyRequiredFixtureTree(fs, mysqlTestDir, "std_data"); + return testFiles.length; +} diff --git a/images/vfs/scripts/shell-vfs-build.ts b/images/vfs/scripts/shell-vfs-build.ts index 8811c4766a..16d70dceef 100644 --- a/images/vfs/scripts/shell-vfs-build.ts +++ b/images/vfs/scripts/shell-vfs-build.ts @@ -41,6 +41,7 @@ import type { SaveImageOptions } from "./vfs-image-helpers"; import { SHELL_DERIVED_VFS_MIN_FREE_BYTES, SHELL_DERIVED_VFS_MIN_FREE_INODES, + SHELL_DERIVED_VFS_PROFILE_MAX_BYTES, } from "../../../web-libs/kandelo-session/src/vfs-capacity"; function depEnvKey(name: string): string { @@ -91,10 +92,34 @@ export function loadShellBaseFileSystem(maxByteLength: number): MemoryFileSystem export function saveShellDerivedVfsImage( fs: MemoryFileSystem, outFile: string, - options: Omit = {}, + options: Omit< + SaveImageOptions, + "headroom" | "expectedMaxByteLength" + > & { + /** Explicit escape hatch for a reviewed product profile above 768 MiB. */ + expectedMaxByteLength?: number; + } = {}, ): Promise { + const { + expectedMaxByteLength = SHELL_DERIVED_VFS_PROFILE_MAX_BYTES, + ...saveOptions + } = options; + if ( + expectedMaxByteLength !== SHELL_DERIVED_VFS_PROFILE_MAX_BYTES && + ( + !Number.isSafeInteger(expectedMaxByteLength) || + expectedMaxByteLength <= SHELL_DERIVED_VFS_PROFILE_MAX_BYTES + ) + ) { + throw new Error( + `${outFile} expectedMaxByteLength must use the standard ` + + `${SHELL_DERIVED_VFS_PROFILE_MAX_BYTES}-byte product profile or ` + + "an explicitly reviewed, strictly larger profile", + ); + } return saveImage(fs, outFile, { - ...options, + ...saveOptions, + expectedMaxByteLength, headroom: { minimumFreeBytes: SHELL_DERIVED_VFS_MIN_FREE_BYTES, minimumFreeInodes: SHELL_DERIVED_VFS_MIN_FREE_INODES, diff --git a/images/vfs/scripts/vfs-image-helpers.ts b/images/vfs/scripts/vfs-image-helpers.ts index 7313bf0a52..8e3c28b1d8 100644 --- a/images/vfs/scripts/vfs-image-helpers.ts +++ b/images/vfs/scripts/vfs-image-helpers.ts @@ -13,9 +13,9 @@ import { } from "fs"; import { join, relative } from "path"; import { zstdCompressSync, constants as zlibConstants } from "node:zlib"; -import type { +import { MemoryFileSystem, - VfsImageMetadata, + type VfsImageMetadata, } from "../../../host/src/vfs/memory-fs"; import { describeWasmArtifactPolicyFailures } from "../../../host/src/constants"; import { ABI_VERSION } from "../../../host/src/generated/abi"; @@ -33,12 +33,17 @@ import { writeVfsBinary, ensureDirRecursive } from "../../../host/src/vfs/image- export interface WalkOptions { exclude?: (relPath: string) => boolean; preserveMode?: boolean; - preserveSymlinks?: boolean; - failOnError?: boolean; + preserveSymlinks?: true; } /** * Walk a host directory and write all files into the VFS under mountPrefix. + * Any host-read or VFS-write failure aborts the build. Product images must not + * silently omit an entry; callers that intentionally exclude content must do + * so through `exclude`. + * + * Unexcluded symlinks must be preserved explicitly. Silently omitting a + * representable entry would produce an incomplete product image. * Returns the number of files written. */ export function walkAndWrite( @@ -56,32 +61,32 @@ export function walkAndWrite( const rel = relative(rootDir, full); const mountPath = mountPrefix + "/" + rel; - try { - const lstat = lstatSync(full); - if (opts?.exclude?.(rel)) continue; - if (lstat.isSymbolicLink()) { - if (opts?.preserveSymlinks) { - ensureDirRecursive(fs, mountPath.slice(0, mountPath.lastIndexOf("/")) || "/"); - fs.symlink(readlinkSync(full), mountPath); - count++; - } - } else if (lstat.isDirectory()) { - ensureDirRecursive(fs, mountPath); - if (opts?.preserveMode) fs.chmod(mountPath, lstat.mode & 0o7777); - walk(full); - } else if (lstat.isFile()) { - const data = readFileSync(full); - writeVfsBinary( - fs, - mountPath, - new Uint8Array(data), - opts?.preserveMode ? lstat.mode & 0o7777 : 0o644, + const lstat = lstatSync(full); + if (opts?.exclude?.(rel)) continue; + if (lstat.isSymbolicLink()) { + if (!opts?.preserveSymlinks) { + throw new Error( + `VFS image source symlink requires preserveSymlinks or an explicit exclude: ${full}`, ); - count++; } - } catch (err) { - if (opts?.failOnError) throw err; - // Skip unreadable files + ensureDirRecursive(fs, mountPath.slice(0, mountPath.lastIndexOf("/")) || "/"); + fs.symlink(readlinkSync(full), mountPath); + count++; + } else if (lstat.isDirectory()) { + ensureDirRecursive(fs, mountPath); + if (opts?.preserveMode) fs.chmod(mountPath, lstat.mode & 0o7777); + walk(full); + } else if (lstat.isFile()) { + const data = readFileSync(full); + writeVfsBinary( + fs, + mountPath, + new Uint8Array(data), + opts?.preserveMode ? lstat.mode & 0o7777 : 0o644, + ); + count++; + } else { + throw new Error(`Unsupported VFS image source entry: ${full}`); } } } @@ -107,6 +112,8 @@ export interface SaveImageOptions { normalizeTimestampsMs?: number; /** Runtime allocation reserve that must remain after build-time population. */ headroom?: VfsImageHeadroom; + /** Exact growth ceiling that the serialized artifact must encode. */ + expectedMaxByteLength?: number; } export interface VfsImageHeadroom { @@ -188,6 +195,27 @@ export function assertVfsImageHeadroom( } } +/** Require a serialized artifact's encoded growth ceiling to match its product profile. */ +export function assertVfsImageCapacity( + image: Uint8Array, + expectedMaxByteLength: number, + label: string, +): void { + if (!Number.isSafeInteger(expectedMaxByteLength) || expectedMaxByteLength <= 0) { + throw new Error( + `${label} expectedMaxByteLength must be a positive safe integer`, + ); + } + const actualMaxByteLength = + MemoryFileSystem.readImageCapacity(image).maxByteLength; + if (actualMaxByteLength !== expectedMaxByteLength) { + throw new Error( + `${label} has a ${actualMaxByteLength}-byte VFS capacity; ` + + `${expectedMaxByteLength} bytes are required by its product profile`, + ); + } +} + function walkVfsFiles(fs: MemoryFileSystem, dir: string, out: string[] = []): string[] { let dh: number; try { @@ -283,6 +311,9 @@ export async function saveImage( metadata, normalizeTimestampsMs: options.normalizeTimestampsMs, }); + if (options.expectedMaxByteLength !== undefined) { + assertVfsImageCapacity(image, options.expectedMaxByteLength, outFile); + } // Level 19 — slow build, smaller download. Decompression speed is // unaffected by compression level, so this is a one-sided trade. const compressed = zstdCompressSync(image, { diff --git a/package-lock.json b/package-lock.json index 37bb506ff6..bfac3c020f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,9 @@ "vite": "^8.0.3" }, "devDependencies": { + "@babel/parser": "^7.29.7", "@playwright/test": "^1.61.0", + "esbuild": "^0.28.1", "playwright": "^1.59.1", "tsx": "^4.22.4", "vitepress": "^1.6.4" diff --git a/package.json b/package.json index d5c13a1783..c9357b57e0 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,9 @@ "pack:packages": "npm run pack:host && npm run pack:sdk" }, "devDependencies": { + "@babel/parser": "^7.29.7", "@playwright/test": "^1.61.0", + "esbuild": "^0.28.1", "playwright": "^1.59.1", "tsx": "^4.22.4", "vitepress": "^1.6.4" diff --git a/packages/registry/cpython/demo/serve.ts b/packages/registry/cpython/demo/serve.ts index e746bf7935..5ac7b7d3da 100644 --- a/packages/registry/cpython/demo/serve.ts +++ b/packages/registry/cpython/demo/serve.ts @@ -20,7 +20,7 @@ const scriptDir = dirname(new URL(import.meta.url).pathname); const repoRoot = resolve(scriptDir, "../../../.."); async function main() { - const pythonWasm = tryResolveBinary("programs/cpython.wasm"); + const pythonWasm = tryResolveBinary("programs/cpython/cpython.wasm"); const pythonHome = resolve(repoRoot, "packages/registry/cpython/cpython-install"); if (!pythonWasm) { diff --git a/packages/registry/cpython/test/debug-test.ts b/packages/registry/cpython/test/debug-test.ts index 7f800e21c4..808f8aecff 100644 --- a/packages/registry/cpython/test/debug-test.ts +++ b/packages/registry/cpython/test/debug-test.ts @@ -25,7 +25,7 @@ function loadBytes(path: string): ArrayBuffer { } async function main() { - const pythonWasm = resolveBinary("programs/cpython.wasm"); + const pythonWasm = resolveBinary("programs/cpython/cpython.wasm"); const pythonHome = resolve(repoRoot, "packages/registry/cpython/cpython-install"); const kernelWasmPath = resolveBinary("kernel.wasm"); diff --git a/packages/registry/dash/build-dash.sh b/packages/registry/dash/build-dash.sh index 64f11edfc9..4938652cbb 100755 --- a/packages/registry/dash/build-dash.sh +++ b/packages/registry/dash/build-dash.sh @@ -170,12 +170,11 @@ echo "==> dash built successfully!" mkdir -p "$SCRIPT_DIR/bin" cp "$DASH_BIN" "$SCRIPT_DIR/bin/dash.wasm" -# Install into local-binaries/ so the resolver picks it up as an override. +# Install the declared dash artifact into local-binaries/ so the resolver picks +# it up as an override. Guest images create `/bin/sh` as a VFS-level shell link; +# there is no separate resolver `sh.wasm` package artifact. source "$REPO_ROOT/scripts/install-local-binary.sh" install_local_binary dash "$SCRIPT_DIR/bin/dash.wasm" -# sh ships as an alias for dash in the release; install the same bytes -# under that name so `programs/sh.wasm` also resolves locally. -install_local_binary sh "$SCRIPT_DIR/bin/dash.wasm" ls -lh "$SCRIPT_DIR/bin/dash.wasm" echo "" diff --git a/packages/registry/mariadb-test/build.toml b/packages/registry/mariadb-test/build.toml index 5c33ddc3f8..bcd3d21b53 100644 --- a/packages/registry/mariadb-test/build.toml +++ b/packages/registry/mariadb-test/build.toml @@ -6,6 +6,7 @@ inputs = [ "images/vfs/scripts/dinit-image-helpers.ts", "images/rootfs/etc/services", "images/vfs/scripts/mariadb-image-helpers.ts", + "images/vfs/scripts/mariadb-test-source-copy.ts", "images/vfs/scripts/source-extract-helper.ts", "images/vfs/scripts/vfs-image-helpers.ts", "host/src/binary-resolver.ts", diff --git a/packages/registry/mariadb-test/package.toml b/packages/registry/mariadb-test/package.toml index 4675d3cdcc..ff242871a7 100644 --- a/packages/registry/mariadb-test/package.toml +++ b/packages/registry/mariadb-test/package.toml @@ -25,7 +25,7 @@ depends_on = [ ] # Pre-built VFS image for the MariaDB test runner: mariadbd binary, -# mysql-test test files (curated 185 tests), system-table SQL, +# mysql-test test files (curated 184 tests), system-table SQL, # and init descriptors. Source at # images/vfs/scripts/build-mariadb-test-vfs-image.ts. Used by # apps/browser-demos/pages/mariadb-test/main.ts (Playwright harness). diff --git a/packages/registry/program-packages.json b/packages/registry/program-packages.json new file mode 100644 index 0000000000..82b29f9cc5 --- /dev/null +++ b/packages/registry/program-packages.json @@ -0,0 +1,2852 @@ +{ + "format": "kandelo-program-packages-v2", + "identities": { + "bash": { + "manifestSha256": "6478060f28d430d18a6ebe7c603392d35a41d9bcf6bc74322351d1450b9c5335", + "cacheKeys": { + "wasm32": "a43cb889225cd487f93af47eafe907467d21df9bc04428a5ad3df7c57cf0c6ba", + "wasm64": "aeb203ba1b78a3ca325ea9635ab2d687ea5e537db2637281f9f1124f5b333428" + } + }, + "bc": { + "manifestSha256": "a65661463bb7047b91ff00153bd99fd962c96e571934ab4e923f5215fb6ecfbd", + "cacheKeys": { + "wasm32": "5cd794c0a8203fa7a96dce3c944b5b5173ff3f39dd08a3ba08635ad5f65cd1e7", + "wasm64": "4641f9442ab1d74294ce74c1e25e236cf35208a984679a98e3f283e603d8d45f" + } + }, + "bzip2": { + "manifestSha256": "59e1e53f5675e7c9148634933ed0f4c7192743727025cae4e843b6924c489abf", + "cacheKeys": { + "wasm32": "8c43dfe973ffc1180e33126913a410f3fd17fa7c5300e2ec421f9896d3f54111", + "wasm64": "6b06242ff2397c0668a68850248f755ff0a97d48351f6a29c825316e91c06267" + } + }, + "coreutils": { + "manifestSha256": "b8baabc9af9283434e0d80f7fdc0aaef242ffa31092ab0fc39a0603a12b3cca7", + "cacheKeys": { + "wasm32": "0d7bcbc4565d31d275d3dda0357abbf40747e5f6f1407762ad00b4ce0673b0ce", + "wasm64": "49b2ba4a2e42ce773866714628ea4979da63dac87eca8d4c54c73c510529f9b3" + } + }, + "cpython": { + "manifestSha256": "7dd4f446697a73941ec940c2ccba4d53be73fe6947f1e701031a4d0aa964c4ff", + "cacheKeys": { + "wasm32": "9ebc9502240decca75ad4c6e155e6748a3d07d4f6fdb695b65d01ea799bc07af", + "wasm64": "ed14c23cc971bd9663da40ffb36c3abac91e87c2cf643998201d6cd6510b9634" + } + }, + "curl": { + "manifestSha256": "55523d50261f46dc4aaaff458d1cd87c6f96eaecd687a7540ead35c96906366e", + "cacheKeys": { + "wasm32": "ba032d8eef4b18230fe5fd7d0496b6abd1f291f3583a48f4ef3c94f441e8eaf1", + "wasm64": "84f37d6a8e59bb8e8242265d3126ef69de49917fb34352c2ebe32e7b88653865" + } + }, + "dash": { + "manifestSha256": "10ce1bb611fefd78a6c9ec9038f6113e29a7062b1a73e08eccf9b33004cea9a8", + "cacheKeys": { + "wasm32": "30d3944c0d6c187c6eed2387e81b4be46eeb7ea9c1627f216499eded89650310", + "wasm64": "437585daf592ebf1c49b784cb31db47d60c40848008672848c0dcf7c31731e6a" + } + }, + "diffutils": { + "manifestSha256": "3a78f0a46bae43ce5ea235c6638c2cbd1d8559b0b62b1fb42443b35968c1aca3", + "cacheKeys": { + "wasm32": "b1c53d03d0e67919a48f3d5eb38e4e9fc7397cfff56a6c47fc9a9faec895d21e", + "wasm64": "faa99de40b13eabd2240aced8794eacd6c1419db7057fe99678dea6a2266cf46" + } + }, + "dinit": { + "manifestSha256": "44e125e1337503e27f8cff531cdbd79c4f406fd2d282b916a7746cf3a14057bf", + "cacheKeys": { + "wasm32": "e4d6e040b3c583d15e3a80feea9db153c1e8b0ff02bc86d00fa28e5870287781", + "wasm64": "262444a46b7c122cadad29850ca42dff322377652f48e822fc88d5ea1cc0b7ed" + } + }, + "erlang": { + "manifestSha256": "475d6037e73a0f1e2f4381067194b2422751206979ea17209e10efa5a2a93bbb", + "cacheKeys": { + "wasm32": "729f2550f5075a36a2fabdee2258bcd37abf016d8ef9885b1d6e324f36bba143", + "wasm64": "8340b36b11cefef75fecaf3c29cd42ff761f67ed201b8262c2bbde7de965a42a" + } + }, + "erlang-vfs": { + "manifestSha256": "15efb74f1825e2b85bedfeb8d98c19fa648c4be39cb9defb65330a8f21eb9141", + "cacheKeys": { + "wasm32": "16a722afa84ec8798fec94df4f5e70cea2735d1d46145c11d13711d9ead34856", + "wasm64": "5ea45655b7637b1dec3b173fa916546759c58efc4f9fc528020e68b3f5099795" + } + }, + "fbdoom": { + "manifestSha256": "a00e0d9c84fcdbb3bd95f296cb3422d60b86dcff4c40734eea1bb0bec4c7d902", + "cacheKeys": { + "wasm32": "89f5953bb40e091907881848e6166675cfd8e5580de02c7848711736746832e3", + "wasm64": "7ec8ff84b79657220d9d70b8b8af743d94df0a72b1a931619b417e0608ecda70" + } + }, + "file": { + "manifestSha256": "7874c1affcbbf2c8ab8c8d4beb087f275af57b5caae6a8947bf5a63a181552b2", + "cacheKeys": { + "wasm32": "28c5bfeb2dddaebf9cc39451d2c2983812bd6c61abc05197ed584c51f310a2b1", + "wasm64": "775c68bb3e4fdb6275d184bbb48939e750953fd23265f9a6019ea13893541c6e" + } + }, + "findutils": { + "manifestSha256": "cceedf52aea67fbb0da03cfce6f2e9b1a7b5561fa1656c2762b43c651a625d02", + "cacheKeys": { + "wasm32": "51fb9260ceb9656bbb34bf723314ea6e6905915d65b736845f0fbf593b5eef00", + "wasm64": "589d688c25db4b5738af84ad782224f3840123277a971e219b6e16472f2f6623" + } + }, + "gawk": { + "manifestSha256": "a2567b8b0778805e385e1d3a41ac8b5d74aaf9d293b222001b17d09b11e5a0ba", + "cacheKeys": { + "wasm32": "cee993ecea8042490957d500095605e9635f5e3247fff32b160a3569749e53fc", + "wasm64": "f19ac2fa56010cf7a8339409dbbb4ec02ba3fdd47d6a182efc9969c2e95d1a97" + } + }, + "git": { + "manifestSha256": "c2bc79e62c9a4e840ae94a16af0f41070460eaf3976a990dc60d2db977bee952", + "cacheKeys": { + "wasm32": "8acb9c9dc14337fe6b17808630ba402c2d649822a618c6c83fad1201d2ef388e", + "wasm64": "e1c01bf30a31e8eb6927808ecfdc826006306dde27dafa2ac743af5917318943" + } + }, + "grep": { + "manifestSha256": "59270d3bfb33167b32d13246bf855b49308f8c9c0a66d9635c804f702421fbc6", + "cacheKeys": { + "wasm32": "cf4ea02c0d5ffbdec575a9fcce7c943d8d45bac06d8ee223171a3b41b6fa8e52", + "wasm64": "239dec4d8748773c19b8de231cc1df16633d5ecfe2ac831bbb002e507a699d00" + } + }, + "gzip": { + "manifestSha256": "33853ebe2301caf2979b667830e6c5afc5256f8717137624f08079e6a27f370a", + "cacheKeys": { + "wasm32": "c3d94eaf953ab465b07e356c413cc4c849ca76d49ae27c9ff2479456f3704479", + "wasm64": "672b7412bf57cb280cfd03eee984688f347ca8f596a1fce51a4bffcab4d8401c" + } + }, + "icu": { + "manifestSha256": "f5b1f02d169ec1108fc32d9da55be53c3efb96e3a7aeba6a03aec62e0175f0dc", + "cacheKeys": { + "wasm32": "5f2d9559cd1173f21fa621bcdf5451a35f3ffcc3435dd7b0e09169834092e402", + "wasm64": "a9a04f2d86a3dee4d26945deec86f95a60856f2b5dcdf73121e5d9bed15e1566" + } + }, + "kandelo-sdk": { + "manifestSha256": "c081879f1becd855917cff96f17429bcf2b9fd61bf95211eca9ff8fb625bb2eb", + "cacheKeys": { + "wasm32": "2f07a46b83da2efcf121a70525f2311e0bf3ac6bdb772be82bc0e8002feadb9d", + "wasm64": "935fade8d935cd8cc32c0aa381fbfd4f799a916e8f2e8a0eba8009adb25bfa01" + } + }, + "kernel": { + "manifestSha256": "db1d66db8575562ac7b3e720dbaa06d7dd5eef577d80220ea4167dc2d69b863b", + "cacheKeys": { + "wasm32": "e98596ee4e363710004ff969cc74e58e433a521011d6331681421c20cbaff721", + "wasm64": "9871e90d8d1106326585793cd39c6aea2fed5d7d31e68b1236505d7f41a287cf" + } + }, + "lamp": { + "manifestSha256": "40b66d115967f8ea3e1229cccfd1e6636f5363b4fd02ef66850a2dd441ccf13d", + "cacheKeys": { + "wasm32": "0325071e7ac57bb634adf7e7373511229d4ff52ab848465acb8039d696ce6cad", + "wasm64": "5527e3885d561de2b2ea8ff2d8682133921b6968c77bbb22ef45e3365af0c769" + } + }, + "less": { + "manifestSha256": "996d61545cfe83dcb663a33e8763e0edbd4db4a605fd69a91b243cf79b1f9b17", + "cacheKeys": { + "wasm32": "a31a123f83a7dbc0e10f54d71b5b74ff0d2d249b5612daa8e1562e6eb36eaf11", + "wasm64": "0e4d9e9f360243eca279d945624dae9af5d296da4920cf18555c303abac85aec" + } + }, + "libcurl": { + "manifestSha256": "c771e6cdc83b43840db4dd1fdce4b6189ba4f281596f7485bcda9c65e2ecda00", + "cacheKeys": { + "wasm32": "cb147677f78c356f20e2581df7dbaab3c77f6e3b879c33f24484bb4eafa6160b", + "wasm64": "d32f46de09c7dad6325b9215db2d1b1f8d20d869d119b91381898ff4585d1857" + } + }, + "libcxx": { + "manifestSha256": "6e1ebfa6914043770eae2a6c156b915acf6a56fe763f07d215ffe6f6f2305007", + "cacheKeys": { + "wasm32": "6d08aa913e932d33232f8076de99312f87dd7b4d866c31f1c168999910a4fffd", + "wasm64": "6b5b96e7d4f2fe4ceef0bca3603548d2544e31ea5d9794c9078e652a261ee6e5" + } + }, + "libiconv": { + "manifestSha256": "fa60b386f4ba0cd5bbbd595f67477cbfb0cc401a14bbd4769630c502bb3f24b1", + "cacheKeys": { + "wasm32": "9328da7f90a267e6eabfb49e8b00d0dd45296f0d9c4339fa1c58e2813def6c70", + "wasm64": "b267f67956dfc5d1234fd16644cb553c9611b29c6a9a632af198579e219012fd" + } + }, + "libpng": { + "manifestSha256": "79ed5c5c072a0267cce5c7a2fe37d8494571b17e44b952f619e04ec1c4a9db10", + "cacheKeys": { + "wasm32": "cefa8c2f3cb40a07bba56f1649a7d12f095d3d59c6f1b114fb9f0dae0147633b", + "wasm64": "d717500e0d357d9b7628c0ec16b8bc72318c21b6c9a4a9a6a9963effa6b96df5" + } + }, + "libxml2": { + "manifestSha256": "908490efcdc7783c76ce36af31d1db24bfcc7ba5f1e7b9b5298cc81cb110256c", + "cacheKeys": { + "wasm32": "ecbc5a52e3f7b2cc5d697d3982fe94e9dbc69e8354f62f145dad302437b1ce61", + "wasm64": "343fbc8f9b9df572517c1d45b270a7b94ff12d2dce793d1b6bc1856df7d0cac5" + } + }, + "libzip": { + "manifestSha256": "83c65d63a6416e79436bf4e792759a4701d568df0e8c010394a9732cf659f0cc", + "cacheKeys": { + "wasm32": "bb7925958ee1261e640a025e5801ffed415d4486e4a6339094f0fd7338f47073", + "wasm64": "54a164c7970389102fe2e3ed6b8ce78270405cbb61fa01b931002aeff717527b" + } + }, + "lsof": { + "manifestSha256": "cfd199bbe082435f7a2e703e2738a368af1eeb5b76b6e93c376054f21ce94419", + "cacheKeys": { + "wasm32": "c448f0352bb9d9cfbb1fd4d63ee278c114a12b8b3292fe846c555319859d9ecc", + "wasm64": "3c12b32d0190b46a6cb3ec61cfb7679f7558be3008123c4ad8df9c34b06e1d28" + } + }, + "m4": { + "manifestSha256": "2c6582d99d6eabfb9da52badf49b899e9fdcba76a728536b25bc3741f3331543", + "cacheKeys": { + "wasm32": "2a76adafbe9deb20f29041be2fe5cba99e207f28afec04911d311d3387f9e664", + "wasm64": "a51df71ba63fe477beb36da30d4c14f6d5fa2e1a8d0cc576d1a4e6a49f9ad6b0" + } + }, + "make": { + "manifestSha256": "f878d2d730f36a4c6dfe1fff1b4ccc704757ea22d8c4c8ccb95b11cd641e5fed", + "cacheKeys": { + "wasm32": "13cd37652953e64a44481f086429f2a44bd8a0dc66824eb7e1ae3bf2e6e08fe0", + "wasm64": "d9bb97fcabe1f4e6a32f99a372cd0955bfa4646695ca8d89cc4b15d43acf42fb" + } + }, + "mariadb": { + "manifestSha256": "aeb221be233e4b57bccb4b3375020f94c3ec593dac2734631f00e3b1aec9f48c", + "cacheKeys": { + "wasm32": "dfd36d445f70b37ac26f03363a3f400cb42c510e3404198b4904012c6bc19c82", + "wasm64": "815f13c9c636ea4ab21b291c35ce333ac30c97b7bdfc15c583a2686055d9065b" + } + }, + "mariadb-test": { + "manifestSha256": "d4ccce161d659a5a87c80fa1117054bbccea870e0d4a46bd8a070d3930ad0e3f", + "cacheKeys": { + "wasm32": "1ec82027f608ea5d6f44194c206ad4d847e86352028c14e1c89c1116db6606e1", + "wasm64": "cec2ff402f1168971a7911f38013fc1708464690466519c9d40f664867810679" + } + }, + "mariadb-vfs": { + "manifestSha256": "c4cd04a06d913650b0bb8c6895fa651a9a05fdba4805c576bb0c35a998276764", + "cacheKeys": { + "wasm32": "6de85a49a8a6aa3c9374fb10ad02421bb797e03d53c2f6463aa0a1f50d354255", + "wasm64": "b3dc3d08084fa6683d03311bf9519c022e8fc68d70dc22a43aeffa6871bc4d2a" + } + }, + "modeset": { + "manifestSha256": "36a8683c1c7309701d361c5f77e543a6c1531397b26c72dbb864528c59c42954", + "cacheKeys": { + "wasm32": "9e3e8604821fd9f2a8ba7e8c849826cbdd7d5d02f6ea56e801d07eb9fd14652b", + "wasm64": "98b1d2d88feaeecb7487aceca8822afc88f2fcd6e5575175058733f6a09a0506" + } + }, + "msmtpd": { + "manifestSha256": "09de04a422ddf631a29a259d816e081f8d317d1077808ede55d8c500baae748f", + "cacheKeys": { + "wasm32": "11fb3461f854cdf5fcd76e3056b0d1e50d4b1c01658a9b7806665b041db1a3c8", + "wasm64": "ea71c2d4cd85faad9aff11588c2790aace4155267d86ab2fa6b0dcbeebcbfe25" + } + }, + "nano": { + "manifestSha256": "1ba6d340c95581319982257afd6a3554b333b880a7e69991b8c573da883f86c4", + "cacheKeys": { + "wasm32": "afea62511b76715a3494ec0b383cc978c2ac2bc7bfe5c9bdb8fc44b2e4078c68", + "wasm64": "c3d38830058d1c17979a2e17a35e8c0051ea782f6a0a1db3924eb49433d97a08" + } + }, + "ncurses": { + "manifestSha256": "120bef02a4783a59f86f20a1953b120b603851e04c7d8400f16435f97f97779c", + "cacheKeys": { + "wasm32": "f7210d1ab36ab5f22676d221945bd944c58e35958947358e9824da81aac15f01", + "wasm64": "90a37ef184c797288709d11b709e986da4edc6fe4c52b6d19401c8c3db34641d" + } + }, + "netcat": { + "manifestSha256": "4cf33cd1ab768b3ad0108da8b68c2ff72e98469cd86a0bf2d0da54a7d4f6fa16", + "cacheKeys": { + "wasm32": "b78885a086ad183e20c8a93c6b657a5ce056046d54cf8145c33bcaa5bfdd5cdc", + "wasm64": "da4b715bd5e37d801e693ce8af01d7ae90e5be155930e33b16bb98ea1410092a" + } + }, + "nethack": { + "manifestSha256": "1a2f12ec2770bd8d40006d6c878a3493b97c71c2bb63ad08c7f6ad99bfd53504", + "cacheKeys": { + "wasm32": "11b1e47c7777eaf5ea106a36a0d7e70cce14a31d52005542c31222ef44473319", + "wasm64": "53664d90d9cf556bb82409ebdaeea5a0f880fa013ddd1fc3a73a65647d2b1c82" + } + }, + "nethack-browser-bundle": { + "manifestSha256": "b8294981da7ce4299dfb271d4aecb90ababa97a4d1acc9b716ae05bbe52beb53", + "cacheKeys": { + "wasm32": "c50cb85a33c8e4dd8157998a3c7fde468d33193054b7f0ce6cef111ccb26b011", + "wasm64": "3fee5741aceb0aae2e488b53849fe6b1b04234e313129ea64463901bb25f8279" + } + }, + "nginx": { + "manifestSha256": "41b0aaf05fdd0d8da42642703f9e60fd7b9545d8029896918b87c771c359147b", + "cacheKeys": { + "wasm32": "7f69eae6a9d05d93c4458093a1ac80d5685ea4ff991c948274400796c770f690", + "wasm64": "a521a8cd7c9b4188de37479410eeb81ffd182924a05c2262e679cd0370ff3861" + } + }, + "node": { + "manifestSha256": "2131a24dfda8587860e57fc65b573086477d376b065b3b9ca4e1dfe4d79f5790", + "cacheKeys": { + "wasm32": "286d3611dc0a71406479ac61a2a0f46f8d51683a78ddb2f59fdddfacf81b9515", + "wasm64": "4761f9fca12543404fb753eb94e4cdf2b63acbd8d41fdc5e91fa7ee3157b3263" + } + }, + "node-vfs": { + "manifestSha256": "33315fb1b3030a4c187ae075eac08f717de8d7ab017b86c6458778ac9070eece", + "cacheKeys": { + "wasm32": "e12fabcaa9d24c48a5e3c331940eaa45638b408aff50f0955ad4ed0065acb8b8", + "wasm64": "98ca6d78b3f4012724bb220056335da845d44522486321fba345a003cefc5eae" + } + }, + "openssl": { + "manifestSha256": "374f5dce6b2691b630b55cb82955b33d2305384e4bd64746c581621bb0793993", + "cacheKeys": { + "wasm32": "ef7615079b83e8bb69afeabcc5abaf026b5fc16a6a4b6234e25d762effdcb362", + "wasm64": "e3240130e7372e5b007c915414527e4b8ef3010e89cc74c3301f3e5329db7bce" + } + }, + "pcre2-source": { + "manifestSha256": "43ac8f9096de68c6f99a1a700d429674ed3d158e874354fc6b3a56b2ed729a7a", + "cacheKeys": { + "wasm32": "3bde5589e09b2b7fd260016854017420db3b597ae3565529a0719b47091b5240", + "wasm64": "3bde5589e09b2b7fd260016854017420db3b597ae3565529a0719b47091b5240" + } + }, + "perl": { + "manifestSha256": "6cdc4dbc54d0e4008cff41f82ec8918c0e44b4aef23903539c1bc0f538fe3ad2", + "cacheKeys": { + "wasm32": "f0d9f886fca2f6741563862628b059964bceefcdf2eb2a0b2ebfde5e30ac8adf", + "wasm64": "bb2309551b9893a8558c2eb85f89980c48a374a9db18c94e097b63eb0db04235" + } + }, + "perl-vfs": { + "manifestSha256": "1345cc102bc8c24a6bc276410c00b4fcc99ba5f6adc9b031f882aaa35d53c8bc", + "cacheKeys": { + "wasm32": "cdfb0cbc41a2413da84de6cf209a510219536837bb955a79103576901a94b063", + "wasm64": "6c5b7e1dc99463635ed805e255199e26cc21ba2528aa66f6469e1955502040a7" + } + }, + "php": { + "manifestSha256": "fcd9d7915ed8935b05418e148edcb312adf0b1e4e627fe4aab75444b35b33bbc", + "cacheKeys": { + "wasm32": "2b3b191f35c8adbfccdcfc138af00f8c00855e808ffb7336b503ac50f224b4a6", + "wasm64": "ed7b817ef25d43f46319fc6081e1dd964a79ace1783ce22d77fa6d0737c791a8" + } + }, + "posix-utils-lite": { + "manifestSha256": "8fd7190b2848ef80143adc7b9268c79e13cd4db3430f7105faf49a4b4407a06f", + "cacheKeys": { + "wasm32": "304227932e1878ec59ab186b6d025f43a2b3df928484313d2ab931a46a5f3ee7", + "wasm64": "628f61fb878c00da7229e5f1d2aec79952ba0b8cf0ba1719343caeea13400b7f" + } + }, + "python-vfs": { + "manifestSha256": "d1aa99feae65f06c0ca8cf52c2176534b2723fd4ee6eba6e72c205245e1c9c26", + "cacheKeys": { + "wasm32": "9ca6745b74b81056ebc4fed975af96753dbca7508cc2e4ad66c20809c1a2f323", + "wasm64": "c3532716be4badd6c4cf63f4ef7ea4e496ed576aa1ef9dc3d9b773f635719976" + } + }, + "redis": { + "manifestSha256": "151fb507de953ba2ba94b8f881bc7d66b4c741c1359a2f590cc07f9a4cd368a3", + "cacheKeys": { + "wasm32": "a7391e556a100804e931a940f6a88b3afee2d7a866fa315e7787a9a5a8948403", + "wasm64": "bef3535753cb37406c4f9d04d751d59d9d72f0cefc96491ccc1352da6affe75c" + } + }, + "rootfs": { + "manifestSha256": "0420201025170f48c32949ac62707d4577cdc13a53ad1f01f59d318ec07c8d6e", + "cacheKeys": { + "wasm32": "afd016a598b5bacde4a1275a142a2fb517a51dc22e120d9562c286765091324b", + "wasm64": "9002598fd087605171e3dcbd76faf847193ec4c771642fc1ceed4624bd791366" + } + }, + "ruby": { + "manifestSha256": "6ea67a246c29cd927a19decbb36ae4c4d478ff6642d2c77e08a6b2cfeb8251d2", + "cacheKeys": { + "wasm32": "84ffe7f564d7631b51dd3951ff655eef315a89691e9efaa3791831643bd1e985", + "wasm64": "7db321d4f77faf87353d09e189792abdfd1d6dd233e0e7f4007290c67790dac9" + } + }, + "sed": { + "manifestSha256": "2a08e9c5dacc5facc8983c1ff35ff2a72db328405e9a1f464cc7ad155c4d08af", + "cacheKeys": { + "wasm32": "f687903421a723b81ead0a60f488b617375ce19ed97edc2820818f96f90e3c82", + "wasm64": "4df0bd8ef915d60f14f95a07f61ffdb53c536b3b8f89c9a0a7a6354a64869e59" + } + }, + "shell": { + "manifestSha256": "5d02cd8fee4e0d46c7552a3046c3898f9e3bd5667094e02cab01851eed97a321", + "cacheKeys": { + "wasm32": "feffb1d8d0b51ef0116bc99f0f7989127ed9bc414ce54b459a8232df05f0a5d2", + "wasm64": "d7e1a7ca970c3dc639b153977383f0f0708495765f9c4ac69a4d7e238b1232bf" + } + }, + "spidermonkey": { + "manifestSha256": "c72ef4e43ee2a4fc74390d03a239fce3b545a6d3b64f02ccc4d44972cec626bd", + "cacheKeys": { + "wasm32": "85aca810215b5f02dcb66956292be3b345aa4469fa34c70a6cd602b8016909dd", + "wasm64": "24c5e3842b8b87ea1d2f6a31fa8487b30edce568028c3771f104e72fdd561cf9" + } + }, + "spidermonkey-node": { + "manifestSha256": "f156c4c5044d2a9447324e7a2a35f8c3e6fa367b3e44f674dcc30ab6b946fae7", + "cacheKeys": { + "wasm32": "4867322d0e6f96809141a2dce6682483a2a6743899f5fd75f2271d82475f0269", + "wasm64": "c5eb2d2032ce40c037d57d769ff398b587fe684d037f326790edd95b6cf6e7dc" + } + }, + "sqlite": { + "manifestSha256": "ab9560b4eab53445034e2a87a27ccee9d05ddb0db73fee28f5000325b0b1cba5", + "cacheKeys": { + "wasm32": "3cabb24c5250387ef8e26e0a1566cb7b198a8783c3cc30fb2ae31c8d6d044e77", + "wasm64": "2daade2833893b1c0e28c556e2f70908e78ea1efb5eb813d328733bd1bd3e3f6" + } + }, + "sqlite-cli": { + "manifestSha256": "1cebb768f9473dc58fc6e72c9b24565760aedaec4cbdb29ad43b22a0a5fd7030", + "cacheKeys": { + "wasm32": "a26d9e2b82748828e1f2e322c535892993237673d1759269341ca7e3d55b77ae", + "wasm64": "4819815d778c7958e2285a57b92fc8d06bfaad6367c1c13c49584bad7df06343" + } + }, + "tar": { + "manifestSha256": "08fa090c122d3105c735d74560bdd7b8b083a6f9ca8bdec17de0b4993e1be7fd", + "cacheKeys": { + "wasm32": "bedb0c34a78b5c4642b340e115feca87ed3752f4a836d3b5534716e3137c034c", + "wasm64": "75f3d189257503912bd668ad66b4c71159ec498b63b5c3253c0307fe4587a864" + } + }, + "tcl": { + "manifestSha256": "67253d47de7df9184e68ec3c49835455746941835a26db9e588aa8fbee7d4636", + "cacheKeys": { + "wasm32": "e18543aa20544ced46cb8a2a7bbee5d5227ec0c1076336eb3db6ad70ad3c4095", + "wasm64": "07f1fba2c47e39e38a8ae9b983cb37d56081b9be30fcd9e02034c910eff39b8b" + } + }, + "texlive": { + "manifestSha256": "028ada023f8a8f9966c8a28575242a339aaaf0e8870fe4a221986133b381c3c0", + "cacheKeys": { + "wasm32": "ff321955a73f9ce9c018eeb9faba1df0274ed6087f6df60f5903dcce9e05956c", + "wasm64": "ac3f28b923066fb4d53fbee134e95fe1f105ae31c852f7dc90a9962ffe9ae67b" + } + }, + "unzip": { + "manifestSha256": "ef4e5e34258827ab3cbc1930da3fd9514f08f2d7d56c7c3e0d5c495a4d3a20e9", + "cacheKeys": { + "wasm32": "cc555c2eb64b33f1ca29d997a36c1e1e98df74a5411b0992dc4127352e6ffefa", + "wasm64": "8d0a48d9a7b84adf582c8194486a866300b3e047b5acb610c4f6f718a608920b" + } + }, + "userspace": { + "manifestSha256": "221176f2a096dee19bbefd89da5c7f50138ecd92d37ec9d7d6b35dbb25e86924", + "cacheKeys": { + "wasm32": "5bbaac807bc93e6acc7e52fbde8377f9d9c6f261155637a15a5d61591f3a2ff1", + "wasm64": "982bad219c6a6816275c2f9e0b44595519c15f0e554f926e53cf08c87b31ac2a" + } + }, + "vim": { + "manifestSha256": "c211660ef41d01f95f3fbdf675b74891e897e3f304e04f1bf5586f326007382c", + "cacheKeys": { + "wasm32": "a4a47859e8a0a1d8f627327f1fb502f2ac7fee615985fa030b5f9fd88d03752b", + "wasm64": "36683fb33b1fec3828b34eacfebf7723c0beb1549075991e14a9fa30de84dc9f" + } + }, + "vim-browser-bundle": { + "manifestSha256": "e31d84057db49c3534381604d0c8f3c7d765e33ede560a7ea7b01a5a8f1aa0d3", + "cacheKeys": { + "wasm32": "cf5f9141bfc5047f7548121cafb993708a8da25b2639967cc603e4d0347a4347", + "wasm64": "79ae99127fa0c1426bb84ad794589923616ffc9dac85ccadff51097a9c18e9f1" + } + }, + "wget": { + "manifestSha256": "d3c7ba9bc1ae708b99850a6eb2cae521c85bf1010c90135a935f5dfae57a53a0", + "cacheKeys": { + "wasm32": "97d80a049d3f04139d1048566d4b0517b73e1200b06c9bb6745c065f5f1ca75d", + "wasm64": "79ebe8b418bda767a3e7e84f8b58f7c24b5200fa5a2c1c791ee3cf5c4da450cc" + } + }, + "wordpress": { + "manifestSha256": "2c95e72f657a05be445c9b264de769becfb50833d6ef7ee4d18feac690ca2b52", + "cacheKeys": { + "wasm32": "7786ef684b85a94ebfaa5aa7bc4f336a0e9a6d632cca51b8bba7efe17e2d17ca", + "wasm64": "0ed74dab0f74457ff3f4c7e786419f7c523bbeade8fd1d0e98ff914b4d7661eb" + } + }, + "xz": { + "manifestSha256": "8707d35b8647b1af8fa165dcb6ce7b2a6a007e19ab2daca2680c39395864a737", + "cacheKeys": { + "wasm32": "e5a775c8202677e7819c82bfdc97d12e616873e776ee51b965246920ace2c052", + "wasm64": "661a91acb2951fb95554498838de01c0c275f903d5e47b2f819cb8b4559d1858" + } + }, + "zip": { + "manifestSha256": "9c9cfa8220aaeaea26736b55f7cd0a7517b8853c07f9f0d241ad67dd43592e36", + "cacheKeys": { + "wasm32": "eadd23e06a078d5f6760074f74e2c2ab18572e92959082bd49018d547243e5c1", + "wasm64": "6a10af8e54adc44b5d684ecf7addf114d0757fd0ac2f60a4a47193bcf04aec94" + } + }, + "zlib": { + "manifestSha256": "d4f4701c0c30dc843ebe2111ed3e2284836cdfc4bc05a130c8fab4f31b8e6357", + "cacheKeys": { + "wasm32": "d17bfc607f79188de27ed14ceac45d191f537d7376d8e69ab83e01a374d5428e", + "wasm64": "81fcb013fc9a89a08ee2e28a95c4eb315f2d24a0e68d6cf6eec88fc4def6c5ff" + } + }, + "zstd": { + "manifestSha256": "89f266938c2c253700a838e6352c9de66c976c84883325aaa979f72b732357e4", + "cacheKeys": { + "wasm32": "f0cff41dee757dcf21bb7933a06c423a8c51eabb7bf94ff907e1d15b2db659ef", + "wasm64": "8ad106bd6a202d37c495d16b78d54efba46f17995694919dffa286edc795a2e1" + } + } + }, + "packages": { + "bash": { + "manifestSha256": "6478060f28d430d18a6ebe7c603392d35a41d9bcf6bc74322351d1450b9c5335", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "a43cb889225cd487f93af47eafe907467d21df9bc04428a5ad3df7c57cf0c6ba" + }, + "dependencyClosures": { + "wasm32": [ + { + "packageName": "ncurses", + "manifestSha256": "120bef02a4783a59f86f20a1953b120b603851e04c7d8400f16435f97f97779c", + "cacheKey": "f7210d1ab36ab5f22676d221945bd944c58e35958947358e9824da81aac15f01" + } + ] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "bash.wasm", + "mirrorPath": "bash.wasm", + "outputName": "bash", + "forkInstrumentation": "auto" + } + ] + }, + "bc": { + "manifestSha256": "a65661463bb7047b91ff00153bd99fd962c96e571934ab4e923f5215fb6ecfbd", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "5cd794c0a8203fa7a96dce3c944b5b5173ff3f39dd08a3ba08635ad5f65cd1e7" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "bc.wasm", + "mirrorPath": "bc.wasm", + "outputName": "bc", + "forkInstrumentation": "auto" + } + ] + }, + "bzip2": { + "manifestSha256": "59e1e53f5675e7c9148634933ed0f4c7192743727025cae4e843b6924c489abf", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "8c43dfe973ffc1180e33126913a410f3fd17fa7c5300e2ec421f9896d3f54111" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "bzip2.wasm", + "mirrorPath": "bzip2.wasm", + "outputName": "bzip2", + "forkInstrumentation": "auto" + } + ] + }, + "coreutils": { + "manifestSha256": "b8baabc9af9283434e0d80f7fdc0aaef242ffa31092ab0fc39a0603a12b3cca7", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "0d7bcbc4565d31d275d3dda0357abbf40747e5f6f1407762ad00b4ce0673b0ce" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "coreutils.wasm", + "mirrorPath": "coreutils.wasm", + "outputName": "coreutils", + "forkInstrumentation": "auto" + } + ] + }, + "cpython": { + "manifestSha256": "7dd4f446697a73941ec940c2ccba4d53be73fe6947f1e701031a4d0aa964c4ff", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "9ebc9502240decca75ad4c6e155e6748a3d07d4f6fdb695b65d01ea799bc07af" + }, + "dependencyClosures": { + "wasm32": [ + { + "packageName": "zlib", + "manifestSha256": "d4f4701c0c30dc843ebe2111ed3e2284836cdfc4bc05a130c8fab4f31b8e6357", + "cacheKey": "d17bfc607f79188de27ed14ceac45d191f537d7376d8e69ab83e01a374d5428e" + } + ] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "python.wasm", + "mirrorPath": "cpython/cpython.wasm", + "outputName": "cpython", + "forkInstrumentation": "auto" + }, + { + "kind": "runtime-file", + "sourceArtifact": "python-runtime.zip", + "mirrorPath": "cpython/python-runtime.zip", + "guestPath": "/usr/share/cpython/python-runtime.zip", + "mode": 420 + } + ] + }, + "curl": { + "manifestSha256": "55523d50261f46dc4aaaff458d1cd87c6f96eaecd687a7540ead35c96906366e", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "ba032d8eef4b18230fe5fd7d0496b6abd1f291f3583a48f4ef3c94f441e8eaf1" + }, + "dependencyClosures": { + "wasm32": [ + { + "packageName": "openssl", + "manifestSha256": "374f5dce6b2691b630b55cb82955b33d2305384e4bd64746c581621bb0793993", + "cacheKey": "ef7615079b83e8bb69afeabcc5abaf026b5fc16a6a4b6234e25d762effdcb362" + }, + { + "packageName": "zlib", + "manifestSha256": "d4f4701c0c30dc843ebe2111ed3e2284836cdfc4bc05a130c8fab4f31b8e6357", + "cacheKey": "d17bfc607f79188de27ed14ceac45d191f537d7376d8e69ab83e01a374d5428e" + } + ] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "curl.wasm", + "mirrorPath": "curl.wasm", + "outputName": "curl", + "forkInstrumentation": "auto" + } + ] + }, + "dash": { + "manifestSha256": "10ce1bb611fefd78a6c9ec9038f6113e29a7062b1a73e08eccf9b33004cea9a8", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "30d3944c0d6c187c6eed2387e81b4be46eeb7ea9c1627f216499eded89650310" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "dash.wasm", + "mirrorPath": "dash.wasm", + "outputName": "dash", + "forkInstrumentation": "auto" + } + ] + }, + "diffutils": { + "manifestSha256": "3a78f0a46bae43ce5ea235c6638c2cbd1d8559b0b62b1fb42443b35968c1aca3", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "b1c53d03d0e67919a48f3d5eb38e4e9fc7397cfff56a6c47fc9a9faec895d21e" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "diff.wasm", + "mirrorPath": "diffutils/diff.wasm", + "outputName": "diff", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "cmp.wasm", + "mirrorPath": "diffutils/cmp.wasm", + "outputName": "cmp", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "diff3.wasm", + "mirrorPath": "diffutils/diff3.wasm", + "outputName": "diff3", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "sdiff.wasm", + "mirrorPath": "diffutils/sdiff.wasm", + "outputName": "sdiff", + "forkInstrumentation": "auto" + } + ] + }, + "dinit": { + "manifestSha256": "44e125e1337503e27f8cff531cdbd79c4f406fd2d282b916a7746cf3a14057bf", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "e4d6e040b3c583d15e3a80feea9db153c1e8b0ff02bc86d00fa28e5870287781" + }, + "dependencyClosures": { + "wasm32": [ + { + "packageName": "libcxx", + "manifestSha256": "6e1ebfa6914043770eae2a6c156b915acf6a56fe763f07d215ffe6f6f2305007", + "cacheKey": "6d08aa913e932d33232f8076de99312f87dd7b4d866c31f1c168999910a4fffd" + } + ] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "dinit.wasm", + "mirrorPath": "dinit/dinit.wasm", + "outputName": "dinit", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "dinitctl.wasm", + "mirrorPath": "dinit/dinitctl.wasm", + "outputName": "dinitctl", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "dinitcheck.wasm", + "mirrorPath": "dinit/dinitcheck.wasm", + "outputName": "dinitcheck", + "forkInstrumentation": "auto" + } + ] + }, + "erlang": { + "manifestSha256": "475d6037e73a0f1e2f4381067194b2422751206979ea17209e10efa5a2a93bbb", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "729f2550f5075a36a2fabdee2258bcd37abf016d8ef9885b1d6e324f36bba143" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "erlang.wasm", + "mirrorPath": "erlang/erlang.wasm", + "outputName": "erlang", + "forkInstrumentation": "auto" + }, + { + "kind": "runtime-file", + "sourceArtifact": "erlang-otp.tar.zst", + "mirrorPath": "erlang/erlang-otp.tar.zst", + "guestPath": "/usr/share/erlang/erlang-otp.tar.zst", + "mode": 420 + } + ] + }, + "erlang-vfs": { + "manifestSha256": "15efb74f1825e2b85bedfeb8d98c19fa648c4be39cb9defb65330a8f21eb9141", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "16a722afa84ec8798fec94df4f5e70cea2735d1d46145c11d13711d9ead34856" + }, + "dependencyClosures": { + "wasm32": [ + { + "packageName": "erlang", + "manifestSha256": "475d6037e73a0f1e2f4381067194b2422751206979ea17209e10efa5a2a93bbb", + "cacheKey": "729f2550f5075a36a2fabdee2258bcd37abf016d8ef9885b1d6e324f36bba143" + } + ] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "erlang-vfs.vfs.zst", + "mirrorPath": "erlang-vfs.vfs.zst", + "outputName": "erlang-vfs", + "forkInstrumentation": "auto" + } + ] + }, + "fbdoom": { + "manifestSha256": "a00e0d9c84fcdbb3bd95f296cb3422d60b86dcff4c40734eea1bb0bec4c7d902", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "89f5953bb40e091907881848e6166675cfd8e5580de02c7848711736746832e3" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "fbdoom.wasm", + "mirrorPath": "fbdoom.wasm", + "outputName": "fbdoom", + "forkInstrumentation": "auto" + } + ] + }, + "file": { + "manifestSha256": "7874c1affcbbf2c8ab8c8d4beb087f275af57b5caae6a8947bf5a63a181552b2", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "28c5bfeb2dddaebf9cc39451d2c2983812bd6c61abc05197ed584c51f310a2b1" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "file.wasm", + "mirrorPath": "file/file.wasm", + "outputName": "file", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "magic.lite", + "mirrorPath": "file/file-magic.lite", + "outputName": "file-magic", + "forkInstrumentation": "auto" + } + ] + }, + "findutils": { + "manifestSha256": "cceedf52aea67fbb0da03cfce6f2e9b1a7b5561fa1656c2762b43c651a625d02", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "51fb9260ceb9656bbb34bf723314ea6e6905915d65b736845f0fbf593b5eef00" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "find.wasm", + "mirrorPath": "findutils/find.wasm", + "outputName": "find", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "xargs.wasm", + "mirrorPath": "findutils/xargs.wasm", + "outputName": "xargs", + "forkInstrumentation": "auto" + } + ] + }, + "gawk": { + "manifestSha256": "a2567b8b0778805e385e1d3a41ac8b5d74aaf9d293b222001b17d09b11e5a0ba", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "cee993ecea8042490957d500095605e9635f5e3247fff32b160a3569749e53fc" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "gawk.wasm", + "mirrorPath": "gawk.wasm", + "outputName": "gawk", + "forkInstrumentation": "auto" + } + ] + }, + "git": { + "manifestSha256": "c2bc79e62c9a4e840ae94a16af0f41070460eaf3976a990dc60d2db977bee952", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "8acb9c9dc14337fe6b17808630ba402c2d649822a618c6c83fad1201d2ef388e" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "git.wasm", + "mirrorPath": "git/git.wasm", + "outputName": "git", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "git-remote-http.wasm", + "mirrorPath": "git/git-remote-http.wasm", + "outputName": "git-remote-http", + "forkInstrumentation": "auto" + } + ] + }, + "grep": { + "manifestSha256": "59270d3bfb33167b32d13246bf855b49308f8c9c0a66d9635c804f702421fbc6", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "cf4ea02c0d5ffbdec575a9fcce7c943d8d45bac06d8ee223171a3b41b6fa8e52" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "grep.wasm", + "mirrorPath": "grep.wasm", + "outputName": "grep", + "forkInstrumentation": "auto" + } + ] + }, + "gzip": { + "manifestSha256": "33853ebe2301caf2979b667830e6c5afc5256f8717137624f08079e6a27f370a", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "c3d94eaf953ab465b07e356c413cc4c849ca76d49ae27c9ff2479456f3704479" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "gzip.wasm", + "mirrorPath": "gzip.wasm", + "outputName": "gzip", + "forkInstrumentation": "auto" + } + ] + }, + "kandelo-sdk": { + "manifestSha256": "c081879f1becd855917cff96f17429bcf2b9fd61bf95211eca9ff8fb625bb2eb", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "2f07a46b83da2efcf121a70525f2311e0bf3ac6bdb772be82bc0e8002feadb9d" + }, + "dependencyClosures": { + "wasm32": [ + { + "packageName": "libcxx", + "manifestSha256": "6e1ebfa6914043770eae2a6c156b915acf6a56fe763f07d215ffe6f6f2305007", + "cacheKey": "6d08aa913e932d33232f8076de99312f87dd7b4d866c31f1c168999910a4fffd" + } + ] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "kandelo-sdk.vfs.zst", + "mirrorPath": "kandelo-sdk.vfs.zst", + "outputName": "kandelo-sdk", + "forkInstrumentation": "auto" + } + ] + }, + "lamp": { + "manifestSha256": "40b66d115967f8ea3e1229cccfd1e6636f5363b4fd02ef66850a2dd441ccf13d", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "0325071e7ac57bb634adf7e7373511229d4ff52ab848465acb8039d696ce6cad" + }, + "dependencyClosures": { + "wasm32": [ + { + "packageName": "dinit", + "manifestSha256": "44e125e1337503e27f8cff531cdbd79c4f406fd2d282b916a7746cf3a14057bf", + "cacheKey": "e4d6e040b3c583d15e3a80feea9db153c1e8b0ff02bc86d00fa28e5870287781" + }, + { + "packageName": "icu", + "manifestSha256": "f5b1f02d169ec1108fc32d9da55be53c3efb96e3a7aeba6a03aec62e0175f0dc", + "cacheKey": "5f2d9559cd1173f21fa621bcdf5451a35f3ffcc3435dd7b0e09169834092e402" + }, + { + "packageName": "libcurl", + "manifestSha256": "c771e6cdc83b43840db4dd1fdce4b6189ba4f281596f7485bcda9c65e2ecda00", + "cacheKey": "cb147677f78c356f20e2581df7dbaab3c77f6e3b879c33f24484bb4eafa6160b" + }, + { + "packageName": "libcxx", + "manifestSha256": "6e1ebfa6914043770eae2a6c156b915acf6a56fe763f07d215ffe6f6f2305007", + "cacheKey": "6d08aa913e932d33232f8076de99312f87dd7b4d866c31f1c168999910a4fffd" + }, + { + "packageName": "libiconv", + "manifestSha256": "fa60b386f4ba0cd5bbbd595f67477cbfb0cc401a14bbd4769630c502bb3f24b1", + "cacheKey": "9328da7f90a267e6eabfb49e8b00d0dd45296f0d9c4339fa1c58e2813def6c70" + }, + { + "packageName": "libxml2", + "manifestSha256": "908490efcdc7783c76ce36af31d1db24bfcc7ba5f1e7b9b5298cc81cb110256c", + "cacheKey": "ecbc5a52e3f7b2cc5d697d3982fe94e9dbc69e8354f62f145dad302437b1ce61" + }, + { + "packageName": "libzip", + "manifestSha256": "83c65d63a6416e79436bf4e792759a4701d568df0e8c010394a9732cf659f0cc", + "cacheKey": "bb7925958ee1261e640a025e5801ffed415d4486e4a6339094f0fd7338f47073" + }, + { + "packageName": "mariadb", + "manifestSha256": "aeb221be233e4b57bccb4b3375020f94c3ec593dac2734631f00e3b1aec9f48c", + "cacheKey": "dfd36d445f70b37ac26f03363a3f400cb42c510e3404198b4904012c6bc19c82" + }, + { + "packageName": "msmtpd", + "manifestSha256": "09de04a422ddf631a29a259d816e081f8d317d1077808ede55d8c500baae748f", + "cacheKey": "11fb3461f854cdf5fcd76e3056b0d1e50d4b1c01658a9b7806665b041db1a3c8" + }, + { + "packageName": "nginx", + "manifestSha256": "41b0aaf05fdd0d8da42642703f9e60fd7b9545d8029896918b87c771c359147b", + "cacheKey": "7f69eae6a9d05d93c4458093a1ac80d5685ea4ff991c948274400796c770f690" + }, + { + "packageName": "openssl", + "manifestSha256": "374f5dce6b2691b630b55cb82955b33d2305384e4bd64746c581621bb0793993", + "cacheKey": "ef7615079b83e8bb69afeabcc5abaf026b5fc16a6a4b6234e25d762effdcb362" + }, + { + "packageName": "pcre2-source", + "manifestSha256": "43ac8f9096de68c6f99a1a700d429674ed3d158e874354fc6b3a56b2ed729a7a", + "cacheKey": "3bde5589e09b2b7fd260016854017420db3b597ae3565529a0719b47091b5240" + }, + { + "packageName": "php", + "manifestSha256": "fcd9d7915ed8935b05418e148edcb312adf0b1e4e627fe4aab75444b35b33bbc", + "cacheKey": "2b3b191f35c8adbfccdcfc138af00f8c00855e808ffb7336b503ac50f224b4a6" + }, + { + "packageName": "shell", + "manifestSha256": "5d02cd8fee4e0d46c7552a3046c3898f9e3bd5667094e02cab01851eed97a321", + "cacheKey": "feffb1d8d0b51ef0116bc99f0f7989127ed9bc414ce54b459a8232df05f0a5d2" + }, + { + "packageName": "sqlite", + "manifestSha256": "ab9560b4eab53445034e2a87a27ccee9d05ddb0db73fee28f5000325b0b1cba5", + "cacheKey": "3cabb24c5250387ef8e26e0a1566cb7b198a8783c3cc30fb2ae31c8d6d044e77" + }, + { + "packageName": "zlib", + "manifestSha256": "d4f4701c0c30dc843ebe2111ed3e2284836cdfc4bc05a130c8fab4f31b8e6357", + "cacheKey": "d17bfc607f79188de27ed14ceac45d191f537d7376d8e69ab83e01a374d5428e" + } + ] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "lamp.vfs.zst", + "mirrorPath": "lamp.vfs.zst", + "outputName": "lamp", + "forkInstrumentation": "auto" + } + ] + }, + "less": { + "manifestSha256": "996d61545cfe83dcb663a33e8763e0edbd4db4a605fd69a91b243cf79b1f9b17", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "a31a123f83a7dbc0e10f54d71b5b74ff0d2d249b5612daa8e1562e6eb36eaf11" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "less.wasm", + "mirrorPath": "less.wasm", + "outputName": "less", + "forkInstrumentation": "auto" + } + ] + }, + "lsof": { + "manifestSha256": "cfd199bbe082435f7a2e703e2738a368af1eeb5b76b6e93c376054f21ce94419", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "c448f0352bb9d9cfbb1fd4d63ee278c114a12b8b3292fe846c555319859d9ecc" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "lsof.wasm", + "mirrorPath": "lsof.wasm", + "outputName": "lsof", + "forkInstrumentation": "auto" + } + ] + }, + "m4": { + "manifestSha256": "2c6582d99d6eabfb9da52badf49b899e9fdcba76a728536b25bc3741f3331543", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "2a76adafbe9deb20f29041be2fe5cba99e207f28afec04911d311d3387f9e664" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "m4.wasm", + "mirrorPath": "m4.wasm", + "outputName": "m4", + "forkInstrumentation": "auto" + } + ] + }, + "make": { + "manifestSha256": "f878d2d730f36a4c6dfe1fff1b4ccc704757ea22d8c4c8ccb95b11cd641e5fed", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "13cd37652953e64a44481f086429f2a44bd8a0dc66824eb7e1ae3bf2e6e08fe0" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "make.wasm", + "mirrorPath": "make.wasm", + "outputName": "make", + "forkInstrumentation": "auto" + } + ] + }, + "mariadb": { + "manifestSha256": "aeb221be233e4b57bccb4b3375020f94c3ec593dac2734631f00e3b1aec9f48c", + "arches": [ + "wasm32", + "wasm64" + ], + "cacheKeys": { + "wasm32": "dfd36d445f70b37ac26f03363a3f400cb42c510e3404198b4904012c6bc19c82", + "wasm64": "815f13c9c636ea4ab21b291c35ce333ac30c97b7bdfc15c583a2686055d9065b" + }, + "dependencyClosures": { + "wasm32": [ + { + "packageName": "libcxx", + "manifestSha256": "6e1ebfa6914043770eae2a6c156b915acf6a56fe763f07d215ffe6f6f2305007", + "cacheKey": "6d08aa913e932d33232f8076de99312f87dd7b4d866c31f1c168999910a4fffd" + }, + { + "packageName": "pcre2-source", + "manifestSha256": "43ac8f9096de68c6f99a1a700d429674ed3d158e874354fc6b3a56b2ed729a7a", + "cacheKey": "3bde5589e09b2b7fd260016854017420db3b597ae3565529a0719b47091b5240" + } + ], + "wasm64": [ + { + "packageName": "libcxx", + "manifestSha256": "6e1ebfa6914043770eae2a6c156b915acf6a56fe763f07d215ffe6f6f2305007", + "cacheKey": "6b5b96e7d4f2fe4ceef0bca3603548d2544e31ea5d9794c9078e652a261ee6e5" + }, + { + "packageName": "pcre2-source", + "manifestSha256": "43ac8f9096de68c6f99a1a700d429674ed3d158e874354fc6b3a56b2ed729a7a", + "cacheKey": "3bde5589e09b2b7fd260016854017420db3b597ae3565529a0719b47091b5240" + } + ] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "mariadbd.wasm", + "mirrorPath": "mariadb/mariadbd.wasm", + "outputName": "mariadbd", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "mysqltest.wasm", + "mirrorPath": "mariadb/mysqltest.wasm", + "outputName": "mysqltest", + "forkInstrumentation": "auto" + } + ] + }, + "mariadb-test": { + "manifestSha256": "d4ccce161d659a5a87c80fa1117054bbccea870e0d4a46bd8a070d3930ad0e3f", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "1ec82027f608ea5d6f44194c206ad4d847e86352028c14e1c89c1116db6606e1" + }, + "dependencyClosures": { + "wasm32": [ + { + "packageName": "coreutils", + "manifestSha256": "b8baabc9af9283434e0d80f7fdc0aaef242ffa31092ab0fc39a0603a12b3cca7", + "cacheKey": "0d7bcbc4565d31d275d3dda0357abbf40747e5f6f1407762ad00b4ce0673b0ce" + }, + { + "packageName": "dash", + "manifestSha256": "10ce1bb611fefd78a6c9ec9038f6113e29a7062b1a73e08eccf9b33004cea9a8", + "cacheKey": "30d3944c0d6c187c6eed2387e81b4be46eeb7ea9c1627f216499eded89650310" + }, + { + "packageName": "dinit", + "manifestSha256": "44e125e1337503e27f8cff531cdbd79c4f406fd2d282b916a7746cf3a14057bf", + "cacheKey": "e4d6e040b3c583d15e3a80feea9db153c1e8b0ff02bc86d00fa28e5870287781" + }, + { + "packageName": "libcxx", + "manifestSha256": "6e1ebfa6914043770eae2a6c156b915acf6a56fe763f07d215ffe6f6f2305007", + "cacheKey": "6d08aa913e932d33232f8076de99312f87dd7b4d866c31f1c168999910a4fffd" + }, + { + "packageName": "mariadb", + "manifestSha256": "aeb221be233e4b57bccb4b3375020f94c3ec593dac2734631f00e3b1aec9f48c", + "cacheKey": "dfd36d445f70b37ac26f03363a3f400cb42c510e3404198b4904012c6bc19c82" + }, + { + "packageName": "pcre2-source", + "manifestSha256": "43ac8f9096de68c6f99a1a700d429674ed3d158e874354fc6b3a56b2ed729a7a", + "cacheKey": "3bde5589e09b2b7fd260016854017420db3b597ae3565529a0719b47091b5240" + } + ] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "mariadb-test.vfs.zst", + "mirrorPath": "mariadb-test.vfs.zst", + "outputName": "mariadb-test", + "forkInstrumentation": "auto" + } + ] + }, + "mariadb-vfs": { + "manifestSha256": "c4cd04a06d913650b0bb8c6895fa651a9a05fdba4805c576bb0c35a998276764", + "arches": [ + "wasm32", + "wasm64" + ], + "cacheKeys": { + "wasm32": "6de85a49a8a6aa3c9374fb10ad02421bb797e03d53c2f6463aa0a1f50d354255", + "wasm64": "b3dc3d08084fa6683d03311bf9519c022e8fc68d70dc22a43aeffa6871bc4d2a" + }, + "dependencyClosures": { + "wasm32": [ + { + "packageName": "coreutils", + "manifestSha256": "b8baabc9af9283434e0d80f7fdc0aaef242ffa31092ab0fc39a0603a12b3cca7", + "cacheKey": "0d7bcbc4565d31d275d3dda0357abbf40747e5f6f1407762ad00b4ce0673b0ce" + }, + { + "packageName": "dash", + "manifestSha256": "10ce1bb611fefd78a6c9ec9038f6113e29a7062b1a73e08eccf9b33004cea9a8", + "cacheKey": "30d3944c0d6c187c6eed2387e81b4be46eeb7ea9c1627f216499eded89650310" + }, + { + "packageName": "dinit", + "manifestSha256": "44e125e1337503e27f8cff531cdbd79c4f406fd2d282b916a7746cf3a14057bf", + "cacheKey": "e4d6e040b3c583d15e3a80feea9db153c1e8b0ff02bc86d00fa28e5870287781" + }, + { + "packageName": "libcxx", + "manifestSha256": "6e1ebfa6914043770eae2a6c156b915acf6a56fe763f07d215ffe6f6f2305007", + "cacheKey": "6d08aa913e932d33232f8076de99312f87dd7b4d866c31f1c168999910a4fffd" + }, + { + "packageName": "mariadb", + "manifestSha256": "aeb221be233e4b57bccb4b3375020f94c3ec593dac2734631f00e3b1aec9f48c", + "cacheKey": "dfd36d445f70b37ac26f03363a3f400cb42c510e3404198b4904012c6bc19c82" + }, + { + "packageName": "pcre2-source", + "manifestSha256": "43ac8f9096de68c6f99a1a700d429674ed3d158e874354fc6b3a56b2ed729a7a", + "cacheKey": "3bde5589e09b2b7fd260016854017420db3b597ae3565529a0719b47091b5240" + } + ], + "wasm64": [ + { + "packageName": "coreutils", + "manifestSha256": "b8baabc9af9283434e0d80f7fdc0aaef242ffa31092ab0fc39a0603a12b3cca7", + "cacheKey": "49b2ba4a2e42ce773866714628ea4979da63dac87eca8d4c54c73c510529f9b3" + }, + { + "packageName": "dash", + "manifestSha256": "10ce1bb611fefd78a6c9ec9038f6113e29a7062b1a73e08eccf9b33004cea9a8", + "cacheKey": "437585daf592ebf1c49b784cb31db47d60c40848008672848c0dcf7c31731e6a" + }, + { + "packageName": "dinit", + "manifestSha256": "44e125e1337503e27f8cff531cdbd79c4f406fd2d282b916a7746cf3a14057bf", + "cacheKey": "262444a46b7c122cadad29850ca42dff322377652f48e822fc88d5ea1cc0b7ed" + }, + { + "packageName": "libcxx", + "manifestSha256": "6e1ebfa6914043770eae2a6c156b915acf6a56fe763f07d215ffe6f6f2305007", + "cacheKey": "6b5b96e7d4f2fe4ceef0bca3603548d2544e31ea5d9794c9078e652a261ee6e5" + }, + { + "packageName": "mariadb", + "manifestSha256": "aeb221be233e4b57bccb4b3375020f94c3ec593dac2734631f00e3b1aec9f48c", + "cacheKey": "815f13c9c636ea4ab21b291c35ce333ac30c97b7bdfc15c583a2686055d9065b" + }, + { + "packageName": "pcre2-source", + "manifestSha256": "43ac8f9096de68c6f99a1a700d429674ed3d158e874354fc6b3a56b2ed729a7a", + "cacheKey": "3bde5589e09b2b7fd260016854017420db3b597ae3565529a0719b47091b5240" + } + ] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "mariadb-vfs.vfs.zst", + "mirrorPath": "mariadb-vfs.vfs.zst", + "outputName": "mariadb-vfs", + "forkInstrumentation": "auto" + } + ] + }, + "modeset": { + "manifestSha256": "36a8683c1c7309701d361c5f77e543a6c1531397b26c72dbb864528c59c42954", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "9e3e8604821fd9f2a8ba7e8c849826cbdd7d5d02f6ea56e801d07eb9fd14652b" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "modeset.wasm", + "mirrorPath": "modeset.wasm", + "outputName": "modeset", + "forkInstrumentation": "auto" + } + ] + }, + "msmtpd": { + "manifestSha256": "09de04a422ddf631a29a259d816e081f8d317d1077808ede55d8c500baae748f", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "11fb3461f854cdf5fcd76e3056b0d1e50d4b1c01658a9b7806665b041db1a3c8" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "msmtpd.wasm", + "mirrorPath": "msmtpd.wasm", + "outputName": "msmtpd", + "forkInstrumentation": "auto" + } + ] + }, + "nano": { + "manifestSha256": "1ba6d340c95581319982257afd6a3554b333b880a7e69991b8c573da883f86c4", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "afea62511b76715a3494ec0b383cc978c2ac2bc7bfe5c9bdb8fc44b2e4078c68" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "nano.wasm", + "mirrorPath": "nano.wasm", + "outputName": "nano", + "forkInstrumentation": "auto" + } + ] + }, + "ncurses": { + "manifestSha256": "120bef02a4783a59f86f20a1953b120b603851e04c7d8400f16435f97f97779c", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "f7210d1ab36ab5f22676d221945bd944c58e35958947358e9824da81aac15f01" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "clear.wasm", + "mirrorPath": "ncurses/clear.wasm", + "outputName": "clear", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "reset.wasm", + "mirrorPath": "ncurses/reset.wasm", + "outputName": "reset", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "tset.wasm", + "mirrorPath": "ncurses/tset.wasm", + "outputName": "tset", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "tput.wasm", + "mirrorPath": "ncurses/tput.wasm", + "outputName": "tput", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "tabs.wasm", + "mirrorPath": "ncurses/tabs.wasm", + "outputName": "tabs", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "tic.wasm", + "mirrorPath": "ncurses/tic.wasm", + "outputName": "tic", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "infocmp.wasm", + "mirrorPath": "ncurses/infocmp.wasm", + "outputName": "infocmp", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "toe.wasm", + "mirrorPath": "ncurses/toe.wasm", + "outputName": "toe", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "captoinfo.wasm", + "mirrorPath": "ncurses/captoinfo.wasm", + "outputName": "captoinfo", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "infotocap.wasm", + "mirrorPath": "ncurses/infotocap.wasm", + "outputName": "infotocap", + "forkInstrumentation": "auto" + } + ] + }, + "netcat": { + "manifestSha256": "4cf33cd1ab768b3ad0108da8b68c2ff72e98469cd86a0bf2d0da54a7d4f6fa16", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "b78885a086ad183e20c8a93c6b657a5ce056046d54cf8145c33bcaa5bfdd5cdc" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "nc.wasm", + "mirrorPath": "nc.wasm", + "outputName": "nc", + "forkInstrumentation": "auto" + } + ] + }, + "nethack": { + "manifestSha256": "1a2f12ec2770bd8d40006d6c878a3493b97c71c2bb63ad08c7f6ad99bfd53504", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "11b1e47c7777eaf5ea106a36a0d7e70cce14a31d52005542c31222ef44473319" + }, + "dependencyClosures": { + "wasm32": [ + { + "packageName": "ncurses", + "manifestSha256": "120bef02a4783a59f86f20a1953b120b603851e04c7d8400f16435f97f97779c", + "cacheKey": "f7210d1ab36ab5f22676d221945bd944c58e35958947358e9824da81aac15f01" + } + ] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "nethack.wasm", + "mirrorPath": "nethack.wasm", + "outputName": "nethack", + "forkInstrumentation": "auto" + } + ] + }, + "nethack-browser-bundle": { + "manifestSha256": "b8294981da7ce4299dfb271d4aecb90ababa97a4d1acc9b716ae05bbe52beb53", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "c50cb85a33c8e4dd8157998a3c7fde468d33193054b7f0ce6cef111ccb26b011" + }, + "dependencyClosures": { + "wasm32": [ + { + "packageName": "ncurses", + "manifestSha256": "120bef02a4783a59f86f20a1953b120b603851e04c7d8400f16435f97f97779c", + "cacheKey": "f7210d1ab36ab5f22676d221945bd944c58e35958947358e9824da81aac15f01" + }, + { + "packageName": "nethack", + "manifestSha256": "1a2f12ec2770bd8d40006d6c878a3493b97c71c2bb63ad08c7f6ad99bfd53504", + "cacheKey": "11b1e47c7777eaf5ea106a36a0d7e70cce14a31d52005542c31222ef44473319" + } + ] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "nethack.zip", + "mirrorPath": "nethack.zip", + "outputName": "nethack", + "forkInstrumentation": "auto" + } + ] + }, + "nginx": { + "manifestSha256": "41b0aaf05fdd0d8da42642703f9e60fd7b9545d8029896918b87c771c359147b", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "7f69eae6a9d05d93c4458093a1ac80d5685ea4ff991c948274400796c770f690" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "nginx.wasm", + "mirrorPath": "nginx.wasm", + "outputName": "nginx", + "forkInstrumentation": "auto" + } + ] + }, + "node": { + "manifestSha256": "2131a24dfda8587860e57fc65b573086477d376b065b3b9ca4e1dfe4d79f5790", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "286d3611dc0a71406479ac61a2a0f46f8d51683a78ddb2f59fdddfacf81b9515" + }, + "dependencyClosures": { + "wasm32": [ + { + "packageName": "libcxx", + "manifestSha256": "6e1ebfa6914043770eae2a6c156b915acf6a56fe763f07d215ffe6f6f2305007", + "cacheKey": "6d08aa913e932d33232f8076de99312f87dd7b4d866c31f1c168999910a4fffd" + }, + { + "packageName": "openssl", + "manifestSha256": "374f5dce6b2691b630b55cb82955b33d2305384e4bd64746c581621bb0793993", + "cacheKey": "ef7615079b83e8bb69afeabcc5abaf026b5fc16a6a4b6234e25d762effdcb362" + }, + { + "packageName": "spidermonkey", + "manifestSha256": "c72ef4e43ee2a4fc74390d03a239fce3b545a6d3b64f02ccc4d44972cec626bd", + "cacheKey": "85aca810215b5f02dcb66956292be3b345aa4469fa34c70a6cd602b8016909dd" + }, + { + "packageName": "zlib", + "manifestSha256": "d4f4701c0c30dc843ebe2111ed3e2284836cdfc4bc05a130c8fab4f31b8e6357", + "cacheKey": "d17bfc607f79188de27ed14ceac45d191f537d7376d8e69ab83e01a374d5428e" + } + ] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "node.wasm", + "mirrorPath": "node.wasm", + "outputName": "node", + "forkInstrumentation": "disabled" + } + ] + }, + "node-vfs": { + "manifestSha256": "33315fb1b3030a4c187ae075eac08f717de8d7ab017b86c6458778ac9070eece", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "e12fabcaa9d24c48a5e3c331940eaa45638b408aff50f0955ad4ed0065acb8b8" + }, + "dependencyClosures": { + "wasm32": [ + { + "packageName": "libcxx", + "manifestSha256": "6e1ebfa6914043770eae2a6c156b915acf6a56fe763f07d215ffe6f6f2305007", + "cacheKey": "6d08aa913e932d33232f8076de99312f87dd7b4d866c31f1c168999910a4fffd" + }, + { + "packageName": "node", + "manifestSha256": "2131a24dfda8587860e57fc65b573086477d376b065b3b9ca4e1dfe4d79f5790", + "cacheKey": "286d3611dc0a71406479ac61a2a0f46f8d51683a78ddb2f59fdddfacf81b9515" + }, + { + "packageName": "openssl", + "manifestSha256": "374f5dce6b2691b630b55cb82955b33d2305384e4bd64746c581621bb0793993", + "cacheKey": "ef7615079b83e8bb69afeabcc5abaf026b5fc16a6a4b6234e25d762effdcb362" + }, + { + "packageName": "shell", + "manifestSha256": "5d02cd8fee4e0d46c7552a3046c3898f9e3bd5667094e02cab01851eed97a321", + "cacheKey": "feffb1d8d0b51ef0116bc99f0f7989127ed9bc414ce54b459a8232df05f0a5d2" + }, + { + "packageName": "spidermonkey", + "manifestSha256": "c72ef4e43ee2a4fc74390d03a239fce3b545a6d3b64f02ccc4d44972cec626bd", + "cacheKey": "85aca810215b5f02dcb66956292be3b345aa4469fa34c70a6cd602b8016909dd" + }, + { + "packageName": "zlib", + "manifestSha256": "d4f4701c0c30dc843ebe2111ed3e2284836cdfc4bc05a130c8fab4f31b8e6357", + "cacheKey": "d17bfc607f79188de27ed14ceac45d191f537d7376d8e69ab83e01a374d5428e" + } + ] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "node-vfs.vfs.zst", + "mirrorPath": "node-vfs.vfs.zst", + "outputName": "node-vfs", + "forkInstrumentation": "disabled" + } + ] + }, + "perl": { + "manifestSha256": "6cdc4dbc54d0e4008cff41f82ec8918c0e44b4aef23903539c1bc0f538fe3ad2", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "f0d9f886fca2f6741563862628b059964bceefcdf2eb2a0b2ebfde5e30ac8adf" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "perl.wasm", + "mirrorPath": "perl.wasm", + "outputName": "perl", + "forkInstrumentation": "auto" + } + ] + }, + "perl-vfs": { + "manifestSha256": "1345cc102bc8c24a6bc276410c00b4fcc99ba5f6adc9b031f882aaa35d53c8bc", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "cdfb0cbc41a2413da84de6cf209a510219536837bb955a79103576901a94b063" + }, + "dependencyClosures": { + "wasm32": [ + { + "packageName": "perl", + "manifestSha256": "6cdc4dbc54d0e4008cff41f82ec8918c0e44b4aef23903539c1bc0f538fe3ad2", + "cacheKey": "f0d9f886fca2f6741563862628b059964bceefcdf2eb2a0b2ebfde5e30ac8adf" + } + ] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "perl-vfs.vfs.zst", + "mirrorPath": "perl-vfs.vfs.zst", + "outputName": "perl-vfs", + "forkInstrumentation": "auto" + } + ] + }, + "php": { + "manifestSha256": "fcd9d7915ed8935b05418e148edcb312adf0b1e4e627fe4aab75444b35b33bbc", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "2b3b191f35c8adbfccdcfc138af00f8c00855e808ffb7336b503ac50f224b4a6" + }, + "dependencyClosures": { + "wasm32": [ + { + "packageName": "icu", + "manifestSha256": "f5b1f02d169ec1108fc32d9da55be53c3efb96e3a7aeba6a03aec62e0175f0dc", + "cacheKey": "5f2d9559cd1173f21fa621bcdf5451a35f3ffcc3435dd7b0e09169834092e402" + }, + { + "packageName": "libcurl", + "manifestSha256": "c771e6cdc83b43840db4dd1fdce4b6189ba4f281596f7485bcda9c65e2ecda00", + "cacheKey": "cb147677f78c356f20e2581df7dbaab3c77f6e3b879c33f24484bb4eafa6160b" + }, + { + "packageName": "libcxx", + "manifestSha256": "6e1ebfa6914043770eae2a6c156b915acf6a56fe763f07d215ffe6f6f2305007", + "cacheKey": "6d08aa913e932d33232f8076de99312f87dd7b4d866c31f1c168999910a4fffd" + }, + { + "packageName": "libiconv", + "manifestSha256": "fa60b386f4ba0cd5bbbd595f67477cbfb0cc401a14bbd4769630c502bb3f24b1", + "cacheKey": "9328da7f90a267e6eabfb49e8b00d0dd45296f0d9c4339fa1c58e2813def6c70" + }, + { + "packageName": "libxml2", + "manifestSha256": "908490efcdc7783c76ce36af31d1db24bfcc7ba5f1e7b9b5298cc81cb110256c", + "cacheKey": "ecbc5a52e3f7b2cc5d697d3982fe94e9dbc69e8354f62f145dad302437b1ce61" + }, + { + "packageName": "libzip", + "manifestSha256": "83c65d63a6416e79436bf4e792759a4701d568df0e8c010394a9732cf659f0cc", + "cacheKey": "bb7925958ee1261e640a025e5801ffed415d4486e4a6339094f0fd7338f47073" + }, + { + "packageName": "openssl", + "manifestSha256": "374f5dce6b2691b630b55cb82955b33d2305384e4bd64746c581621bb0793993", + "cacheKey": "ef7615079b83e8bb69afeabcc5abaf026b5fc16a6a4b6234e25d762effdcb362" + }, + { + "packageName": "sqlite", + "manifestSha256": "ab9560b4eab53445034e2a87a27ccee9d05ddb0db73fee28f5000325b0b1cba5", + "cacheKey": "3cabb24c5250387ef8e26e0a1566cb7b198a8783c3cc30fb2ae31c8d6d044e77" + }, + { + "packageName": "zlib", + "manifestSha256": "d4f4701c0c30dc843ebe2111ed3e2284836cdfc4bc05a130c8fab4f31b8e6357", + "cacheKey": "d17bfc607f79188de27ed14ceac45d191f537d7376d8e69ab83e01a374d5428e" + } + ] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "php.wasm", + "mirrorPath": "php/php.wasm", + "outputName": "php", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "php-fpm.wasm", + "mirrorPath": "php/php-fpm.wasm", + "outputName": "php-fpm", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "opcache.so", + "mirrorPath": "php/opcache.so", + "outputName": "opcache", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "curl.so", + "mirrorPath": "php/curl.so", + "outputName": "curl", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "phar.so", + "mirrorPath": "php/phar.so", + "outputName": "phar", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "zend_test.so", + "mirrorPath": "php/zend_test.so", + "outputName": "zend_test", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "zip.so", + "mirrorPath": "php/zip.so", + "outputName": "zip", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "intl.so", + "mirrorPath": "php/intl.so", + "outputName": "intl", + "forkInstrumentation": "auto" + }, + { + "kind": "runtime-file", + "sourceArtifact": "icu.dat", + "mirrorPath": "php/icu.dat", + "guestPath": "/usr/lib/php/icu.dat", + "mode": 420 + } + ] + }, + "posix-utils-lite": { + "manifestSha256": "8fd7190b2848ef80143adc7b9268c79e13cd4db3430f7105faf49a4b4407a06f", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "304227932e1878ec59ab186b6d025f43a2b3df928484313d2ab931a46a5f3ee7" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "ar.wasm", + "mirrorPath": "posix-utils-lite/ar.wasm", + "outputName": "ar", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "asa.wasm", + "mirrorPath": "posix-utils-lite/asa.wasm", + "outputName": "asa", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "cal.wasm", + "mirrorPath": "posix-utils-lite/cal.wasm", + "outputName": "cal", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "cflow.wasm", + "mirrorPath": "posix-utils-lite/cflow.wasm", + "outputName": "cflow", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "compress.wasm", + "mirrorPath": "posix-utils-lite/compress.wasm", + "outputName": "compress", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "ctags.wasm", + "mirrorPath": "posix-utils-lite/ctags.wasm", + "outputName": "ctags", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "cxref.wasm", + "mirrorPath": "posix-utils-lite/cxref.wasm", + "outputName": "cxref", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "ed.wasm", + "mirrorPath": "posix-utils-lite/ed.wasm", + "outputName": "ed", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "ex.wasm", + "mirrorPath": "posix-utils-lite/ex.wasm", + "outputName": "ex", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "fuser.wasm", + "mirrorPath": "posix-utils-lite/fuser.wasm", + "outputName": "fuser", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "gencat.wasm", + "mirrorPath": "posix-utils-lite/gencat.wasm", + "outputName": "gencat", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "getconf.wasm", + "mirrorPath": "posix-utils-lite/getconf.wasm", + "outputName": "getconf", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "gettext.wasm", + "mirrorPath": "posix-utils-lite/gettext.wasm", + "outputName": "gettext", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "iconv.wasm", + "mirrorPath": "posix-utils-lite/iconv.wasm", + "outputName": "iconv", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "ipcrm.wasm", + "mirrorPath": "posix-utils-lite/ipcrm.wasm", + "outputName": "ipcrm", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "ipcs.wasm", + "mirrorPath": "posix-utils-lite/ipcs.wasm", + "outputName": "ipcs", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "lex.wasm", + "mirrorPath": "posix-utils-lite/lex.wasm", + "outputName": "lex", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "locale.wasm", + "mirrorPath": "posix-utils-lite/locale.wasm", + "outputName": "locale", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "logger.wasm", + "mirrorPath": "posix-utils-lite/logger.wasm", + "outputName": "logger", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "man.wasm", + "mirrorPath": "posix-utils-lite/man.wasm", + "outputName": "man", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "more.wasm", + "mirrorPath": "posix-utils-lite/more.wasm", + "outputName": "more", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "msgfmt.wasm", + "mirrorPath": "posix-utils-lite/msgfmt.wasm", + "outputName": "msgfmt", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "ngettext.wasm", + "mirrorPath": "posix-utils-lite/ngettext.wasm", + "outputName": "ngettext", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "nm.wasm", + "mirrorPath": "posix-utils-lite/nm.wasm", + "outputName": "nm", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "patch.wasm", + "mirrorPath": "posix-utils-lite/patch.wasm", + "outputName": "patch", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "pax.wasm", + "mirrorPath": "posix-utils-lite/pax.wasm", + "outputName": "pax", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "pgrep.wasm", + "mirrorPath": "posix-utils-lite/pgrep.wasm", + "outputName": "pgrep", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "ps.wasm", + "mirrorPath": "posix-utils-lite/ps.wasm", + "outputName": "ps", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "renice.wasm", + "mirrorPath": "posix-utils-lite/renice.wasm", + "outputName": "renice", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "strings.wasm", + "mirrorPath": "posix-utils-lite/strings.wasm", + "outputName": "strings", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "strip.wasm", + "mirrorPath": "posix-utils-lite/strip.wasm", + "outputName": "strip", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "uncompress.wasm", + "mirrorPath": "posix-utils-lite/uncompress.wasm", + "outputName": "uncompress", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "uudecode.wasm", + "mirrorPath": "posix-utils-lite/uudecode.wasm", + "outputName": "uudecode", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "uuencode.wasm", + "mirrorPath": "posix-utils-lite/uuencode.wasm", + "outputName": "uuencode", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "what.wasm", + "mirrorPath": "posix-utils-lite/what.wasm", + "outputName": "what", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "xgettext.wasm", + "mirrorPath": "posix-utils-lite/xgettext.wasm", + "outputName": "xgettext", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "yacc.wasm", + "mirrorPath": "posix-utils-lite/yacc.wasm", + "outputName": "yacc", + "forkInstrumentation": "auto" + } + ] + }, + "python-vfs": { + "manifestSha256": "d1aa99feae65f06c0ca8cf52c2176534b2723fd4ee6eba6e72c205245e1c9c26", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "9ca6745b74b81056ebc4fed975af96753dbca7508cc2e4ad66c20809c1a2f323" + }, + "dependencyClosures": { + "wasm32": [ + { + "packageName": "cpython", + "manifestSha256": "7dd4f446697a73941ec940c2ccba4d53be73fe6947f1e701031a4d0aa964c4ff", + "cacheKey": "9ebc9502240decca75ad4c6e155e6748a3d07d4f6fdb695b65d01ea799bc07af" + }, + { + "packageName": "zlib", + "manifestSha256": "d4f4701c0c30dc843ebe2111ed3e2284836cdfc4bc05a130c8fab4f31b8e6357", + "cacheKey": "d17bfc607f79188de27ed14ceac45d191f537d7376d8e69ab83e01a374d5428e" + } + ] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "python-vfs.vfs.zst", + "mirrorPath": "python-vfs.vfs.zst", + "outputName": "python-vfs", + "forkInstrumentation": "auto" + } + ] + }, + "redis": { + "manifestSha256": "151fb507de953ba2ba94b8f881bc7d66b4c741c1359a2f590cc07f9a4cd368a3", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "a7391e556a100804e931a940f6a88b3afee2d7a866fa315e7787a9a5a8948403" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "redis-server.wasm", + "mirrorPath": "redis/redis-server.wasm", + "outputName": "redis-server", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "redis-cli.wasm", + "mirrorPath": "redis/redis-cli.wasm", + "outputName": "redis-cli", + "forkInstrumentation": "auto" + } + ] + }, + "rootfs": { + "manifestSha256": "0420201025170f48c32949ac62707d4577cdc13a53ad1f01f59d318ec07c8d6e", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "afd016a598b5bacde4a1275a142a2fb517a51dc22e120d9562c286765091324b" + }, + "dependencyClosures": { + "wasm32": [ + { + "packageName": "bash", + "manifestSha256": "6478060f28d430d18a6ebe7c603392d35a41d9bcf6bc74322351d1450b9c5335", + "cacheKey": "a43cb889225cd487f93af47eafe907467d21df9bc04428a5ad3df7c57cf0c6ba" + }, + { + "packageName": "bc", + "manifestSha256": "a65661463bb7047b91ff00153bd99fd962c96e571934ab4e923f5215fb6ecfbd", + "cacheKey": "5cd794c0a8203fa7a96dce3c944b5b5173ff3f39dd08a3ba08635ad5f65cd1e7" + }, + { + "packageName": "coreutils", + "manifestSha256": "b8baabc9af9283434e0d80f7fdc0aaef242ffa31092ab0fc39a0603a12b3cca7", + "cacheKey": "0d7bcbc4565d31d275d3dda0357abbf40747e5f6f1407762ad00b4ce0673b0ce" + }, + { + "packageName": "dash", + "manifestSha256": "10ce1bb611fefd78a6c9ec9038f6113e29a7062b1a73e08eccf9b33004cea9a8", + "cacheKey": "30d3944c0d6c187c6eed2387e81b4be46eeb7ea9c1627f216499eded89650310" + }, + { + "packageName": "diffutils", + "manifestSha256": "3a78f0a46bae43ce5ea235c6638c2cbd1d8559b0b62b1fb42443b35968c1aca3", + "cacheKey": "b1c53d03d0e67919a48f3d5eb38e4e9fc7397cfff56a6c47fc9a9faec895d21e" + }, + { + "packageName": "file", + "manifestSha256": "7874c1affcbbf2c8ab8c8d4beb087f275af57b5caae6a8947bf5a63a181552b2", + "cacheKey": "28c5bfeb2dddaebf9cc39451d2c2983812bd6c61abc05197ed584c51f310a2b1" + }, + { + "packageName": "findutils", + "manifestSha256": "cceedf52aea67fbb0da03cfce6f2e9b1a7b5561fa1656c2762b43c651a625d02", + "cacheKey": "51fb9260ceb9656bbb34bf723314ea6e6905915d65b736845f0fbf593b5eef00" + }, + { + "packageName": "gawk", + "manifestSha256": "a2567b8b0778805e385e1d3a41ac8b5d74aaf9d293b222001b17d09b11e5a0ba", + "cacheKey": "cee993ecea8042490957d500095605e9635f5e3247fff32b160a3569749e53fc" + }, + { + "packageName": "grep", + "manifestSha256": "59270d3bfb33167b32d13246bf855b49308f8c9c0a66d9635c804f702421fbc6", + "cacheKey": "cf4ea02c0d5ffbdec575a9fcce7c943d8d45bac06d8ee223171a3b41b6fa8e52" + }, + { + "packageName": "m4", + "manifestSha256": "2c6582d99d6eabfb9da52badf49b899e9fdcba76a728536b25bc3741f3331543", + "cacheKey": "2a76adafbe9deb20f29041be2fe5cba99e207f28afec04911d311d3387f9e664" + }, + { + "packageName": "make", + "manifestSha256": "f878d2d730f36a4c6dfe1fff1b4ccc704757ea22d8c4c8ccb95b11cd641e5fed", + "cacheKey": "13cd37652953e64a44481f086429f2a44bd8a0dc66824eb7e1ae3bf2e6e08fe0" + }, + { + "packageName": "ncurses", + "manifestSha256": "120bef02a4783a59f86f20a1953b120b603851e04c7d8400f16435f97f97779c", + "cacheKey": "f7210d1ab36ab5f22676d221945bd944c58e35958947358e9824da81aac15f01" + }, + { + "packageName": "posix-utils-lite", + "manifestSha256": "8fd7190b2848ef80143adc7b9268c79e13cd4db3430f7105faf49a4b4407a06f", + "cacheKey": "304227932e1878ec59ab186b6d025f43a2b3df928484313d2ab931a46a5f3ee7" + }, + { + "packageName": "sed", + "manifestSha256": "2a08e9c5dacc5facc8983c1ff35ff2a72db328405e9a1f464cc7ad155c4d08af", + "cacheKey": "f687903421a723b81ead0a60f488b617375ce19ed97edc2820818f96f90e3c82" + } + ] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "rootfs.vfs", + "mirrorPath": "rootfs.vfs", + "outputName": "rootfs", + "forkInstrumentation": "auto" + } + ] + }, + "ruby": { + "manifestSha256": "6ea67a246c29cd927a19decbb36ae4c4d478ff6642d2c77e08a6b2cfeb8251d2", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "84ffe7f564d7631b51dd3951ff655eef315a89691e9efaa3791831643bd1e985" + }, + "dependencyClosures": { + "wasm32": [ + { + "packageName": "zlib", + "manifestSha256": "d4f4701c0c30dc843ebe2111ed3e2284836cdfc4bc05a130c8fab4f31b8e6357", + "cacheKey": "d17bfc607f79188de27ed14ceac45d191f537d7376d8e69ab83e01a374d5428e" + } + ] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "ruby.wasm", + "mirrorPath": "ruby/ruby.wasm", + "outputName": "ruby", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "ruby-runtime.zip", + "mirrorPath": "ruby/ruby-runtime.zip", + "outputName": "ruby-runtime", + "forkInstrumentation": "auto" + } + ] + }, + "sed": { + "manifestSha256": "2a08e9c5dacc5facc8983c1ff35ff2a72db328405e9a1f464cc7ad155c4d08af", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "f687903421a723b81ead0a60f488b617375ce19ed97edc2820818f96f90e3c82" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "sed.wasm", + "mirrorPath": "sed.wasm", + "outputName": "sed", + "forkInstrumentation": "auto" + } + ] + }, + "shell": { + "manifestSha256": "5d02cd8fee4e0d46c7552a3046c3898f9e3bd5667094e02cab01851eed97a321", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "feffb1d8d0b51ef0116bc99f0f7989127ed9bc414ce54b459a8232df05f0a5d2" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "shell.vfs.zst", + "mirrorPath": "shell.vfs.zst", + "outputName": "shell", + "forkInstrumentation": "auto" + } + ] + }, + "spidermonkey": { + "manifestSha256": "c72ef4e43ee2a4fc74390d03a239fce3b545a6d3b64f02ccc4d44972cec626bd", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "85aca810215b5f02dcb66956292be3b345aa4469fa34c70a6cd602b8016909dd" + }, + "dependencyClosures": { + "wasm32": [ + { + "packageName": "libcxx", + "manifestSha256": "6e1ebfa6914043770eae2a6c156b915acf6a56fe763f07d215ffe6f6f2305007", + "cacheKey": "6d08aa913e932d33232f8076de99312f87dd7b4d866c31f1c168999910a4fffd" + }, + { + "packageName": "openssl", + "manifestSha256": "374f5dce6b2691b630b55cb82955b33d2305384e4bd64746c581621bb0793993", + "cacheKey": "ef7615079b83e8bb69afeabcc5abaf026b5fc16a6a4b6234e25d762effdcb362" + }, + { + "packageName": "zlib", + "manifestSha256": "d4f4701c0c30dc843ebe2111ed3e2284836cdfc4bc05a130c8fab4f31b8e6357", + "cacheKey": "d17bfc607f79188de27ed14ceac45d191f537d7376d8e69ab83e01a374d5428e" + } + ] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "js.wasm", + "mirrorPath": "js.wasm", + "outputName": "js", + "forkInstrumentation": "disabled" + } + ] + }, + "spidermonkey-node": { + "manifestSha256": "f156c4c5044d2a9447324e7a2a35f8c3e6fa367b3e44f674dcc30ab6b946fae7", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "4867322d0e6f96809141a2dce6682483a2a6743899f5fd75f2271d82475f0269" + }, + "dependencyClosures": { + "wasm32": [ + { + "packageName": "libcxx", + "manifestSha256": "6e1ebfa6914043770eae2a6c156b915acf6a56fe763f07d215ffe6f6f2305007", + "cacheKey": "6d08aa913e932d33232f8076de99312f87dd7b4d866c31f1c168999910a4fffd" + }, + { + "packageName": "openssl", + "manifestSha256": "374f5dce6b2691b630b55cb82955b33d2305384e4bd64746c581621bb0793993", + "cacheKey": "ef7615079b83e8bb69afeabcc5abaf026b5fc16a6a4b6234e25d762effdcb362" + }, + { + "packageName": "spidermonkey", + "manifestSha256": "c72ef4e43ee2a4fc74390d03a239fce3b545a6d3b64f02ccc4d44972cec626bd", + "cacheKey": "85aca810215b5f02dcb66956292be3b345aa4469fa34c70a6cd602b8016909dd" + }, + { + "packageName": "zlib", + "manifestSha256": "d4f4701c0c30dc843ebe2111ed3e2284836cdfc4bc05a130c8fab4f31b8e6357", + "cacheKey": "d17bfc607f79188de27ed14ceac45d191f537d7376d8e69ab83e01a374d5428e" + } + ] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "node.wasm", + "mirrorPath": "spidermonkey-node.wasm", + "outputName": "spidermonkey-node", + "forkInstrumentation": "disabled" + } + ] + }, + "sqlite-cli": { + "manifestSha256": "1cebb768f9473dc58fc6e72c9b24565760aedaec4cbdb29ad43b22a0a5fd7030", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "a26d9e2b82748828e1f2e322c535892993237673d1759269341ca7e3d55b77ae" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "sqlite3.wasm", + "mirrorPath": "sqlite-cli.wasm", + "outputName": "sqlite-cli", + "forkInstrumentation": "auto" + } + ] + }, + "tar": { + "manifestSha256": "08fa090c122d3105c735d74560bdd7b8b083a6f9ca8bdec17de0b4993e1be7fd", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "bedb0c34a78b5c4642b340e115feca87ed3752f4a836d3b5534716e3137c034c" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "tar.wasm", + "mirrorPath": "tar.wasm", + "outputName": "tar", + "forkInstrumentation": "auto" + } + ] + }, + "tcl": { + "manifestSha256": "67253d47de7df9184e68ec3c49835455746941835a26db9e588aa8fbee7d4636", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "e18543aa20544ced46cb8a2a7bbee5d5227ec0c1076336eb3db6ad70ad3c4095" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "tclsh.wasm", + "mirrorPath": "tcl.wasm", + "outputName": "tcl", + "forkInstrumentation": "auto" + } + ] + }, + "texlive": { + "manifestSha256": "028ada023f8a8f9966c8a28575242a339aaaf0e8870fe4a221986133b381c3c0", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "ff321955a73f9ce9c018eeb9faba1df0274ed6087f6df60f5903dcce9e05956c" + }, + "dependencyClosures": { + "wasm32": [ + { + "packageName": "libpng", + "manifestSha256": "79ed5c5c072a0267cce5c7a2fe37d8494571b17e44b952f619e04ec1c4a9db10", + "cacheKey": "cefa8c2f3cb40a07bba56f1649a7d12f095d3d59c6f1b114fb9f0dae0147633b" + }, + { + "packageName": "zlib", + "manifestSha256": "d4f4701c0c30dc843ebe2111ed3e2284836cdfc4bc05a130c8fab4f31b8e6357", + "cacheKey": "d17bfc607f79188de27ed14ceac45d191f537d7376d8e69ab83e01a374d5428e" + } + ] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "pdftex.wasm", + "mirrorPath": "texlive/pdftex.wasm", + "outputName": "pdftex", + "forkInstrumentation": "auto" + }, + { + "kind": "output", + "sourceArtifact": "texlive-bundle.json", + "mirrorPath": "texlive/texlive-bundle.json", + "outputName": "texlive-bundle", + "forkInstrumentation": "auto" + } + ] + }, + "unzip": { + "manifestSha256": "ef4e5e34258827ab3cbc1930da3fd9514f08f2d7d56c7c3e0d5c495a4d3a20e9", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "cc555c2eb64b33f1ca29d997a36c1e1e98df74a5411b0992dc4127352e6ffefa" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "unzip.wasm", + "mirrorPath": "unzip.wasm", + "outputName": "unzip", + "forkInstrumentation": "auto" + } + ] + }, + "vim": { + "manifestSha256": "c211660ef41d01f95f3fbdf675b74891e897e3f304e04f1bf5586f326007382c", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "a4a47859e8a0a1d8f627327f1fb502f2ac7fee615985fa030b5f9fd88d03752b" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "vim.wasm", + "mirrorPath": "vim.wasm", + "outputName": "vim", + "forkInstrumentation": "auto" + } + ] + }, + "vim-browser-bundle": { + "manifestSha256": "e31d84057db49c3534381604d0c8f3c7d765e33ede560a7ea7b01a5a8f1aa0d3", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "cf5f9141bfc5047f7548121cafb993708a8da25b2639967cc603e4d0347a4347" + }, + "dependencyClosures": { + "wasm32": [ + { + "packageName": "vim", + "manifestSha256": "c211660ef41d01f95f3fbdf675b74891e897e3f304e04f1bf5586f326007382c", + "cacheKey": "a4a47859e8a0a1d8f627327f1fb502f2ac7fee615985fa030b5f9fd88d03752b" + } + ] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "vim.zip", + "mirrorPath": "vim.zip", + "outputName": "vim", + "forkInstrumentation": "auto" + } + ] + }, + "wget": { + "manifestSha256": "d3c7ba9bc1ae708b99850a6eb2cae521c85bf1010c90135a935f5dfae57a53a0", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "97d80a049d3f04139d1048566d4b0517b73e1200b06c9bb6745c065f5f1ca75d" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "wget.wasm", + "mirrorPath": "wget.wasm", + "outputName": "wget", + "forkInstrumentation": "auto" + } + ] + }, + "wordpress": { + "manifestSha256": "2c95e72f657a05be445c9b264de769becfb50833d6ef7ee4d18feac690ca2b52", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "7786ef684b85a94ebfaa5aa7bc4f336a0e9a6d632cca51b8bba7efe17e2d17ca" + }, + "dependencyClosures": { + "wasm32": [ + { + "packageName": "dinit", + "manifestSha256": "44e125e1337503e27f8cff531cdbd79c4f406fd2d282b916a7746cf3a14057bf", + "cacheKey": "e4d6e040b3c583d15e3a80feea9db153c1e8b0ff02bc86d00fa28e5870287781" + }, + { + "packageName": "icu", + "manifestSha256": "f5b1f02d169ec1108fc32d9da55be53c3efb96e3a7aeba6a03aec62e0175f0dc", + "cacheKey": "5f2d9559cd1173f21fa621bcdf5451a35f3ffcc3435dd7b0e09169834092e402" + }, + { + "packageName": "libcurl", + "manifestSha256": "c771e6cdc83b43840db4dd1fdce4b6189ba4f281596f7485bcda9c65e2ecda00", + "cacheKey": "cb147677f78c356f20e2581df7dbaab3c77f6e3b879c33f24484bb4eafa6160b" + }, + { + "packageName": "libcxx", + "manifestSha256": "6e1ebfa6914043770eae2a6c156b915acf6a56fe763f07d215ffe6f6f2305007", + "cacheKey": "6d08aa913e932d33232f8076de99312f87dd7b4d866c31f1c168999910a4fffd" + }, + { + "packageName": "libiconv", + "manifestSha256": "fa60b386f4ba0cd5bbbd595f67477cbfb0cc401a14bbd4769630c502bb3f24b1", + "cacheKey": "9328da7f90a267e6eabfb49e8b00d0dd45296f0d9c4339fa1c58e2813def6c70" + }, + { + "packageName": "libxml2", + "manifestSha256": "908490efcdc7783c76ce36af31d1db24bfcc7ba5f1e7b9b5298cc81cb110256c", + "cacheKey": "ecbc5a52e3f7b2cc5d697d3982fe94e9dbc69e8354f62f145dad302437b1ce61" + }, + { + "packageName": "libzip", + "manifestSha256": "83c65d63a6416e79436bf4e792759a4701d568df0e8c010394a9732cf659f0cc", + "cacheKey": "bb7925958ee1261e640a025e5801ffed415d4486e4a6339094f0fd7338f47073" + }, + { + "packageName": "msmtpd", + "manifestSha256": "09de04a422ddf631a29a259d816e081f8d317d1077808ede55d8c500baae748f", + "cacheKey": "11fb3461f854cdf5fcd76e3056b0d1e50d4b1c01658a9b7806665b041db1a3c8" + }, + { + "packageName": "nginx", + "manifestSha256": "41b0aaf05fdd0d8da42642703f9e60fd7b9545d8029896918b87c771c359147b", + "cacheKey": "7f69eae6a9d05d93c4458093a1ac80d5685ea4ff991c948274400796c770f690" + }, + { + "packageName": "openssl", + "manifestSha256": "374f5dce6b2691b630b55cb82955b33d2305384e4bd64746c581621bb0793993", + "cacheKey": "ef7615079b83e8bb69afeabcc5abaf026b5fc16a6a4b6234e25d762effdcb362" + }, + { + "packageName": "php", + "manifestSha256": "fcd9d7915ed8935b05418e148edcb312adf0b1e4e627fe4aab75444b35b33bbc", + "cacheKey": "2b3b191f35c8adbfccdcfc138af00f8c00855e808ffb7336b503ac50f224b4a6" + }, + { + "packageName": "shell", + "manifestSha256": "5d02cd8fee4e0d46c7552a3046c3898f9e3bd5667094e02cab01851eed97a321", + "cacheKey": "feffb1d8d0b51ef0116bc99f0f7989127ed9bc414ce54b459a8232df05f0a5d2" + }, + { + "packageName": "sqlite", + "manifestSha256": "ab9560b4eab53445034e2a87a27ccee9d05ddb0db73fee28f5000325b0b1cba5", + "cacheKey": "3cabb24c5250387ef8e26e0a1566cb7b198a8783c3cc30fb2ae31c8d6d044e77" + }, + { + "packageName": "zlib", + "manifestSha256": "d4f4701c0c30dc843ebe2111ed3e2284836cdfc4bc05a130c8fab4f31b8e6357", + "cacheKey": "d17bfc607f79188de27ed14ceac45d191f537d7376d8e69ab83e01a374d5428e" + } + ] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "wordpress.vfs.zst", + "mirrorPath": "wordpress.vfs.zst", + "outputName": "wordpress", + "forkInstrumentation": "auto" + } + ] + }, + "xz": { + "manifestSha256": "8707d35b8647b1af8fa165dcb6ce7b2a6a007e19ab2daca2680c39395864a737", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "e5a775c8202677e7819c82bfdc97d12e616873e776ee51b965246920ace2c052" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "xz.wasm", + "mirrorPath": "xz.wasm", + "outputName": "xz", + "forkInstrumentation": "auto" + } + ] + }, + "zip": { + "manifestSha256": "9c9cfa8220aaeaea26736b55f7cd0a7517b8853c07f9f0d241ad67dd43592e36", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "eadd23e06a078d5f6760074f74e2c2ab18572e92959082bd49018d547243e5c1" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "zip.wasm", + "mirrorPath": "zip.wasm", + "outputName": "zip", + "forkInstrumentation": "auto" + } + ] + }, + "zstd": { + "manifestSha256": "89f266938c2c253700a838e6352c9de66c976c84883325aaa979f72b732357e4", + "arches": [ + "wasm32" + ], + "cacheKeys": { + "wasm32": "f0cff41dee757dcf21bb7933a06c423a8c51eabb7bf94ff907e1d15b2db659ef" + }, + "dependencyClosures": { + "wasm32": [] + }, + "members": [ + { + "kind": "output", + "sourceArtifact": "zstd.wasm", + "mirrorPath": "zstd.wasm", + "outputName": "zstd", + "forkInstrumentation": "auto" + } + ] + } + } +} diff --git a/run.sh b/run.sh index 7ee05ad471..8d1b45b441 100755 --- a/run.sh +++ b/run.sh @@ -275,8 +275,9 @@ pkg_remove_local_output() { # or `local-binaries/`. This is the single source of truth for "is # this package built?" — replaces ~30 hand-coded has_ checks # that hardcoded the flat-vs-nested layout convention and silently -# drifted (e.g. the `programs/erlang.wasm` vs `programs/erlang/erlang.wasm` -# bug). Layout decisions live in `output_dest_rel_for` only. +# drifted when a package moved from scalar to package-directory layout +# (the Erlang build exposed this bug). Layout decisions live in +# `output_dest_rel_for` only. # # The wasm-basename arg is the file listed in `[[outputs]].wasm` # (e.g. `python.wasm`, `mariadbd.wasm`), NOT the output `name` field. diff --git a/scripts/browser-binary-package-roots.mjs b/scripts/browser-binary-package-roots.mjs index d2a875a203..bf537a6742 100644 --- a/scripts/browser-binary-package-roots.mjs +++ b/scripts/browser-binary-package-roots.mjs @@ -6,7 +6,9 @@ import { readFileSync, statSync, } from "node:fs"; -import { dirname, join, resolve } from "node:path"; +import { createHash } from "node:crypto"; +import { parse } from "@babel/parser"; +import { dirname, isAbsolute, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; const scriptPath = fileURLToPath(import.meta.url); @@ -22,12 +24,57 @@ export const registryPackagesWithoutBuildToml = new Set([ "sqlite-cli", ]); -function registryPackageDirs(repoRoot) { - const registryRoot = join(repoRoot, "packages", "registry"); - return readdirSync(registryRoot) - .map((name) => join(registryRoot, name)) - .filter((path) => statSync(path).isDirectory()) - .filter((path) => existsSync(join(path, "package.toml"))); +function hasExactObjectKeys(value, expectedKeys) { + const actual = Object.keys(value).sort(); + const expected = [...expectedKeys].sort(); + return actual.length === expected.length + && actual.every((key, index) => key === expected[index]); +} + +function safeSinglePathComponent(value) { + return typeof value === "string" + && value.length > 0 + && value !== "." + && value !== ".." + && !value.includes("/") + && !value.includes("\\") + && !value.includes("\0"); +} + +function portableArtifactPath(value) { + return typeof value === "string" + && value.length > 0 + && !value.startsWith("/") + && !value.includes("\\") + && !value.includes("\0") + && value.split("/").every( + (component) => + component.length > 0 && component !== "." && component !== "..", + ); +} + +function filePathsConflict(left, right) { + return left === right + || left.startsWith(`${right}/`) + || right.startsWith(`${left}/`); +} + +export function configuredProgramRegistryRoots( + repoRoot = defaultRepoRoot, + registryPath = process.env.WASM_POSIX_DEPS_REGISTRY, +) { + if (registryPath === undefined) { + return [join(repoRoot, "packages", "registry")]; + } + return registryPath + .split(":") + .filter(Boolean) + .map((entry) => { + if (entry.startsWith("~/") && process.env.HOME !== undefined) { + return resolve(process.env.HOME, entry.slice(2)); + } + return isAbsolute(entry) ? resolve(entry) : resolve(repoRoot, entry); + }); } function walkFiles(root) { @@ -49,55 +96,386 @@ export function firstTomlString(text, key) { return match?.[1] ?? null; } -function parseArches(text) { - const match = text.match(/^\s*arches\s*=\s*\[([\s\S]*?)\]/m); - if (!match) return ["wasm32"]; - return [...match[1].matchAll(/"([^"]+)"/g)].map((arch) => arch[1]); +function readProgramPackageProjection(registryRoot) { + const indexPath = join(registryRoot, "program-packages.json"); + const parsed = JSON.parse(readFileSync(indexPath, "utf8")); + if ( + typeof parsed !== "object" + || parsed === null + || !hasExactObjectKeys(parsed, ["format", "identities", "packages"]) + || parsed.format !== "kandelo-program-packages-v2" + || typeof parsed.identities !== "object" + || parsed.identities === null + || Array.isArray(parsed.identities) + || typeof parsed.packages !== "object" + || parsed.packages === null + || Array.isArray(parsed.packages) + ) { + throw new Error(`invalid generated program package projection: ${indexPath}`); + } + for (const [packageName, identity] of Object.entries(parsed.identities)) { + const cacheKeys = identity?.cacheKeys; + if ( + !safeSinglePathComponent(packageName) + || typeof identity !== "object" + || identity === null + || !hasExactObjectKeys(identity, ["manifestSha256", "cacheKeys"]) + || typeof identity.manifestSha256 !== "string" + || !/^[a-f0-9]{64}$/.test(identity.manifestSha256) + || typeof cacheKeys !== "object" + || cacheKeys === null + || Array.isArray(cacheKeys) + || !hasExactObjectKeys(cacheKeys, ["wasm32", "wasm64"]) + || Object.values(cacheKeys).some( + (cacheKey) => + typeof cacheKey !== "string" || !/^[a-f0-9]{64}$/.test(cacheKey), + ) + ) { + throw new Error( + `invalid generated package identity for ${packageName}: ${indexPath}`, + ); + } + } + for (const [packageName, packageProjection] of Object.entries(parsed.packages)) { + const arches = packageProjection?.arches; + const cacheKeys = packageProjection?.cacheKeys; + const dependencyClosures = packageProjection?.dependencyClosures; + const members = packageProjection?.members; + if ( + !safeSinglePathComponent(packageName) + || typeof packageProjection !== "object" + || packageProjection === null + || !hasExactObjectKeys(packageProjection, [ + "manifestSha256", + "arches", + "cacheKeys", + "dependencyClosures", + "members", + ]) + || typeof packageProjection?.manifestSha256 !== "string" + || !/^[a-f0-9]{64}$/.test(packageProjection.manifestSha256) + || !Array.isArray(arches) + || arches.length === 0 + || new Set(arches).size !== arches.length + || arches.some((arch) => arch !== "wasm32" && arch !== "wasm64") + || typeof cacheKeys !== "object" + || cacheKeys === null + || Array.isArray(cacheKeys) + || !hasExactObjectKeys(cacheKeys, arches) + || Object.values(cacheKeys).some( + (cacheKey) => + typeof cacheKey !== "string" || !/^[a-f0-9]{64}$/.test(cacheKey), + ) + || typeof dependencyClosures !== "object" + || dependencyClosures === null + || Array.isArray(dependencyClosures) + || !hasExactObjectKeys(dependencyClosures, arches) + || !Array.isArray(members) + || members.length === 0 + ) { + throw new Error( + `invalid generated program package projection for ${packageName}: ${indexPath}`, + ); + } + for (const arch of arches) { + const closure = dependencyClosures[arch]; + // Match the host parser: order is non-semantic and uniqueness is + // required. Rust still emits one deterministic array so the checked-in + // projection is reproducible and freshness-checkable. + const seen = new Set(); + if ( + !Array.isArray(closure) + || closure.some((dependency) => { + if ( + typeof dependency !== "object" + || dependency === null + || !hasExactObjectKeys(dependency, [ + "packageName", + "manifestSha256", + "cacheKey", + ]) + || !safeSinglePathComponent(dependency.packageName) + || dependency.packageName === packageName + || seen.has(dependency.packageName) + || typeof dependency.manifestSha256 !== "string" + || !/^[a-f0-9]{64}$/.test(dependency.manifestSha256) + || typeof dependency.cacheKey !== "string" + || !/^[a-f0-9]{64}$/.test(dependency.cacheKey) + ) return true; + seen.add(dependency.packageName); + const contextualIdentity = parsed.identities[dependency.packageName]; + if ( + contextualIdentity === undefined + || contextualIdentity.manifestSha256 + !== dependency.manifestSha256 + || contextualIdentity.cacheKeys[arch] !== dependency.cacheKey + ) return true; + return false; + }) + ) { + throw new Error( + `invalid generated dependency closure for ${packageName} (${arch}): ${indexPath}`, + ); + } + } + for (const member of members) { + const outputKeys = [ + "kind", + "sourceArtifact", + "mirrorPath", + "outputName", + "forkInstrumentation", + ]; + const runtimeKeys = [ + "kind", + "sourceArtifact", + "mirrorPath", + "guestPath", + "mode", + ]; + if ( + typeof member !== "object" + || member === null + || ( + member.kind !== "output" + && member.kind !== "runtime-file" + ) + || !hasExactObjectKeys( + member, + member.kind === "output" ? outputKeys : runtimeKeys, + ) + || !portableArtifactPath(member.sourceArtifact) + || !portableArtifactPath(member.mirrorPath) + || ( + member.kind === "output" + && ( + !safeSinglePathComponent(member.outputName) + || ( + member.forkInstrumentation !== "auto" + && member.forkInstrumentation !== "disabled" + ) + ) + ) + || ( + member.kind === "runtime-file" + && ( + typeof member.guestPath !== "string" + || !member.guestPath.startsWith("/") + || !Number.isInteger(member.mode) + || member.mode < 0 + || member.mode > 0o777 + ) + ) + ) { + throw new Error( + `invalid generated program package projection member for ${packageName}: ${indexPath}`, + ); + } + } + if ( + new Set(members.map((member) => member.sourceArtifact)).size + !== members.length + || new Set(members.map((member) => member.mirrorPath)).size + !== members.length + || ( + members.length === 1 + && members[0].mirrorPath.includes("/") + ) + || ( + members.length > 1 + && members.some( + (member) => !member.mirrorPath.startsWith(`${packageName}/`), + ) + ) + ) { + throw new Error( + `invalid generated program package projection layout for ${packageName}: ${indexPath}`, + ); + } + const identity = parsed.identities[packageName]; + if ( + identity === undefined + || identity.manifestSha256 !== packageProjection.manifestSha256 + || arches.some( + (arch) => identity.cacheKeys[arch] !== packageProjection.cacheKeys[arch], + ) + ) { + throw new Error( + `generated program projection does not match package identity for ${packageName}: ${indexPath}`, + ); + } + } + return { + indexPath, + identities: parsed.identities, + packages: parsed.packages, + }; } -function parseProgramOutputs(text) { - return text - .split(/^\s*\[\[outputs\]\]\s*$/m) - .slice(1) - .map((block) => ({ - name: firstTomlString(block, "name"), - wasm: firstTomlString(block, "wasm"), - })) - .filter((output) => output.name && output.wasm); +function selectedRegistryPackages(repoRoot, registryPath) { + const selected = new Map(); + let authoritativeIdentities = null; + let authoritativeProjections = null; + let authoritativeProjectionPath = null; + for ( + const registryRoot of configuredProgramRegistryRoots(repoRoot, registryPath) + ) { + if (!existsSync(registryRoot)) continue; + if (!statSync(registryRoot).isDirectory()) { + throw new Error(`program registry root is not a directory: ${registryRoot}`); + } + const indexPath = join(registryRoot, "program-packages.json"); + if (!existsSync(indexPath)) { + throw new Error( + `program registry ${registryRoot} is missing program-packages.json`, + ); + } + const projection = readProgramPackageProjection(registryRoot); + authoritativeIdentities ??= projection.identities; + authoritativeProjections ??= projection.packages; + authoritativeProjectionPath ??= projection.indexPath; + const entries = readdirSync(registryRoot, { withFileTypes: true }) + .filter((entry) => entry.isDirectory() || entry.isSymbolicLink()) + .sort((left, right) => left.name.localeCompare(right.name)); + for (const entry of entries) { + const packageName = entry.name; + if (selected.has(packageName)) continue; + const packageDir = join(registryRoot, packageName); + const manifestPath = join(packageDir, "package.toml"); + if (!existsSync(manifestPath)) continue; + let manifestIsFile = false; + try { + manifestIsFile = statSync(manifestPath).isFile(); + } catch { + manifestIsFile = false; + } + if (!manifestIsFile) continue; + const packageProjection = + authoritativeProjections[packageName] ?? null; + const packageIdentity = + authoritativeIdentities[packageName] ?? null; + if (packageIdentity !== null) { + const manifestDigest = createHash("sha256") + .update(readFileSync(manifestPath)) + .digest("hex"); + if (manifestDigest !== packageIdentity.manifestSha256) { + throw new Error( + `stale generated package identity for ${packageName}: ${projection.indexPath}`, + ); + } + } + selected.set(packageName, { + packageDir, + manifestPath, + identity: packageIdentity, + projection: packageProjection, + projectionPath: authoritativeProjectionPath, + }); + } + } + return selected; } -function outputExtension(wasmPath) { - const basename = wasmPath.split("/").pop() ?? wasmPath; - const dot = basename.indexOf("."); - return dot === -1 ? "" : basename.slice(dot); +function validateSelectedProgramDependencyContext( + packageName, + selected, + selectedPackages, +) { + const packageProjection = selected.projection; + for (const arch of packageProjection.arches) { + if (!selected.identity) { + throw new Error( + `program package ${packageName} at ${selected.manifestPath} has no ` + + `authoritative contextual identity for ${arch}; regenerate ` + + "program-packages.json with the exact ordered registry roots", + ); + } + const selectedProgramCacheKey = selected.identity.cacheKeys[arch]; + if ( + selected.identity.manifestSha256 + !== packageProjection.manifestSha256 + || selectedProgramCacheKey !== packageProjection.cacheKeys[arch] + ) { + throw new Error( + `program package ${packageName} was projected with manifest ` + + `${packageProjection.manifestSha256} and cache key ` + + `${packageProjection.cacheKeys[arch]} for ${arch}, but the ` + + `authoritative first-hit registry context requires manifest ` + + `${selected.identity.manifestSha256} and cache key ` + + `${selectedProgramCacheKey ?? ""}. Regenerate this program ` + + "projection with the exact ordered registry roots; the highest-priority " + + "index must carry the complete combined-context projection rather than " + + "relying on a lower suffix-context build identity.", + ); + } + for (const expected of packageProjection.dependencyClosures[arch]) { + const dependency = selectedPackages.get(expected.packageName); + if (!dependency) { + throw new Error( + `program package ${packageName} was generated against dependency ` + + `${expected.packageName}, but that dependency is absent from the ` + + "configured first-hit registry roots", + ); + } + if (!dependency.identity) { + throw new Error( + `program package ${packageName} was generated against dependency ` + + `${expected.packageName}, but the first-hit package at ` + + `${dependency.manifestPath} has no contextual identity`, + ); + } + const selectedCacheKey = dependency.identity.cacheKeys[arch]; + if ( + dependency.identity.manifestSha256 !== expected.manifestSha256 + || selectedCacheKey !== expected.cacheKey + ) { + throw new Error( + `program package ${packageName} has a contextual cache identity ` + + `mismatch for ${arch}: its projection expects dependency ` + + `${expected.packageName} manifest ${expected.manifestSha256} and ` + + `cache key ${expected.cacheKey}, but first-hit selection at ` + + `${dependency.manifestPath} provides manifest ` + + `${dependency.identity.manifestSha256} and cache key ` + + `${selectedCacheKey ?? ""}. Regenerate the program ` + + "projection with the exact ordered registry roots; the complete " + + "highest-priority projection must bind every selected program to " + + "the same combined dependency context.", + ); + } + } + } } -export function packageOutputOwners(repoRoot = defaultRepoRoot) { +export function packageOutputOwners( + repoRoot = defaultRepoRoot, + { registryPath } = {}, +) { const owners = new Map(); - - for (const packageDir of registryPackageDirs(repoRoot)) { - const manifest = readFileSync(join(packageDir, "package.toml"), "utf8"); - if (firstTomlString(manifest, "kind") !== "program") continue; - - const packageName = firstTomlString(manifest, "name"); - if (!packageName) continue; - - const outputs = parseProgramOutputs(manifest); - if (outputs.length === 0) continue; - + const selectedPackages = selectedRegistryPackages(repoRoot, registryPath); + for ( + const [packageName, selected] of selectedPackages + ) { + const { packageDir, projection: packageProjection } = selected; + if (packageProjection === null) continue; + validateSelectedProgramDependencyContext( + packageName, + selected, + selectedPackages, + ); const hasBuildToml = existsSync(join(packageDir, "build.toml")); - for (const arch of parseArches(manifest)) { - for (const output of outputs) { - const dest = outputs.length > 1 - ? `${packageName}/${output.name}${outputExtension(output.wasm)}` - : `${output.name}${outputExtension(output.wasm)}`; - const rel = `programs/${arch}/${dest}`; - const previous = owners.get(rel); - if (previous && previous.packageName !== packageName) { - throw new Error( - `browser binary output ${rel} is owned by both ` + - `${previous.packageName} and ${packageName}`, - ); + for (const arch of packageProjection.arches) { + for (const member of packageProjection.members) { + const rel = `programs/${arch}/${member.mirrorPath}`; + for (const [previousRel, previous] of owners) { + if ( + previous.packageName !== packageName + && filePathsConflict(previousRel, rel) + ) { + throw new Error( + `browser binary outputs ${previousRel} and ${rel} conflict ` + + `between ${previous.packageName} and ${packageName}`, + ); + } } owners.set(rel, { packageName, hasBuildToml }); } @@ -115,39 +493,93 @@ function normalizeBinariesRel(rel) { return `programs/wasm32/${tail}`; } +function staticModuleSpecifiers(text, file) { + const ast = parse(text, { + sourceType: "unambiguous", + sourceFilename: file, + plugins: ["jsx", "typescript", "importAttributes"], + }); + const specifiers = []; + const pending = [ast.program]; + while (pending.length > 0) { + const node = pending.pop(); + if (!node || typeof node !== "object") continue; + if ( + ( + node.type === "ImportDeclaration" + || node.type === "ExportNamedDeclaration" + || node.type === "ExportAllDeclaration" + ) + && node.source?.type === "StringLiteral" + ) { + specifiers.push(node.source.value); + } else if ( + node.type === "CallExpression" + && node.callee?.type === "Import" + && node.arguments?.length === 1 + && node.arguments[0]?.type === "StringLiteral" + ) { + specifiers.push(node.arguments[0].value); + } else if ( + node.type === "ImportExpression" + && node.source?.type === "StringLiteral" + ) { + specifiers.push(node.source.value); + } + for (const value of Object.values(node)) { + if (Array.isArray(value)) { + for (const child of value) pending.push(child); + } else if (value && typeof value === "object") { + pending.push(value); + } + } + } + return specifiers; +} + export function browserBinariesImports(repoRoot = defaultRepoRoot) { const browserRoot = join(repoRoot, "apps", "browser-demos"); const imports = new Set(); - const patterns = [ - /\bfrom\s+["']@binaries\/([^"'?]+)(?:\?[^"']*)?["']/g, - /\bimport\(\s*["']@binaries\/([^"'?]+)(?:\?[^"']*)?["']\s*\)/g, - ]; for (const file of walkFiles(browserRoot)) { const text = readFileSync(file, "utf8"); - for (const pattern of patterns) { - for (const match of text.matchAll(pattern)) { - imports.add(normalizeBinariesRel(match[1])); - } + for (const specifier of staticModuleSpecifiers(text, file)) { + if (!specifier.startsWith("@binaries/")) continue; + const rel = specifier.slice("@binaries/".length).split("?", 1)[0]; + imports.add(normalizeBinariesRel(rel)); } } return [...imports].sort(); } -export function fetchableRegistryPackageNames(repoRoot = defaultRepoRoot) { +export function fetchableRegistryPackageNames( + repoRoot = defaultRepoRoot, + { registryPath } = {}, +) { const names = new Set(); - for (const packageDir of registryPackageDirs(repoRoot)) { - if (!existsSync(join(packageDir, "build.toml"))) continue; - const manifest = readFileSync(join(packageDir, "package.toml"), "utf8"); - const packageName = firstTomlString(manifest, "name"); - if (packageName) names.add(packageName); + const selectedPackages = selectedRegistryPackages(repoRoot, registryPath); + for (const [packageName, selected] of selectedPackages) { + if ( + selected.projection !== null + && existsSync(join(selected.packageDir, "build.toml")) + ) { + validateSelectedProgramDependencyContext( + packageName, + selected, + selectedPackages, + ); + names.add(packageName); + } } return names; } -export function inspectBrowserBinaryDependencies(repoRoot = defaultRepoRoot) { - const owners = packageOutputOwners(repoRoot); +export function inspectBrowserBinaryDependencies( + repoRoot = defaultRepoRoot, + { registryPath } = {}, +) { + const owners = packageOutputOwners(repoRoot, { registryPath }); const imports = browserBinariesImports(repoRoot); const missingOwners = []; const unfetchableOwners = []; @@ -176,9 +608,13 @@ export function inspectBrowserBinaryDependencies(repoRoot = defaultRepoRoot) { export function browserBinaryPackageRoots( repoRoot = defaultRepoRoot, - { includePackages = [], excludePackages = [] } = {}, + { + includePackages = [], + excludePackages = [], + registryPath, + } = {}, ) { - const audit = inspectBrowserBinaryDependencies(repoRoot); + const audit = inspectBrowserBinaryDependencies(repoRoot, { registryPath }); if (audit.missingOwners.length > 0) { throw new Error( `browser @binaries imports without registry owners:\n${audit.missingOwners.join("\n")}`, @@ -191,7 +627,7 @@ export function browserBinaryPackageRoots( ); } - const fetchable = fetchableRegistryPackageNames(repoRoot); + const fetchable = fetchableRegistryPackageNames(repoRoot, { registryPath }); const includes = new Set(includePackages); const excludes = new Set(excludePackages); for (const packageName of [...includes, ...excludes]) { diff --git a/scripts/build-resolve-binary-bundle.sh b/scripts/build-resolve-binary-bundle.sh new file mode 100755 index 0000000000..1101ed5ea1 --- /dev/null +++ b/scripts/build-resolve-binary-bundle.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "$0")/.." && pwd)" +output="${1:-$repo_root/scripts/resolve-binary.bundle.mjs}" + +mkdir -p "$(dirname "$output")" +cd "$repo_root" +npx --no-install esbuild scripts/resolve-binary.ts \ + --bundle \ + --platform=node \ + --format=esm \ + --target=node22 \ + --minify \ + --legal-comments=none \ + --banner:js='// Generated by scripts/build-resolve-binary-bundle.sh; do not edit. Third-party notices: resolve-binary.bundle.LICENSES.txt' \ + --outfile="$output" diff --git a/scripts/ci-check-pages-deployment.sh b/scripts/ci-check-pages-deployment.sh index 1af0ad7c7d..a8d387346b 100755 --- a/scripts/ci-check-pages-deployment.sh +++ b/scripts/ci-check-pages-deployment.sh @@ -90,6 +90,8 @@ for required_path in \ 'host/tsup.config.ts' \ 'package.json' \ 'package-lock.json' \ + 'packages/registry/**' \ + 'scripts/browser-binary-package-roots.mjs' \ 'scripts/check-pages-publish-size.mjs' \ 'scripts/check-pages-run-freshness.sh' \ 'scripts/ci-check-pages-deployment.sh' \ @@ -100,6 +102,8 @@ for required_path in \ fail "the complete Pages publisher does not watch $required_path" done +projection_line="$(step_line "Verify browser package projection is current")" +prepare_browser_line="$(step_line "Prepare browser demo assets")" guide_build_line="$(step_line "Build user guide for the complete Pages tree")" api_build_line="$(step_line "Build API docs for the complete Pages tree")" assembly_line="$(step_line "Add documentation to the complete Pages tree")" @@ -107,7 +111,9 @@ size_line="$(step_line "Enforce the GitHub Pages published-site size limit")" freshness_line="$(step_line "Confirm this is the newest Pages run")" deploy_line="$(step_line "Deploy to gh-pages")" -[ -n "$guide_build_line" ] && [ -n "$api_build_line" ] && +[ -n "$projection_line" ] && [ -n "$prepare_browser_line" ] && + [ "$projection_line" -lt "$prepare_browser_line" ] && + [ -n "$guide_build_line" ] && [ -n "$api_build_line" ] && [ -n "$assembly_line" ] && [ -n "$size_line" ] && [ -n "$freshness_line" ] && [ -n "$deploy_line" ] && [ "$guide_build_line" -lt "$assembly_line" ] && @@ -117,6 +123,14 @@ deploy_line="$(step_line "Deploy to gh-pages")" [ "$freshness_line" -lt "$deploy_line" ] || fail "one job must assemble and size-check the complete tree before its freshness check and deployment" +projection_block="$( + step_block "$PAGES_WORKFLOW" "Verify browser package projection is current" +)" +grep -Fq 'build-deps program-index-check' <<<"$projection_block" && + grep -Fq 'packages/registry packages/registry/program-packages.json' \ + <<<"$projection_block" || + fail "the Pages publisher must verify the generated package projection before preparing assets" + between_freshness_and_deploy="$( sed -n "${freshness_line},${deploy_line}p" "$PAGES_WORKFLOW" | awk '/^ - name:/ { count += 1 } END { print count + 0 }' diff --git a/scripts/install-local-binary.sh b/scripts/install-local-binary.sh index 8937d77c55..f8bcbc0a6e 100755 --- a/scripts/install-local-binary.sh +++ b/scripts/install-local-binary.sh @@ -1,66 +1,372 @@ #!/usr/bin/env bash # -# install-local-binary.sh — copy a freshly-built wasm into -# local-binaries/ so the resolver picks it up as an override over -# anything `scripts/fetch-binaries.sh` downloaded. +# Install freshly-built package artifacts into local-binaries/. Normal local +# installs are manifest-driven and fail closed: one Rust metadata lookup +# selects the exact output path and fork policy before source instrumentation +# or destination mutation. Sealed package builds may avoid Cargo only by +# disabling the local mirror and explicitly declaring the fork policy. # -# Sourced or called from each ported program's build script after -# producing its output binary. The resolver (host/src/binary-resolver.ts -# + scripts/resolve-binary.sh) prefers local-binaries/ over binaries/, -# so running any program's local build automatically shadows the -# released version. +# Usage: +# source scripts/install-local-binary.sh +# install_local_binary [declared-output-artifact] +# install_local_runtime_file [declared-runtime-artifact] # -# Path discovery: the destination relative path under -# `local-binaries/programs//` is read from the package's -# `package.toml` via `xtask build-deps output-path `. -# This is the SAME path the resolver writes to from a published -# archive — keeping local builds and releases interchangeable at the -# resolver layer (single-output is flat `.`, -# multi-output nests under `/`). Without this lookup, a -# package whose `program.name != output.name` (e.g. texlive/pdftex) had -# divergent local-vs-release paths and the demo could never see a -# fresh local build. -# -# Usage (each call is one install target): -# source scripts/install-local-binary.sh # adds install_local_binary() -# -# install_local_binary -# -# Where: -# logical program name matching a package.toml `name` field -# in the registry (e.g. "dash", "git", "texlive"). -# path to the freshly-built file. Its basename must -# match one of the `[[outputs]].wasm` filenames declared -# in the package's package.toml. -# -# Legacy 3-arg form `install_local_binary ` -# is silently accepted: the third arg is ignored when the package.toml -# lookup succeeds (the lookup is the source of truth) and falls -# through to the legacy multi-binary subdir layout otherwise. Treat -# the 2-arg form as canonical for new build scripts. -# -# Arch is taken from $WASM_POSIX_DEP_TARGET_ARCH (set by the resolver -# while running build scripts) and falls back to "wasm32" for direct -# build-script invocations like `bash packages/registry/dash/build-dash.sh`. -# Sealed callers that only consume $WASM_POSIX_DEP_OUT_DIR can set -# WASM_POSIX_INSTALL_LOCAL_MIRROR=0 to retain all writes in caller-owned -# scratch space while preserving the normal artifact guards below. +# The optional artifact is the exact `[[outputs]].wasm` or +# `[[runtime_files]].artifact` path. A source basename is accepted only when it +# identifies one output unambiguously. source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/wasm-artifact-guards.sh" -install_local_binary() { - local program="$1" - local src="$2" - local legacy_dest_name="${3:-}" +if [ -z "${WASM_POSIX_LOCAL_INSTALL_SESSION:-}" ]; then + WASM_POSIX_LOCAL_INSTALL_SESSION="shell-${BASHPID:-$$}-${RANDOM:-0}-${RANDOM:-0}" +fi + +_wasm_posix_require_portable_relative_path() { + case "$1" in + ""|/*|*\\*|.|..|./*|../*|*/.|*/..|*//*|*/./*|*/../*|*/) + return 1 + ;; + esac +} + +_wasm_posix_directory_identity() { + local path="$1" + local identity + if [ ! -d "$path" ] || [ -L "$path" ]; then + return 1 + fi + if identity="$(stat -c '%d:%i:%u:%g:%a' "$path" 2>/dev/null)"; then + printf '%s\n' "$identity" + return + fi + identity="$(stat -f '%d:%i:%u:%g:%Lp' "$path" 2>/dev/null)" || return + printf '%s\n' "$identity" +} + +_wasm_posix_sha256_file() { + local path="$1" + local output + if command -v sha256sum >/dev/null 2>&1; then + output="$(sha256sum "$path")" || return + elif command -v shasum >/dev/null 2>&1; then + output="$(shasum -a 256 "$path")" || return + elif command -v openssl >/dev/null 2>&1; then + output="$(openssl dgst -sha256 "$path")" || return + printf '%s\n' "${output##* }" + return + else + echo "install-local-binary: no SHA-256 implementation is available" >&2 + return 1 + fi + printf '%s\n' "${output%% *}" +} + +# The filesystem identity plus exact bytes of one regular file. Cleanup uses +# this state to avoid deleting a path that another writer replaced or changed. +_wasm_posix_regular_file_state() { + local path="$1" + local identity digest + if [ ! -f "$path" ] || [ -L "$path" ]; then + return 1 + fi + if ! identity="$(stat -c '%d:%i:%u:%g:%a:%s' "$path" 2>/dev/null)"; then + identity="$(stat -f '%d:%i:%u:%g:%Lp:%z' "$path" 2>/dev/null)" || return + fi + digest="$(_wasm_posix_sha256_file "$path")" || return + printf '%s:%s\n' "$identity" "$digest" +} + +# Shell path operations cannot provide openat-style race freedom. The sealed +# install contract therefore requires a caller-owned, single-writer scratch +# root. Enforce the observable part of that contract: this user owns every +# directory we traverse and no group/other writer can replace its entries. +_wasm_posix_require_single_writer_directory() { + local path="$1" + local label="$2" + local identity="${3:-}" + if [ -z "$identity" ]; then + identity="$(_wasm_posix_directory_identity "$path")" || { + echo "install-local-binary: $label must be a real directory: $path" >&2 + return 1 + } + fi + local device inode owner group mode + IFS=: read -r device inode owner group mode <<<"$identity" + if [ "$owner" != "$(id -u)" ]; then + echo "install-local-binary: $label must be owned by the current user: $path" >&2 + return 1 + fi + if ! [[ "$mode" =~ ^[0-7]{3,4}$ ]] || (( (8#$mode & 8#022) != 0 )); then + echo "install-local-binary: $label must not be writable by group or other users: $path" >&2 + return 1 + fi +} + +_wasm_posix_require_unchanged_directory() { + local path="$1" + local expected="$2" + local label="$3" + local actual + actual="$(_wasm_posix_directory_identity "$path")" || { + echo "install-local-binary: $label changed or disappeared; preserving transaction state: $path" >&2 + return 1 + } + if [ "$actual" != "$expected" ]; then + echo "install-local-binary: $label changed filesystem identity; preserving transaction state: $path" >&2 + return 1 + fi +} + +_wasm_posix_remove_unchanged_transaction_file() { + local transaction="$1" + local transaction_identity="$2" + local path="$3" + local expected="$4" + local label="$5" + local actual + _wasm_posix_require_unchanged_directory \ + "$transaction" "$transaction_identity" "private transaction directory" || return + actual="$(_wasm_posix_regular_file_state "$path")" || { + echo "install-local-binary: $label changed type or disappeared; refusing cleanup: $path" >&2 + return 1 + } + if [ "$actual" != "$expected" ]; then + echo "install-local-binary: $label changed identity or contents; refusing cleanup: $path" >&2 + return 1 + fi + rm -f "$path" || return + if [ -e "$path" ] || [ -L "$path" ]; then + echo "install-local-binary: $label remained after cleanup: $path" >&2 + return 1 + fi +} + +# Copy below one authorized caller-owned scratch directory. Every +# created/existing descendant directory is checked without following symlinks, +# and publication uses a private transaction plus create-once hard link. The +# caller must keep the root single-writer for the duration of this function; +# general shared-directory race freedom requires dirfd/openat operations and is +# intentionally not claimed by this packaging-only shell path. +_wasm_posix_copy_file_no_follow() { + local src="$1" + local authorized_root="$2" + local relative_dest="$3" + + if [ ! -f "$src" ] || [ -L "$src" ]; then + echo "install-local-binary: source must be a regular non-symlink file: $src" >&2 + return 1 + fi + if [ ! -d "$authorized_root" ] || [ -L "$authorized_root" ]; then + echo "install-local-binary: authorized destination root must be a real directory: $authorized_root" >&2 + return 1 + fi + if ! _wasm_posix_require_portable_relative_path "$relative_dest"; then + echo "install-local-binary: destination must be a normalized portable relative path: $relative_dest" >&2 + return 1 + fi + + local root + root="$(cd "$authorized_root" && pwd -P)" || return 1 + local root_identity + root_identity="$(_wasm_posix_directory_identity "$root")" || return 1 + _wasm_posix_require_single_writer_directory \ + "$root" "authorized destination root" "$root_identity" || return + local parent_relative + parent_relative="$(dirname "$relative_dest")" + local parent="$root" + if [ "$parent_relative" != "." ]; then + local remainder="$parent_relative" + while [ -n "$remainder" ]; do + local component="${remainder%%/*}" + if [ "$component" = "$remainder" ]; then + remainder="" + else + remainder="${remainder#*/}" + fi + local next="$parent/$component" + if [ -L "$next" ]; then + echo "install-local-binary: refusing destination symlink ancestor: $next" >&2 + return 1 + fi + if [ -e "$next" ]; then + if [ ! -d "$next" ]; then + echo "install-local-binary: destination ancestor is not a directory: $next" >&2 + return 1 + fi + elif ! mkdir -m 755 "$next"; then + return 1 + fi + _wasm_posix_require_single_writer_directory \ + "$next" "destination ancestor" || return + parent="$next" + done + fi + + local parent_identity + parent_identity="$(_wasm_posix_directory_identity "$parent")" || return 1 + local source_state + source_state="$(_wasm_posix_regular_file_state "$src")" || { + echo "install-local-binary: could not capture source identity and contents: $src" >&2 + return 1 + } + + local dest="$root/$relative_dest" + local transaction + transaction="$(mktemp -d "$root/.kandelo-install.XXXXXX")" || return 1 + chmod 700 "$transaction" || { + echo "install-local-binary: could not protect transaction directory: $transaction" >&2 + return 1 + } + local transaction_identity + transaction_identity="$(_wasm_posix_directory_identity "$transaction")" || return 1 + _wasm_posix_require_single_writer_directory \ + "$transaction" "private transaction directory" "$transaction_identity" || return + local stage="$transaction/stage" + local backup="$transaction/backup" + + if ! cp -p "$src" "$stage"; then + echo "install-local-binary: source copy failed; preserving transaction state: $transaction" >&2 + return 1 + fi + local source_after stage_state + source_after="$(_wasm_posix_regular_file_state "$src")" || return 1 + stage_state="$(_wasm_posix_regular_file_state "$stage")" || return 1 + if [ "$source_after" != "$source_state" ] || \ + [ "${stage_state##*:}" != "${source_state##*:}" ]; then + echo "install-local-binary: source changed during copy; preserving transaction state: $transaction" >&2 + return 1 + fi + _wasm_posix_require_unchanged_directory \ + "$root" "$root_identity" "authorized destination root" || return + _wasm_posix_require_unchanged_directory \ + "$parent" "$parent_identity" "destination parent" || return + _wasm_posix_require_unchanged_directory \ + "$transaction" "$transaction_identity" "private transaction directory" || return + + local old_moved=0 + local old_state="" + if [ -e "$dest" ] || [ -L "$dest" ]; then + if [ -d "$dest" ] && [ ! -L "$dest" ]; then + echo "install-local-binary: refusing to replace directory: $dest" >&2 + return 1 + fi + if [ -L "$dest" ] || [ ! -f "$dest" ]; then + echo "install-local-binary: refusing to replace a non-regular destination: $dest" >&2 + return 1 + fi + old_state="$(_wasm_posix_regular_file_state "$dest")" || return 1 + if ! mv "$dest" "$backup"; then + return 1 + fi + old_moved=1 + local quarantined_state="" + quarantined_state="$(_wasm_posix_regular_file_state "$backup")" || true + if [ "$quarantined_state" != "$old_state" ]; then + echo "install-local-binary: destination changed during quarantine; refusing publication" >&2 + if [ ! -e "$dest" ] && [ ! -L "$dest" ]; then + # Preserve the entry that won the pathname race by returning + # it to the live name. Never delete an unrecognized backup. + mv "$backup" "$dest" || { + echo "install-local-binary: could not restore changed quarantine; preserved it at $backup" >&2 + return 1 + } + fi + return 1 + fi + fi + + if ! ln "$stage" "$dest"; then + if [ "$old_moved" = "1" ] && [ ! -e "$dest" ] && [ ! -L "$dest" ]; then + local rollback_state + rollback_state="$(_wasm_posix_regular_file_state "$backup")" || { + echo "install-local-binary: refusing to restore changed quarantine: $backup" >&2 + return 1 + } + if [ "$rollback_state" != "$old_state" ]; then + echo "install-local-binary: refusing to restore changed quarantine: $backup" >&2 + return 1 + fi + if ! mv "$backup" "$dest" || \ + [ "$(_wasm_posix_regular_file_state "$dest" || true)" != "$old_state" ]; then + echo "install-local-binary: failed to restore the previous destination: $dest" >&2 + return 1 + fi + fi + echo "install-local-binary: publication failed; preserving transaction state: $transaction" >&2 + return 1 + fi + + local published_stage_state published_dest_state + published_stage_state="$(_wasm_posix_regular_file_state "$stage")" || return 1 + published_dest_state="$(_wasm_posix_regular_file_state "$dest")" || return 1 + if [ "$published_stage_state" != "$published_dest_state" ] || \ + [ "${published_dest_state##*:}" != "${source_state##*:}" ]; then + echo "install-local-binary: published destination changed identity or contents; preserving transaction state: $transaction" >&2 + return 1 + fi + + if [ "$old_moved" = "1" ]; then + local final_backup_state + final_backup_state="$(_wasm_posix_regular_file_state "$backup")" || { + echo "install-local-binary: quarantined destination changed type or disappeared; preserving transaction state: $transaction" >&2 + return 1 + } + if [ "$final_backup_state" != "$old_state" ]; then + echo "install-local-binary: quarantined destination changed identity or contents; preserving transaction state: $transaction" >&2 + return 1 + fi + fi + _wasm_posix_remove_unchanged_transaction_file \ + "$transaction" "$transaction_identity" "$stage" \ + "$published_stage_state" "staged artifact" || return + if [ "$(_wasm_posix_regular_file_state "$dest" || true)" != "$published_dest_state" ]; then + echo "install-local-binary: published destination changed during staged-link cleanup: $dest" >&2 + return 1 + fi + if [ "$old_moved" = "1" ]; then + _wasm_posix_remove_unchanged_transaction_file \ + "$transaction" "$transaction_identity" "$backup" \ + "$old_state" "quarantined destination" || return + fi + if [ "$(_wasm_posix_regular_file_state "$dest" || true)" != "$published_dest_state" ]; then + echo "install-local-binary: published destination changed during transaction cleanup: $dest" >&2 + return 1 + fi + _wasm_posix_require_unchanged_directory \ + "$transaction" "$transaction_identity" "private transaction directory" || return + rmdir "$transaction" || { + echo "install-local-binary: private transaction is not empty; refusing recursive cleanup: $transaction" >&2 + return 1 + } +} + +_wasm_posix_output_metadata() { + local repo_root="$1" + local host_target="$2" + local package="$3" + local artifact="$4" + ( + cd "$repo_root" + env -u CC -u CXX -u AR -u RANLIB -u CFLAGS -u CXXFLAGS -u CPPFLAGS -u LDFLAGS \ + cargo run -p xtask --target "$host_target" --quiet -- \ + build-deps output-metadata "$package" "$artifact" + ) +} + +install_local_binary() { + local program="${1:-}" + local src="${2:-}" + local requested_artifact="${3:-}" if [ -z "$program" ] || [ -z "$src" ]; then - echo "install_local_binary: usage: install_local_binary " >&2 + echo "install_local_binary: usage: install_local_binary [declared-output-artifact]" >&2 return 2 fi - if [ ! -f "$src" ]; then - echo "install_local_binary: source file not found: $src" >&2 + if [ ! -f "$src" ] || [ -L "$src" ]; then + echo "install_local_binary: source must be a regular non-symlink file: $src" >&2 return 1 fi + local arch="${WASM_POSIX_DEP_TARGET_ARCH:-wasm32}" case "$arch" in wasm32|wasm64) ;; @@ -70,13 +376,16 @@ install_local_binary() { ;; esac - # Repo root must be derived from this helper, not from the caller's - # current directory: package builds often `cd` into an upstream git - # checkout before installing artifacts. local repo_root repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" local src_basename src_basename="$(basename "$src")" + requested_artifact="${requested_artifact:-$src_basename}" + if ! _wasm_posix_require_portable_relative_path "$requested_artifact"; then + echo "install_local_binary: declared artifact must be a normalized portable relative path: $requested_artifact" >&2 + return 2 + fi + local install_local_mirror="${WASM_POSIX_INSTALL_LOCAL_MIRROR:-1}" case "$install_local_mirror" in 0) @@ -92,22 +401,67 @@ install_local_binary() { ;; esac - if ! wasm_require_no_legacy_asyncify "$src"; then - return 1 - fi local host_target="" + local mirror_path="" + local declared_artifact="$requested_artifact" + local declared_policy="" if [ "$install_local_mirror" = "1" ]; then host_target="$(rustc -vV 2>/dev/null | awk '/^host/ {print $2}')" + if [ -z "$host_target" ]; then + echo "install_local_binary: rustc did not report a host target" >&2 + return 1 + fi + + # Resolve destination and policy together before any operation below + # can instrument the source or mutate a destination. + local metadata + if ! metadata="$(_wasm_posix_output_metadata \ + "$repo_root" "$host_target" "$program" "$requested_artifact")"; then + echo "install_local_binary: package '$program' does not uniquely declare output '$requested_artifact'" >&2 + return 1 + fi + local fields + if ! fields="$(node -e ' + const value = JSON.parse(process.argv[1]); + for (const key of ["mirror_path", "fork_instrumentation", "source_artifact"]) { + if (typeof value[key] !== "string" || value[key].length === 0 || + value[key].includes("\t") || value[key].includes("\n")) { + throw new Error(`invalid output metadata field ${key}`); + } + } + process.stdout.write( + `${value.mirror_path}\t${value.fork_instrumentation}\t${value.source_artifact}`, + ); + ' "$metadata")"; then + echo "install_local_binary: invalid output metadata for '$program:$requested_artifact'" >&2 + return 1 + fi + IFS=$'\t' read -r mirror_path declared_policy declared_artifact <<<"$fields" + if ! _wasm_posix_require_portable_relative_path "$mirror_path" \ + || ! _wasm_posix_require_portable_relative_path "$declared_artifact"; then + echo "install_local_binary: xtask returned an unsafe output path" >&2 + return 1 + fi + else + declared_policy="${WASM_POSIX_INSTALL_FORK_INSTRUMENTATION:-}" + if [ -z "$declared_policy" ]; then + echo "install_local_binary: sealed installs require explicit WASM_POSIX_INSTALL_FORK_INSTRUMENTATION=auto|disabled" >&2 + return 2 + fi fi - local fork_instrumentation="${WASM_POSIX_INSTALL_FORK_INSTRUMENTATION:-}" - if [ -z "$fork_instrumentation" ] && [ -n "$host_target" ]; then - fork_instrumentation="$(cd "$repo_root" && \ - env -u CC -u CXX -u AR -u RANLIB -u CFLAGS -u CXXFLAGS -u CPPFLAGS -u LDFLAGS \ - cargo run -p xtask --target "$host_target" --quiet -- \ - build-deps output-fork-instrumentation "$program" "$src_basename" 2>/dev/null || true)" + + local requested_policy="${WASM_POSIX_INSTALL_FORK_INSTRUMENTATION:-}" + if [ "$install_local_mirror" = "1" ] \ + && [ -n "$requested_policy" ] \ + && [ "$requested_policy" != "$declared_policy" ]; then + echo "install_local_binary: requested fork policy '$requested_policy' disagrees with package policy '$declared_policy'" >&2 + return 1 fi - fork_instrumentation="${fork_instrumentation:-auto}" - case "$fork_instrumentation" in + + if ! wasm_require_no_legacy_asyncify "$src"; then + return 1 + fi + case "$declared_policy" in auto) if wasm_imports_kernel_fork "$src" && ! wasm_has_complete_fork_instrumentation "$src"; then if wasm_has_any_wpk_fork_export "$src"; then @@ -123,105 +477,58 @@ install_local_binary() { fi mv "$instrumented" "$src" fi - if ! wasm_require_fork_instrumentation_if_needed "$src"; then - return 1 - fi + wasm_require_fork_instrumentation_if_needed "$src" || return 1 ;; disabled) - if ! wasm_require_no_fork_instrumentation "$src"; then - return 1 - fi + wasm_require_no_fork_instrumentation "$src" || return 1 ;; *) - echo "install_local_binary: unsupported WASM_POSIX_INSTALL_FORK_INSTRUMENTATION='$fork_instrumentation' (expected auto or disabled)" >&2 + echo "install_local_binary: unsupported fork policy '$declared_policy' (expected auto or disabled)" >&2 return 2 ;; esac if [ "$install_local_mirror" = "1" ]; then - # Take everything from the FIRST dot in the source basename onward - # so compound extensions like `.vfs.zst` round-trip intact (matches - # the resolver's `place_binaries_symlinks` extension handling). - local src_ext="" - case "$src_basename" in - *.*) src_ext=".${src_basename#*.}" ;; - esac - - # Ask xtask for the package.toml-driven destination relative path. - # On hit, that's the canonical location matching the resolver's - # symlink layout (tools/xtask/src/build_deps.rs `place_binaries_symlinks`). - # On miss (package not in the registry, e.g. the dash→sh alias - # call site, or no [[outputs]] entry for this basename) fall back - # to the legacy heuristic so existing build scripts keep working. - local rel="" - if [ -n "$host_target" ]; then - rel="$(cd "$repo_root" && \ - env -u CC -u CXX -u AR -u RANLIB -u CFLAGS -u CXXFLAGS -u CPPFLAGS -u LDFLAGS \ + local source_parent + source_parent="$(cd "$(dirname "$src")" && pwd -P)" || return 1 + local source_abs="$source_parent/$src_basename" + if ! ( + cd "$repo_root" + env -u CC -u CXX -u AR -u RANLIB -u CFLAGS -u CXXFLAGS -u CPPFLAGS -u LDFLAGS \ + WASM_POSIX_LOCAL_INSTALL_SOURCE="$source_abs" \ + WASM_POSIX_LOCAL_INSTALL_SESSION="$WASM_POSIX_LOCAL_INSTALL_SESSION" \ cargo run -p xtask --target "$host_target" --quiet -- \ - build-deps output-path "$program" "$src_basename" 2>/dev/null || true)" - fi - - local dest - if [ -n "$rel" ]; then - dest="$repo_root/local-binaries/programs/$arch/$rel" - elif [ -n "$legacy_dest_name" ]; then - # Legacy multi-binary subdir layout. Used to be the only way to - # express "this program produces multiple wasms"; package.toml's - # [[outputs]] now does that explicitly. Reachable today only - # for callers whose program name isn't in the registry. - dest="$repo_root/local-binaries/programs/$arch/$program/$legacy_dest_name" - else - # Legacy single-binary fallback. Used by aliasing call sites - # like `install_local_binary sh "$BIN_DIR/dash.wasm"` where - # the "program" is a name registered nowhere. Uses the full - # compound extension so `.vfs.zst` round-trips intact. - dest="$repo_root/local-binaries/programs/$arch/$program$src_ext" + build-deps --arch "$arch" \ + --binaries-dir "$repo_root/local-binaries" \ + install-local-artifact "$program" "$declared_artifact" + ); then + return 1 fi + fi - mkdir -p "$(dirname "$dest")" - cp "$src" "$dest" - echo " installed $dest" - fi - - # When invoked under the package-system resolver (`xtask build-deps - # resolve`, `xtask archive-stage`), WASM_POSIX_DEP_OUT_DIR points at - # the resolver's scratch dir. The build script must install its - # declared `[[outputs]].wasm` files there so `validate_outputs` - # finds them and `archive_stage` packs them into the release - # archive — and `validate_outputs` looks them up by EXACT - # `[[outputs]].wasm` filename (tools/xtask/src/build_deps.rs:1136). - # - # The src filename (build script's own output) is what the build - # script declared via `[[outputs]].wasm`, so basename(src) is - # always the right key. No translation needed. - # - # Outside the resolver, WASM_POSIX_DEP_OUT_DIR is unset and this - # path is a no-op. if [ -n "${WASM_POSIX_DEP_OUT_DIR:-}" ]; then - local resolver_dest="$WASM_POSIX_DEP_OUT_DIR/$src_basename" - mkdir -p "$(dirname "$resolver_dest")" - cp "$src" "$resolver_dest" - echo " installed $resolver_dest (resolver scratch)" + if ! _wasm_posix_copy_file_no_follow \ + "$src" "$WASM_POSIX_DEP_OUT_DIR" "$declared_artifact"; then + echo "install_local_binary: failed to publish resolver scratch artifact: $WASM_POSIX_DEP_OUT_DIR/$declared_artifact" >&2 + return 1 + fi + echo " installed $WASM_POSIX_DEP_OUT_DIR/$declared_artifact (resolver scratch)" fi } -# Install a declared non-Wasm `[[runtime_files]]` artifact into the same local -# and caller-owned resolver destinations used by executable outputs. This is -# intentionally separate from install_local_binary: data files must not pass -# Wasm/fork guards or be described as executable outputs. install_local_runtime_file() { - local program="$1" - local src="$2" + local program="${1:-}" + local src="${2:-}" local artifact="${3:-}" - if [ -z "$program" ] || [ -z "$src" ]; then - echo "install_local_runtime_file: usage: install_local_runtime_file [artifact]" >&2 + echo "install_local_runtime_file: usage: install_local_runtime_file [declared-runtime-artifact]" >&2 return 2 fi if [ ! -f "$src" ] || [ -L "$src" ]; then echo "install_local_runtime_file: source must be a regular non-symlink file: $src" >&2 return 1 fi + local arch="${WASM_POSIX_DEP_TARGET_ARCH:-wasm32}" case "$arch" in wasm32|wasm64) ;; @@ -230,18 +537,15 @@ install_local_runtime_file() { return 2 ;; esac - local repo_root repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" local src_basename src_basename="$(basename "$src")" artifact="${artifact:-$src_basename}" - case "$artifact" in - ""|/*|*\\*|.|..|./*|../*|*/.|*/..|*//*|*/./*|*/../*|*/) - echo "install_local_runtime_file: artifact must be a portable relative path: $artifact" >&2 - return 2 - ;; - esac + if ! _wasm_posix_require_portable_relative_path "$artifact"; then + echo "install_local_runtime_file: artifact must be a portable relative path with normalized components: $artifact" >&2 + return 2 + fi local install_local_mirror="${WASM_POSIX_INSTALL_LOCAL_MIRROR:-1}" case "$install_local_mirror" in @@ -250,10 +554,12 @@ install_local_runtime_file() { echo "install_local_runtime_file: WASM_POSIX_INSTALL_LOCAL_MIRROR=0 requires WASM_POSIX_DEP_OUT_DIR" >&2 return 2 fi - local resolver_dest="$WASM_POSIX_DEP_OUT_DIR/$artifact" - mkdir -p "$(dirname "$resolver_dest")" - cp "$src" "$resolver_dest" - echo " installed $resolver_dest (resolver scratch)" + if ! _wasm_posix_copy_file_no_follow \ + "$src" "$WASM_POSIX_DEP_OUT_DIR" "$artifact"; then + echo "install_local_runtime_file: failed to publish resolver scratch artifact: $WASM_POSIX_DEP_OUT_DIR/$artifact" >&2 + return 1 + fi + echo " installed $WASM_POSIX_DEP_OUT_DIR/$artifact (resolver scratch)" return 0 ;; 1) ;; @@ -269,18 +575,17 @@ install_local_runtime_file() { echo "install_local_runtime_file: rustc did not report a host target" >&2 return 1 fi - local rel - rel="$(cd "$repo_root" && \ + local source_parent + source_parent="$(cd "$(dirname "$src")" && pwd -P)" || return 1 + local source_abs="$source_parent/$src_basename" + ( + cd "$repo_root" env -u CC -u CXX -u AR -u RANLIB -u CFLAGS -u CXXFLAGS -u CPPFLAGS -u LDFLAGS \ - cargo run -p xtask --target "$host_target" --quiet -- \ - build-deps runtime-file-path "$program" "$artifact")" || return 1 - if [ -z "$rel" ]; then - echo "install_local_runtime_file: manifest lookup returned an empty path" >&2 - return 1 - fi - - local dest="$repo_root/local-binaries/programs/$arch/$rel" - mkdir -p "$(dirname "$dest")" - cp "$src" "$dest" - echo " installed $dest" + WASM_POSIX_LOCAL_INSTALL_SOURCE="$source_abs" \ + WASM_POSIX_LOCAL_INSTALL_SESSION="$WASM_POSIX_LOCAL_INSTALL_SESSION" \ + cargo run -p xtask --target "$host_target" --quiet -- \ + build-deps --arch "$arch" \ + --binaries-dir "$repo_root/local-binaries" \ + install-local-artifact "$program" "$artifact" + ) } diff --git a/scripts/prepare-host-package.sh b/scripts/prepare-host-package.sh index 2232a048c3..97fb967ff3 100755 --- a/scripts/prepare-host-package.sh +++ b/scripts/prepare-host-package.sh @@ -5,6 +5,18 @@ REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" HOST_WASM_DIR="$REPO_ROOT/host/wasm" mkdir -p "$HOST_WASM_DIR" +# Keep the standalone npm package on the same Rust-generated program closure +# and artifact policy as source checkouts. The package has no registry TOML to +# inspect at runtime. Refuse to copy stale policy into a publishable package. +HOST_TARGET="$(rustc -vV | awk '/^host/ {print $2}')" +cargo run -p xtask --target "$HOST_TARGET" --quiet -- \ + build-deps program-index-check \ + "$REPO_ROOT/packages/registry" \ + "$REPO_ROOT/packages/registry/program-packages.json" +cp \ + "$REPO_ROOT/packages/registry/program-packages.json" \ + "$HOST_WASM_DIR/program-packages.json" + copy_first_existing() { local dest="$1" shift diff --git a/scripts/publish-package-source.sh b/scripts/publish-package-source.sh index e89c8613f4..9fb37b4caf 100755 --- a/scripts/publish-package-source.sh +++ b/scripts/publish-package-source.sh @@ -58,13 +58,17 @@ PACKAGE_LIST="${PACKAGE_LIST:-$PACKAGE_SOURCE_ROOT/packages.txt}" cd "$KANDELO_ROOT" source "$KANDELO_ROOT/sdk/activate.sh" +HOST_TARGET="$(rustc -vV | awk '/^host/ {print $2}')" +export WASM_POSIX_DEPS_REGISTRY="$PACKAGE_SOURCE_ROOT/packages:$KANDELO_ROOT/packages/registry" +cargo run -p xtask --target "$HOST_TARGET" --quiet -- \ + build-deps program-index-context-check + "$KANDELO_ROOT/scripts/sync-package-source.sh" \ --package-source-root "$PACKAGE_SOURCE_ROOT" \ --kandelo-root "$KANDELO_ROOT" ABI="$(grep -oE 'ABI_VERSION: u32 = [0-9]+' crates/shared/src/lib.rs | awk '{print $4}')" TARGET_TAG="${TARGET_TAG:-binaries-abi-v${ABI}}" -HOST_TARGET="$(rustc -vV | awk '/^host/ {print $2}')" BUILD_TIMESTAMP="$(git -C "$PACKAGE_SOURCE_ROOT" log -1 --format=%aI HEAD 2>/dev/null || date -u +%FT%TZ)" BUILD_COMMIT="$(git -C "$PACKAGE_SOURCE_ROOT" rev-parse HEAD 2>/dev/null || echo local)" BUILD_HOST="${REPOSITORY}@${BUILD_COMMIT}" diff --git a/scripts/resolve-binary.bundle.LICENSES.txt b/scripts/resolve-binary.bundle.LICENSES.txt new file mode 100644 index 0000000000..efcfe92170 --- /dev/null +++ b/scripts/resolve-binary.bundle.LICENSES.txt @@ -0,0 +1,47 @@ +The standalone resolver bundle contains these third-party components: + +fflate 0.8.3 +MIT License + +Copyright (c) 2026 Arjun Barrett + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +fzstd 0.1.1 +MIT License + +Copyright (c) 2020 Arjun Barrett + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/scripts/resolve-binary.bundle.mjs b/scripts/resolve-binary.bundle.mjs new file mode 100644 index 0000000000..962aacbb80 --- /dev/null +++ b/scripts/resolve-binary.bundle.mjs @@ -0,0 +1,13 @@ +// Generated by scripts/build-resolve-binary-bundle.sh; do not edit. Third-party notices: resolve-binary.bundle.LICENSES.txt +var Zi=Object.defineProperty;var pn=(r,e,t)=>()=>{if(t)throw t[0];try{return r&&(e=r(r=0)),e}catch(n){throw t=[n],n}};var mr=(r,e)=>{for(var t in e)Zi(r,t,{get:e[t],enumerable:!0})};import{createRequire as Ds}from"module";function Qr(r,e){return Jr(r,{i:2},e&&e.out,e&&e.dictionary)}var Us,ot,Gs,Ks,J,it,Zs,Wr,Hr,Ws,Vr,ot,qr,Hs,jr,Vs,Ga,$n,ze,M,Lt,At,M,M,M,M,Xr,M,qs,js,Nn,ye,Mn,Yr,jt,Xs,le,Jr,Ys,Js,st,ei,Qs,eo,Fn=pn(()=>{Us=Ds("/");try{ot=Us("worker_threads"),Gs=ot.Worker,Ks=ot.isMarkedAsUntransferable}catch{}J=Uint8Array,it=Uint16Array,Zs=Int32Array,Wr=new J([0,0,0,0,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,0,0,0,0]),Hr=new J([0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13,0,0]),Ws=new J([16,17,18,0,8,7,9,6,10,5,11,4,12,3,13,2,14,1,15]),Vr=function(r,e){for(var t=new it(31),n=0;n<31;++n)t[n]=e+=1<>1|(M&21845)<<1,ze=(ze&52428)>>2|(ze&13107)<<2,ze=(ze&61680)>>4|(ze&3855)<<4,$n[M]=((ze&65280)>>8|(ze&255)<<8)>>1;Lt=(function(r,e,t){for(var n=r.length,i=0,o=new it(e);i>c]=l}else for(a=new it(n),i=0;i>15-r[i]);return a}),At=new J(288);for(M=0;M<144;++M)At[M]=8;for(M=144;M<256;++M)At[M]=9;for(M=256;M<280;++M)At[M]=7;for(M=280;M<288;++M)At[M]=8;Xr=new J(32);for(M=0;M<32;++M)Xr[M]=5;qs=Lt(At,9,1),js=Lt(Xr,5,1),Nn=function(r){for(var e=r[0],t=1;te&&(e=r[t]);return e},ye=function(r,e,t){var n=e/8|0;return(r[n]|r[n+1]<<8)>>(e&7)&t},Mn=function(r,e){var t=e/8|0;return(r[t]|r[t+1]<<8|r[t+2]<<16)>>(e&7)},Yr=function(r){return(r+7)/8|0},jt=function(r,e,t){return(e==null||e<0)&&(e=0),(t==null||t>r.length)&&(t=r.length),new J(r.subarray(e,t))},Xs=["unexpected EOF","invalid block type","invalid length/literal","invalid distance","stream finished","no stream handler",,"no callback","invalid UTF-8 data","extra field too long","date not in range 1980-2099","filename too long","stream finishing","invalid zip data"],le=function(r,e,t){var n=new Error(e||Xs[r]);if(n.code=r,Error.captureStackTrace&&Error.captureStackTrace(n,le),!t)throw n;return n},Jr=function(r,e,t,n){var i=r.length,o=n?n.length:0;if(!i||e.f&&!e.l)return t||new J(0);var s=!t,a=s||e.i!=2,c=e.i;s&&(t=new J(i*3));var l=function(Le){var Ae=t.length;if(Le>Ae){var Mt=new J(Math.max(Ae*2,Le));Mt.set(t),t=Mt}},u=e.f||0,f=e.p||0,h=e.b||0,y=e.l,g=e.d,d=e.m,m=e.n,p=i*8;do{if(!y){u=ye(r,f,1);var w=ye(r,f+1,3);if(f+=3,w)if(w==1)y=qs,g=js,d=9,m=5;else if(w==2){var E=ye(r,f,31)+257,k=ye(r,f+10,15)+4,x=E+ye(r,f+5,31)+1;f+=14;for(var I=new J(x),B=new J(19),C=0;C>4;if(v<16)I[C++]=v;else{var A=0,W=0;for(v==16?(W=3+ye(r,f,3),f+=2,A=I[C-1]):v==17?(W=3+ye(r,f,7),f+=3):v==18&&(W=11+ye(r,f,127),f+=7);W--;)I[C++]=A}}var Ce=I.subarray(0,E),te=I.subarray(E);d=Nn(Ce),m=Nn(te),y=Lt(Ce,d,1),g=Lt(te,m,1)}else le(1);else{var v=Yr(f)+4,S=r[v-4]|r[v-3]<<8,b=v+S;if(b>i){c&&le(0);break}a&&l(h+S),t.set(r.subarray(v,b),h),e.b=h+=S,e.p=f=b*8,e.f=u;continue}if(f>p){c&&le(0);break}}a&&l(h+131072);for(var ut=(1<>4;if(f+=A&15,f>p){c&&le(0);break}if(A||le(2),ve<256)t[h++]=ve;else if(ve==256){Ne=f,y=null;break}else{var ht=ve-254;if(ve>264){var C=ve-257,xe=Wr[C];ht=ye(r,f,(1<>4;He||le(3),f+=He&15;var te=Vs[ue];if(ue>3){var xe=Hr[ue];te+=Mn(r,f)&(1<p){c&&le(0);break}a&&l(h+131072);var Ie=h+ht;if(h>3&1)+(e>>4&1);n>0;n-=!r[t++]);return t+(e&2)},st=(function(){function r(e,t){typeof e=="function"&&(t=e,e={}),this.ondata=t;var n=e&&e.dictionary&&e.dictionary.subarray(-32768);this.s={i:0,b:n?n.length:0},this.o=new J(32768),this.p=new J(0),n&&this.o.set(n)}return r.prototype.e=function(e){if(this.ondata||le(5),this.d&&le(4),!this.p.length)this.p=e;else if(e.length){var t=new J(this.p.length+e.length);t.set(this.p),t.set(e,this.p.length),this.p=t}},r.prototype.c=function(e){this.s.i=+(this.d=e||!1);var t=this.s.b,n=Jr(this.p,this.s,this.o);this.ondata(jt(n,t,this.s.b),this.d),this.o=jt(n,this.s.b-32768),this.s.b=this.o.length,this.p=jt(this.p,this.s.p/8|0),this.s.p&=7},r.prototype.push=function(e,t){this.e(e),this.c(t)},r})();ei=(function(){function r(e,t){this.v=1,this.r=0,st.call(this,e,t)}return r.prototype.push=function(e,t){if(st.prototype.e.call(this,e),this.r+=e.length,this.v){var n=this.p.subarray(this.v-1),i=n.length>3?Js(n):4;if(i>n.length){if(!t)return}else this.v>1&&this.onmember&&this.onmember(this.r-n.length);this.p=n.subarray(i),this.v=0}st.prototype.c.call(this,0),this.s.f&&!this.s.l?(this.v=Yr(this.s.p)+9,this.s={i:0},this.o=new J(0),this.push(new J(0),t)):t&&st.prototype.c.call(this,t)},r})(),Qs=typeof TextDecoder<"u"&&new TextDecoder,eo=0;try{Qs.decode(Ys,{stream:!0}),eo=1}catch{}});var Gn={};mr(Gn,{extractZipEntry:()=>co,extractZipEntryBounded:()=>lo,fetchZipCentralDirectory:()=>uo,parseZipCentralDirectory:()=>_t});function oi(r){let e=new DataView(r.buffer,r.byteOffset,r.byteLength),t=Math.max(0,r.length-ri);for(let n=r.length-ro;n>=t;n--)if(e.getUint32(n,!0)===to)return n;throw new Error("Zip EOCD record not found")}function _t(r){let e=new DataView(r.buffer,r.byteOffset,r.byteLength),t=oi(r),n=e.getUint16(t+10,!0),i=e.getUint32(t+16,!0),o=[],s=i;for(let a=0;a>8,S;v===ti?S=d>>16&65535:w.startsWith("bin/")||w.startsWith("sbin/")||w.includes("/bin/")||w.includes("/sbin/")?S=493:S=420;let b=w.endsWith("/"),E=v===ti&&(S&so)===io;o.push({fileName:w,fileNameBytes:p,compressedSize:u,uncompressedSize:f,compressionMethod:l,localHeaderOffset:m,mode:S,isDirectory:b,isSymlink:E,externalAttrs:d,creatorOS:v}),s+=Dn+h+y+g}return o}function ai(r,e){if(r.byteLength!==e.byteLength)return!1;for(let t=0;t{if(a.byteLength>t-o)throw new Error(`ZIP member ${e.fileName} expands beyond ${t} bytes`);i.set(a,o),o+=a.byteLength}).push(n,!0),o!==t)throw new Error(`ZIP member ${e.fileName} expanded ${o} bytes, expected ${t}`);return i}function fo(r,e){let t=new DataView(r.buffer,r.byteOffset,r.byteLength),n=e.localHeaderOffset;if(n<0||n>r.byteLength-Un||t.getUint32(n,!0)!==ni)throw new Error(`Invalid local file header signature at offset ${n}`);let i=t.getUint16(n+8,!0),o=t.getUint16(n+26,!0),s=t.getUint16(n+28,!0),a=n+Un,c=a+o+s,l=c+e.compressedSize;if(i!==e.compressionMethod||cr.byteLength||!ai(r.subarray(a,a+o),e.fileNameBytes))throw new Error(`ZIP member ${e.fileName} has inconsistent local metadata`);return r.subarray(c,l)}async function uo(r){let e=await fetch(r,{method:"HEAD"});if(!e.ok)throw new Error(`HEAD request failed: ${e.status} ${e.statusText}`);let t=parseInt(e.headers.get("content-length")||"0",10),n=e.headers.get("accept-ranges");if(!t||n!=="bytes"){let p=await fetch(r);if(!p.ok)throw new Error(`Fetch failed: ${p.status} ${p.statusText}`);let w=new Uint8Array(await p.arrayBuffer());return{entries:_t(w),totalSize:w.length}}let i=Math.min(t,ri),o=t-i,s=await fetch(r,{headers:{Range:`bytes=${o}-${t-1}`}});if(s.status!==206){let p=await fetch(r);if(!p.ok)throw new Error(`Fetch failed: ${p.status} ${p.statusText}`);let w=new Uint8Array(await p.arrayBuffer());return{entries:_t(w),totalSize:w.length}}let a=new Uint8Array(await s.arrayBuffer()),c=new DataView(a.buffer,a.byteOffset,a.byteLength),l=oi(a),u=c.getUint32(l+12,!0),f=c.getUint32(l+16,!0);if(f>=o){let p=t,w=new Uint8Array(p);return w.set(a,o),{entries:_t(w),totalSize:p}}let h=f+u-1,y=await fetch(r,{headers:{Range:`bytes=${f}-${h}`}});if(y.status!==206)throw new Error(`Range request for CD failed: ${y.status}`);let g=new Uint8Array(await y.arrayBuffer()),d=t,m=new Uint8Array(d);return m.set(g,f),m.set(a,o),{entries:_t(m),totalSize:d}}var to,no,ni,ri,ro,Dn,Un,ii,si,ti,io,so,oo,ao,Kn=pn(()=>{"use strict";Fn();to=101010256,no=33639248,ni=67324752,ri=65557,ro=22,Dn=46,Un=30,ii=0,si=8,ti=3,io=40960,so=61440,oo=new TextDecoder("utf-8",{fatal:!0,ignoreBOM:!0}),ao=new TextEncoder});var yi={};mr(yi,{DEFAULT_TAR_GZIP_LIMITS:()=>hi,TarParseError:()=>_,parseTarGzip:()=>po});function po(r,e={}){let t=e.label??"TAR gzip archive",n=wo(e.limits,t);if(r.byteLength===0||r.byteLength>n.maxCompressedBytes)throw new _(`${t}: compressed byte count ${r.byteLength} is outside 1..${n.maxCompressedBytes}`);let i=vo(r,t);if(i===0||i>n.maxUncompressedBytes)throw new _(`${t}: declared uncompressed byte count ${i} is outside 1..${n.maxUncompressedBytes}`);let o=Eo(r,t,i);if(o.byteLength!==i)throw new _(`${t}: gzip expanded to ${o.byteLength} bytes, expected ${i}`);let s=new DataView(r.buffer,r.byteOffset,r.byteLength).getUint32(r.byteLength-8,!0);if(So(o)!==s)throw new _(`${t}: gzip CRC32 mismatch`);return mo(o,t,n)}function mo(r,e,t){if(r.byteLength%be!==0)throw new _(`${e}: TAR byte count is not block-aligned`);let n=[],i=0,o=0,s=0,a=null,c={},l=!1;for(;i+be<=r.byteLength;){let u=r.subarray(i,i+be);if(i+=be,Wn(u)){if(i+be>r.byteLength)throw new _(`${e}: TAR end marker is truncated`);let b=r.subarray(i,i+be);if(!Wn(b))throw new _(`${e}: TAR has only one zero end block`);if(i+=be,!Wn(r.subarray(i)))throw new _(`${e}: TAR has nonzero data after its end marker`);l=!0;break}xo(u,e);let f=Pt(u,156,1,e)||"0",h=Vn(u,124,12,`${e}: TAR entry size`),y=Vn(u,100,8,`${e}: TAR entry mode`)&ho,g=Io(u,e,t.maxPathBytes),d=Pt(u,157,100,e);if(f==="x"||f==="g"){if(s+=1,s>t.maxEntries+1)throw new _(`${e}: TAR extension header count exceeds ${t.maxEntries+1}`);let b=li(r,i,h,e);i=fi(i,h,r.byteLength,e);let E=bo(b,e,t);f==="x"?a=E:c={...c,...E};continue}if(o+=1,o>t.maxEntries)throw new _(`${e}: TAR entry count exceeds ${t.maxEntries}`);let m={...c,...a??{}};a=null;let p=m.size===void 0?h:ko(m.size,`${e}: PAX entry size`),w=li(r,i,p,e);i=fi(i,p,r.byteLength,e);let v=Hn(m.path??g,e,t.maxPathBytes),S=m.linkpath??d;switch(f){case"0":case"\0":n.push({path:v,type:"file",mode:y,data:w});break;case"5":Zn(p,e,"directory",v),n.push({path:v,type:"directory",mode:y});break;case"2":Zn(p,e,"symlink",v),di(S,e,v,t.maxLinkBytes,!1),n.push({path:v,type:"symlink",mode:y,linkName:S});break;case"1":Zn(p,e,"hardlink",v),di(S,e,v,t.maxLinkBytes,!0),n.push({path:v,type:"hardlink",mode:y,linkName:Hn(S,`${e}: hardlink target`,t.maxPathBytes)});break;case"3":case"4":case"6":throw new _(`${e}: unsupported TAR device/FIFO entry ${v}`);default:throw new _(`${e}: unsupported TAR entry type ${JSON.stringify(f)} for ${v}`)}}if(!l)throw new _(`${e}: TAR is missing its two-block end marker`);if(a!==null)throw new _(`${e}: local PAX header has no following entry`);return n}function wo(r,e){let t={...hi,...r};for(let[n,i]of Object.entries(t))if(!Number.isSafeInteger(i)||i<=0)throw new _(`${e}: ${n} must be a positive safe integer`);return t}function vo(r,e){if(r.byteLength<18||r[0]!==31||r[1]!==139||r[2]!==8)throw new _(`${e}: invalid gzip header`);return new DataView(r.buffer,r.byteOffset,r.byteLength).getUint32(r.byteLength-4,!0)}function Eo(r,e,t){let n=new Uint8Array(t),i=0,o=!1,s=new ei(a=>{if(a.byteLength>t-i)throw new _(`${e}: gzip expansion exceeds its declared ${t} bytes`);n.set(a,i),i+=a.byteLength});s.onmember=()=>{throw o=!0,new _(`${e}: concatenated gzip members are unsupported`)};try{s.push(r,!0)}catch(a){throw a instanceof _?a:new _(`${e}: cannot gunzip archive: ${Ao(a)}`)}if(o)throw new _(`${e}: concatenated gzip members are unsupported`);return n.subarray(0,i)}function So(r){let e=4294967295;for(let t of r)e=go[(e^t)&255]^e>>>8;return(e^4294967295)>>>0}function zo(){let r=new Uint32Array(256);for(let e=0;e>>1^((t&1)===0?0:3988292384);r[e]=t>>>0}return r}function li(r,e,t,n){if(t>r.byteLength-e)throw new _(`${n}: TAR entry is truncated`);return r.subarray(e,e+t)}function fi(r,e,t,n){let o=Math.ceil(e/be)*be;if(!Number.isSafeInteger(o)||o>t-r)throw new _(`${n}: TAR entry padding is truncated`);return r+o}function bo(r,e,t){let n={},i=0;for(;i9)throw new _(`${e}: invalid PAX record length`);if(s=s*10+d,!Number.isSafeInteger(s))throw new _(`${e}: invalid PAX record length`)}let a=i+s;if(s<=o-i+2||a>r.byteLength||r[a-1]!==10)throw new _(`${e}: truncated PAX record`);let c=o+1;for(;c=a-1)throw new _(`${e}: invalid PAX record`);let l=r.subarray(o+1,c);if(l.byteLength>256)throw new _(`${e}: PAX record key is too long`);let u=qn(l,`${e}: PAX record key`),f=r.subarray(c+1,a-1),h=u==="path"?t.maxPathBytes:u==="linkpath"?t.maxLinkBytes:u==="size"?32:0;if(h===0){i=a;continue}if(f.byteLength>h)throw new _(`${e}: PAX ${u} value is too long`);let y=qn(f,`${e}: PAX record value`);n[u]=y,i=a}return n}function ko(r,e){if(!/^(0|[1-9][0-9]*)$/.test(r))throw new _(`${e} is invalid`);let t=Number(r);if(!Number.isSafeInteger(t)||t<0)throw new _(`${e} is invalid`);return t}function xo(r,e){let t=Vn(r,148,8,`${e}: TAR checksum`),n=0;for(let i=0;i=148&&i<156?32:r[i];if(t!==n)throw new _(`${e}: TAR checksum mismatch`)}function Io(r,e,t){let n=Pt(r,0,100,e),i=Pt(r,345,155,e);return Hn(i?`${i}/${n}`:n,e,t)}function Hn(r,e,t){let n=r;for(;n.startsWith("./");)n=n.slice(2);return n=n.replace(/\/+$/g,""),Lo(n,`${e}: TAR path`,t),n}function Pt(r,e,t,n){let i=e,o=e+t;for(;in||r.includes("\0"))throw new _(`${e}: link target for ${t} is invalid`);if(i&&r.includes("\\"))throw new _(`${e}: hardlink target for ${t} is invalid`)}function Lo(r,e,t){if(r.length===0||r.startsWith("/")||r.includes("\0")||r.includes("\\")||ui.encode(r).byteLength>t)throw new _(`${e} ${JSON.stringify(r)} must be a bounded relative POSIX path`);for(let n of r.split("/"))if(n.length===0||n==="."||n==="..")throw new _(`${e} ${JSON.stringify(r)} contains an unsafe path segment`)}function Wn(r){for(let e of r)if(e!==0)return!1;return!0}function qn(r,e){try{return yo.decode(r)}catch{throw new _(`${e} contains non-UTF-8 text`)}}function Ao(r){return r instanceof Error?r.message:String(r)}var be,ho,ci,yo,ui,go,hi,_,gi=pn(()=>{"use strict";Fn();be=512,ho=4095,ci=1024*1024,yo=new TextDecoder("utf-8",{fatal:!0,ignoreBOM:!0}),ui=new TextEncoder,go=zo(),hi=Object.freeze({maxCompressedBytes:256*ci,maxUncompressedBytes:512*ci,maxEntries:1e5,maxPathBytes:4096,maxLinkBytes:65536}),_=class extends Error{constructor(e){super(e),this.name="TarParseError"}}});import{existsSync as Rt,lstatSync as hn,readdirSync as Jo,readFileSync as Ge,realpathSync as pe,statSync as Ke}from"node:fs";import{createHash as Bi}from"node:crypto";import{spawnSync as fr}from"node:child_process";import{basename as Qo,dirname as Ct,isAbsolute as yn,join as F,relative as ea,resolve as ge,sep as ta}from"node:path";import{fileURLToPath as na}from"node:url";var wr=["__abi_version","kernel_alloc_scratch","kernel_create_process","kernel_create_process_with_stdio","kernel_get_parent_pid","kernel_get_process_exit_signal","kernel_get_process_state","kernel_handle_channel","kernel_has_sa_nocldstop","kernel_host_adapter_manifest_len","kernel_host_adapter_manifest_ptr","kernel_mark_process_signaled","kernel_pipe_has_readers","kernel_posix_timer_fire","kernel_prepare_write_operation","kernel_reap_exited_child","kernel_remove_process","kernel_wait_child_poll"];var K={LINK_MAX:0,MAX_CANON:1,MAX_INPUT:2,NAME_MAX:3,PATH_MAX:4,PIPE_BUF:5,CHOWN_RESTRICTED:6,NO_TRUNC:7,VDISABLE:8,SYNC_IO:9,ASYNC_IO:10,PRIO_IO:11,SOCK_MAXBUF:12,FILESIZEBITS:13,REC_INCR_XFER_SIZE:14,REC_MAX_XFER_SIZE:15,REC_MIN_XFER_SIZE:16,REC_XFER_ALIGN:17,ALLOC_SIZE_MIN:18,SYMLINK_MAX:19,POSIX2_SYMLINKS:20,FALLOC:21,TEXTDOMAIN_MAX:22,TIMESTAMP_RESOLUTION:23};function L(r,e){let t=0,n=0,i=e;for(;;){let o=r[i++];if(t|=(o&127)<=21&&n<=34?yt(e,t):n===84||n>=92&&n<=99||n>=112&&n<=123||n>=124&&n<=131||n>=156&&n<=159?t+1:t:r===254?n===0||n===1||n===2?yt(e,t):n===3?t:n>=16&&n<=79?yt(e,t):null:null}function qi(r,e,t){let[n,i]=L(r,e);e+=i+n;let[o,s]=L(r,e);e+=s+o;let a=r[e++];if(a===0){t.funcImports++;let[,c]=L(r,e);e+=c}else if(a===1){e++;let c=r[e++],[,l]=L(r,e);if(e+=l,c&1){let[,u]=L(r,e);e+=u}}else if(a===2){let c=r[e++],[,l]=L(r,e);if(e+=l,c&1){let[,u]=L(r,e);e+=u}}else a===3&&(t.globalImports++,e+=2);return e}function vn(r){return r.length>=8&&r[0]===0&&r[1]===97&&r[2]===115&&r[3]===109}function $t(r,e){let[t,n]=L(r,e);return e+=n,[new TextDecoder().decode(r.subarray(e,e+t)),e+t]}function ji(r,e){if(e.length===0)return!0;let t=new TextEncoder().encode(e);e:for(let n=0;n<=r.length-t.length;n++){for(let i=0;it.startsWith("reloc."))}function vr(r,e={}){let t=[];if(Qi(r)&&t.push("contains asyncify_"),e.expectedAbi!==void 0&&e.expectedAbi!==null){let s=rs(r);s!==null&&s!==e.expectedAbi&&t.push(`ABI ${s}, expected ${e.expectedAbi}`)}let n=new Set(Yi(r));if(e.requiredExports){let s=e.requiredExports.filter(a=>!n.has(a));s.length>0&&t.push(`missing required exports: ${s.join(", ")}`)}let i=mn.filter(s=>n.has(s));if(e.forbidForkInstrumentation&&i.length>0&&t.push("contains wasm-fork-instrument exports"),e.requireForkInstrumentation??!ts(r)){let s=i.length===mn.length;if(i.length>0&&!s){let a=mn.filter(c=>!n.has(c));t.push(`incomplete wasm-fork-instrument exports; missing ${a.join(", ")}`)}es(r)&&!s&&t.push("imports kernel.kernel_fork without complete wasm-fork-instrument exports")}return t}function ns(r,e){let t=new Uint8Array(r);if(t.length<8)return null;let n=0,i=null,o=null,s=8;for(;s=c)return null;let d=a;for(let w=0;w=g)return null;let[d,m]=L(t,y);y+=m;for(let p=0;pg)return null}return y}function h(y,g=0){if(g>4)return null;let d=u(y);if(!d)return null;let m=f(d.start,d.end);if(m===null)return null;let p=m,w=d.end;for(;p=32&&v<=38||v===208){let[,S]=L(t,p);p+=S}else if(v>=40&&v<=62)p=yt(t,p);else if(v===63||v===64)p++;else if(v===66){let[,S]=Wi(t,p);p+=S}else if(v===67)p+=4;else if(v===68)p+=8;else if(v===252||v===253||v===254){let S=Vi(v,t,p);if(S===null)return null;p=S}}return null}return h(i)}function rs(r){return ns(r,"__abi_version")}var is=ArrayBuffer,H=Uint8Array,Ft=Uint16Array,ss=Int16Array;var Dt=Int32Array,En=function(r,e,t){if(H.prototype.slice)return H.prototype.slice.call(r,e,t);(e==null||e<0)&&(e=0),(t==null||t>r.length)&&(t=r.length);var n=new H(t-e);return n.set(r.subarray(e,t)),n},pt=function(r,e,t,n){if(H.prototype.fill)return H.prototype.fill.call(r,e,t,n);for((t==null||t<0)&&(t=0),(n==null||n>r.length)&&(n=r.length);tr.length)&&(n=r.length);t2046MB)","invalid block type","FSE accuracy too high","match distance too far back","unexpected EOF"],V=function(r,e,t){var n=new Error(e||as[r]);if(n.code=r,Error.captureStackTrace&&Error.captureStackTrace(n,V),!t)throw n;return n},Er=function(r,e,t){for(var n=0,i=0;n>>0},ls=function(r,e){var t=r[0]|r[1]<<8|r[2]<<16;if(t==3126568&&r[3]==253){var n=r[4],i=n>>5&1,o=n>>2&1,s=n&3,a=n>>6;n&8&&V(0);var c=6-i,l=s==3?4:s,u=Er(r,c,l);c+=l;var f=a?1<>3);y=g+(g>>3)*(r[5]&7)}y>2145386496&&V(1);var d=new H((e==1?h||y:e?0:y)+12);return d[0]=1,d[4]=4,d[8]=8,{b:c+f,y:0,l:0,d:u,w:e&&e!=1?e:d.subarray(12),e:y,o:new Dt(d.buffer,0,3),u:h,c:o,m:Math.min(131072,y)}}else if((t>>4|r[3]<<20)==25481893)return cs(r,4)+8;V(0)},$e=function(r){for(var e=0;1<t&&V(3);for(var o=1<0;){var w=$e(s+1),v=n>>3,S=(1<>(n&7)&S,E=(1<E&&(b-=k)),h[++a]=--b,b==-1?(s+=b,m[--u]=a):s-=b,!b)do{var I=n>>3;c=(r[I]|r[I+1]<<8)>>(n&7)&3,n+=2,a+=c}while(c==3)}(a>255||s)&&V(0);for(var B=0,C=(o>>1)+(o>>3)+3,ee=o-1,j=0;j<=a;++j){var R=h[j];if(R<1){y[j]=-R;continue}for(l=0;l=u)}}for(B&&V(0),l=0;l>3,{b:i,s:m,n:p,t:g}]},fs=function(r,e){var t=0,n=-1,i=new H(292),o=r[e],s=i.subarray(0,256),a=i.subarray(256,268),c=new Ft(i.buffer,268);if(o<128){var l=mt(r,e+1,6),u=l[0],f=l[1];e+=o;var h=u<<3,y=r[e];y||V(0);for(var g=0,d=0,m=f.b,p=m,w=(++e<<3)-8+$e(y);w-=m,!(w>3;if(g+=(r[v]|r[v+1]<<8)>>(w&7)&(1<>3,d+=(r[v]|r[v+1]<<8)>>(w&7)&(1<255&&V(0)}else{for(n=o-127;t>4,s[t+1]=S&15}++e}var b=0;for(t=0;t11&&V(0),b+=E&&1<0;--t){var j=c[t];pt(ee,t,j,c[t-1]=j+a[t]*(1<a&&f>3,y=(r[h]|r[h+1]<<8|r[h+2]<<16)>>(u&7);c=(c<>2,s=o<<1,a=o+s;gt(r.subarray(n,n+=r[0]|r[1]<<8),e.subarray(0,o),t),gt(r.subarray(n,n+=r[2]|r[3]<<8),e.subarray(o,s),t),gt(r.subarray(n,n+=r[4]|r[5]<<8),e.subarray(s,a),t),gt(r.subarray(n),e.subarray(a),t)},ms=function(r,e,t){var n,i=e.b,o=r[i],s=o>>1&3;e.l=o&1;var a=o>>3|r[i+1]<<5|r[i+2]<<13,c=(i+=3)+a;if(s==1)return i>=r.length?void 0:(e.b=i+1,t?(pt(t,r[i],e.y,e.y+=a),t):pt(new H(a),r[i]));if(!(c>r.length)){if(s==0)return e.b=c,t?(t.set(r.subarray(i,c),e.y),e.y+=a,t):En(r,i,c);if(s==2){var l=r[i],u=l&3,f=l>>2&3,h=l>>4,y=0,g=0;u<2?f&1?h|=r[++i]<<4|(f&2&&r[++i]<<12):h=l>>3:(g=f,f<2?(h|=(r[++i]&63)<<4,y=r[i]>>6|r[++i]<<2):f==2?(h|=r[++i]<<4|(r[++i]&3)<<12,y=r[i]>>2|r[++i]<<6):(h|=r[++i]<<4|(r[++i]&63)<<12,y=r[i]>>6|r[++i]<<2|r[++i]<<10)),++i;var d=t?t.subarray(e.y,e.y+e.m):new H(e.m),m=d.length-h;if(u==0)d.set(r.subarray(i,i+=h),m);else if(u==1)pt(d,r[i++],m);else{var p=e.h;if(u==2){var w=fs(r,i);y+=i-(i=w[0]),e.h=p=w[1]}else p||V(0);(g?ps:gt)(r.subarray(i,i+=y),d.subarray(m),p)}var v=r[i++];if(v){v==255?v=(r[i++]|r[i++]<<8)+32512:v>127&&(v=v-128<<8|r[i++]);var S=r[i++];S&3&&V(0);for(var b=[us,hs,ds],E=2;E>-1;--E){var k=S>>(E<<1)+2&3;if(k==1){var x=new H([0,0,r[i++]]);b[E]={s:x.subarray(2,3),n:x.subarray(0,1),t:new Ft(x.buffer,0,1),b:0}}else k==2?(n=mt(r,i,9-(E&1)),i=n[0],b[E]=n[1]):k==3&&(e.t||V(0),b[E]=e.t[E])}var I=e.t=b,B=I[0],C=I[1],ee=I[2],j=r[c-1];j||V(0);var R=(c<<3)-8+$e(j)-ee.b,P=R>>3,A=0,W=(r[P]|r[P+1]<<8)>>(R&7)&(1<>3;var Ce=(r[P]|r[P+1]<<8)>>(R&7)&(1<>3;var te=(r[P]|r[P+1]<<8)>>(R&7)&(1<>3;var He=1<>>(R&7)&He-1);P=(R-=zn[Ne])>>3;var Ie=gs[Ne]+((r[P]|r[P+1]<<8|r[P+2]<<16)>>(R&7)&(1<>3;var Me=ys[ut]+((r[P]|r[P+1]<<8|r[P+2]<<16)>>(R&7)&(1<>3,W=ee.t[W]+((r[P]|r[P+1]<<8)>>(R&7)&(1<>3,te=B.t[te]+((r[P]|r[P+1]<<8)>>(R&7)&(1<>3,Ce=C.t[Ce]+((r[P]|r[P+1]<<8)>>(R&7)&(1<3)e.o[2]=e.o[1],e.o[1]=e.o[0],e.o[0]=ue-=3;else{var Ve=ue-(Me!=0);Ve?(ue=Ve==3?e.o[0]-1:e.o[Ve],Ve>1&&(e.o[2]=e.o[1]),e.o[1]=e.o[0],e.o[0]=ue):ue=e.o[0]}for(var E=0;EIe&&(Ae=Ie);for(var E=0;E=i){let x=(y+1)*4096;try{e.grow(x)}catch{throw new z(Y)}if(i=Math.floor(e.byteLength/4096),y>=i)throw new z(Y)}new Uint8Array(e).fill(0);let g=new r(e);g.w32(Ln,xn),g.w32(An,In),g.w32(Kt,4096),g.w32(je,i),g.w32(_e,s),g.w32(Fe,u),g.w32(Wt,f),g.w32(xr,h),g.w32(Ht,y),g.w32(As,a),g.w32(_s,c),g.w32(Ps,l),g.w32(Et,o),g.w32(Ir,256);let d=f*4096;for(let x=0;x>2)+(x>>5);g.i32[I]|=1<<(x&31)}let m=i-y;Atomics.store(g.i32,Xe>>2,m),g.blockAllocHint=y;let p=u*4096;g.i32[p>>2]|=3,Atomics.store(g.i32,Zt>>2,s-2),g.inodeAllocHint=2;let w=g.inodeOffset(1);g.w32(w+N,U|493),g.w32(w+D,2),g.w64(w+se,1);let v=g.blockAlloc();if(v<0)throw new z(Y);g.w32(w+X,v);let S=v*4096,b=Oe(O+1),E=Oe(O+2);g.w32(S,1),g.view.setUint16(S+4,b,!0),g.view.setUint16(S+6,1,!0),g.u8[S+O]=46;let k=S+b;return g.w32(k,1),g.view.setUint16(k+4,E,!0),g.view.setUint16(k+6,2,!0),g.u8[k+O]=46,g.u8[k+O+1]=46,g.w64(w+T,b+E),Atomics.store(g.i32,_n>>2,1),g}static inspectImageCapacity(e){if(e.byteLengththis.snapshotBytesUnlocked(e))}snapshotState(e){return this.withNamespaceLock(()=>({bytes:this.snapshotBytesUnlocked(e),identities:this.collectIdentityStateUnlocked()}))}identityState(){return this.withNamespaceLock(()=>this.collectIdentityStateUnlocked())}snapshotBytesUnlocked(e){let t=e?.normalizeTimestampsMs;if(t!==void 0&&(!Number.isSafeInteger(t)||t<0))throw new z(Z,"Snapshot timestamp must be a non-negative safe integer in milliseconds");let n=t===void 0?void 0:BigInt(t);for(let a=0;a>2)!==0)throw new z(On,"Cannot save a VFS image with open descriptors")}let i=this.r32(_e);for(let a=0;a=1&&this.inodeIsAllocated(a)?n:0n;s.setBigUint64(c+St,l,!0),s.setBigUint64(c+ne,l,!0),s.setBigUint64(c+q,l,!0)}}return o}collectIdentityStateUnlocked(){let e=new Map,t=[{ino:1,path:"/"}],n=new Set;for(;t.length>0;){let i=t.pop();if(n.has(i.ino))throw new z($);n.add(i.ino);let o=this.inodeOffset(i.ino);if((this.r32(o+N)&G)!==U)throw new z($);let s=this.r64(o+T),a=0;for(;a>2)>>>0,paths:[]},e.set(E,k)),k.paths.push(v),(this.r32(S+N)&G)===U&&t.push({ino:d,path:v})}}y+=m}a+=h}}return e}statfs(){let e=this.r32(Kt),t=this.r32(je),n=this.r32(Et),i=typeof this.buffer.maxByteLength=="number"?this.buffer.maxByteLength:this.buffer.byteLength,o=Math.floor(i/e),s=Math.max(t,Math.min(n,o)),a=Atomics.load(this.i32,Xe>>2),c=Math.max(0,s-t);return{blockSize:e,totalBlocks:s,freeBlocks:a+c,totalInodes:this.r32(_e),freeInodes:Atomics.load(this.i32,Zt>>2),maxName:255}}r32(e){return this.view.getUint32(e,!0)}w32(e,t){this.view.setUint32(e,t,!0)}r64(e){return Number(this.view.getBigUint64(e,!0))}w64(e,t){this.view.setBigUint64(e,BigInt(t),!0)}waitForAtomicChange(e,t){if(this.atomicsWaitAllowed!==!1)try{Atomics.wait(this.i32,e,t),this.atomicsWaitAllowed=!0;return}catch(n){if(!(n instanceof TypeError))throw n;this.atomicsWaitAllowed=!1}for(;Atomics.load(this.i32,e)===t;);}resetAllocationHints(){this.blockAllocHint=this.findNextFreeBlockHint(),this.inodeAllocHint=this.findNextFreeInodeHint()}findNextFreeBlockHint(){let e=this.r32(je),t=this.r32(Ht),n=this.r32(Wt)*4096;for(let i=t;i>2)+(i>>5),s=i&31;if((Atomics.load(this.i32,o)&1<>2)+(n>>5),o=n&31;if((Atomics.load(this.i32,i)&1<>2;for(;;){if(Atomics.compareExchange(this.i32,e,0,1)===0)return;this.waitForAtomicChange(e,1)}}sbUnlock(){let e=Vt>>2;Atomics.store(this.i32,e,0),Atomics.notify(this.i32,e,1/0)}namespaceLock(){let e=qt>>2;for(;;){if(Atomics.compareExchange(this.i32,e,0,1)===0)return;this.waitForAtomicChange(e,1)}}namespaceUnlock(){let e=qt>>2;Atomics.store(this.i32,e,0),Atomics.notify(this.i32,e,1/0)}withNamespaceLock(e){this.namespaceLock();try{return e()}finally{this.namespaceUnlock()}}resetRestoredRuntimeState(){Atomics.store(this.i32,Vt>>2,0),Atomics.store(this.i32,qt>>2,0),this.u8.fill(0,256,4096);let e=this.r32(_e),t=this.r32(Fe)*4096;for(let n=0;n>5)*4)&1<<(n&31))===0||this.r32(i+D)!==0)continue;let s=this.r32(i+N),a=this.r64(i+T);(s&G)===vt&&a<=40?(this.u8.fill(0,i+X,i+X+40),this.w64(i+T,0)):this.inodeTruncate(n,0),this.inodeFree(n)}}blockAlloc(){let e=this.r32(je),t=this.r32(Wt)*4096,n=this.r32(Ht),i=this.blockAllocHint>=n&&this.blockAllocHint>2)+(a>>5),l=a&31,u=Atomics.load(this.i32,c);if(u&1<>2,1),this.blockAllocHint=a+1>2)+(e>>5),i=e&31;for(;;){let o=Atomics.load(this.i32,n),s=o&~(1<>2,1),e>=this.r32(Ht)&&e>2)>0)return 0;let e=this.r32(je),t=this.r32(Et),n=this.r32(Ir),i=e+n;if(i>t&&(i=t,n=i-e,n===0))return Y;let o=i*4096;if(this.buffer.byteLength>2,n),Atomics.add(this.i32,_n>>2,1),this.blockAllocHint=e,0}finally{this.sbUnlock()}}inodeOffset(e){let n=this.r32(xr)+Math.floor(e/32),i=e%32*128;return n*4096+i}inodeAlloc(){let e=this.r32(_e),t=this.r32(Fe)*4096,n=this.inodeAllocHint>=2&&this.inodeAllocHint>2)+(s>>5),c=s&31,l=Atomics.load(this.i32,a);if(l&1<>2,1),this.inodeAllocHint=s+1>2,1)+1}inodeFree(e){let n=(this.r32(Fe)*4096>>2)+(e>>5),i=e&31;for(;;){let o=Atomics.load(this.i32,n);if((o&1<>2,1),e>=2&&e0&&this.w32(n+Pe,i-1),i<=1&&this.r32(n+D)===0&&(this.inodeTruncate(e,0),t=!0)}finally{this.inodeWriteUnlock(e)}t&&this.inodeFree(e)}inodeDropLinkRefLocked(e){let t=this.inodeOffset(e),n=this.r32(t+D);return n>1?(this.w32(t+D,n-1),this.w64(t+q,Date.now()),!1):this.inodeOrphanLocked(e)}inodeOrphanLocked(e){let t=this.inodeOffset(e);if(this.w32(t+D,0),this.w64(t+q,Date.now()),this.r32(t+Pe)>0)return!1;let n=this.r32(t+N),i=this.r64(t+T);return(n&G)===vt&&i<=40?(this.u8.fill(0,t+X,t+X+40),this.w64(t+T,0)):this.inodeTruncate(e,0),!0}inodeReadLock(e){let t=this.inodeOffset(e)+Ye>>2;for(;;){let n=Atomics.load(this.i32,t);if(n&Rr){this.waitForAtomicChange(t,n);continue}if(Atomics.compareExchange(this.i32,t,n,n+1)===n)return}}inodeReadUnlock(e){let t=this.inodeOffset(e)+Ye>>2;(Atomics.sub(this.i32,t,1)&Os)===1&&Atomics.notify(this.i32,t,1)}inodeWriteLock(e){let t=this.inodeOffset(e)+Ye>>2;for(;;){let n=Atomics.load(this.i32,t);if(n!==0){this.waitForAtomicChange(t,n);continue}if(Atomics.compareExchange(this.i32,t,0,Rr)===0)return}}inodeWriteUnlock(e){let t=this.inodeOffset(e)+Ye>>2;Atomics.store(this.i32,t,0),Atomics.notify(this.i32,t,1/0)}inodeBlockMap(e,t,n){let i=this.inodeOffset(e);if(t<10){let o=this.r32(i+X+t*4);if(o!==0)return o;if(!n)return 0;let s=this.blockAllocWithGrow();return s<0||this.w32(i+X+t*4,s),s}if(t-=10,t<1024){let o=this.r32(i+zt),s=!1;if(o===0){if(!n)return 0;if(o=this.blockAllocWithGrow(),o<0)return o;this.w32(i+zt,o),s=!0}let a=o*4096+t*4,c=this.r32(a);if(c!==0)return c;if(!n)return 0;let l=this.blockAllocWithGrow();return l<0?(s&&(this.w32(i+zt,0),this.blockFree(o)),l):(this.w32(a,l),l)}if(t-=1024,t<1024*1024){let o=Math.floor(t/1024),s=t%1024,a=this.r32(i+Je),c=!1;if(a===0){if(!n)return 0;if(a=this.blockAllocWithGrow(),a<0)return a;this.w32(i+Je,a),c=!0}let l=a*4096+o*4,u=this.r32(l),f=!1;if(u===0){if(!n)return 0;if(u=this.blockAllocWithGrow(),u<0)return c&&(this.w32(i+Je,0),this.blockFree(a)),u;this.w32(l,u),f=!0}let h=u*4096+s*4,y=this.r32(h);if(y!==0)return y;if(!n)return 0;let g=this.blockAllocWithGrow();return g<0?(f&&(this.w32(l,0),this.blockFree(u)),c&&(this.w32(i+Je,0),this.blockFree(a)),g):(this.w32(h,g),g)}return Z}inodeReadData(e,t,n,i){let o=this.inodeOffset(e),s=this.r64(o+T);if(t>=s)return 0;t+i>s&&(i=s-t);let a=0,c=0;for(;i>0;){let l=Math.floor(t/4096),u=t%4096,f=4096-u;f>i&&(f=i);let h=this.inodeBlockMap(e,l,!1);if(h<=0)n.fill(0,c,c+f);else{let y=h*4096+u;n.set(this.u8.subarray(y,y+f),c)}c+=f,t+=f,i-=f,a+=f}return a}inodeWriteData(e,t,n,i){let o=this.inodeOffset(e),s=this.r64(o+T);t>s&&this.zeroOldEofTail(e,s);let a=0,c=0;for(;i>0;){let l=Math.floor(t/4096),u=t%4096,f=4096-u;f>i&&(f=i);let h=this.inodeBlockMap(e,l,!0);if(h<0){if(a===0)return h;break}let y=h*4096+u;this.u8.set(n.subarray(c,c+f),y),c+=f,t+=f,i-=f,a+=f}if(a>0&&t>this.r64(o+T)&&this.w64(o+T,t),a>0){let l=Date.now();this.w64(o+ne,l),this.w64(o+q,l),Atomics.add(this.i32,o+oe>>2,1)}return a}zeroInodeRange(e,t,n){for(;t0){let c=a*4096+o;this.u8.fill(0,c,c+s)}t+=s}}zeroOldEofTail(e,t){let n=t%4096;if(n===0)return;let i=Math.floor(t/4096),o=this.inodeBlockMap(e,i,!1);if(o<=0)return;let s=o*4096+n;this.u8.fill(0,s,o*4096+4096)}freeBlocksFrom(e,t){let n=this.inodeOffset(e);for(let s=t;s<10;s++){let a=this.r32(n+X+s*4);a&&(this.blockFree(a),this.w32(n+X+s*4,0))}let i=this.r32(n+zt);if(i){let s=t>10?t-10:0;for(let a=s;a<1024;a++){let c=i*4096+a*4,l=this.r32(c);l&&(this.blockFree(l),this.w32(c,0))}s===0&&(this.blockFree(i),this.w32(n+zt,0))}let o=this.r32(n+Je);if(o){let s=t>1034?t-10-1024:0,a=Math.floor(s/1024);for(let c=a;c<1024;c++){let l=o*4096+c*4,u=this.r32(l);if(!u)continue;let f=c===a?s%1024:0;for(let h=f;h<1024;h++){let y=u*4096+h*4,g=this.r32(y);g&&(this.blockFree(g),this.w32(y,0))}f===0&&(this.blockFree(u),this.w32(l,0))}a===0&&(this.blockFree(o),this.w32(n+Je,0))}}inodeTruncate(e,t,n=!1){let i=this.inodeOffset(e),o=this.r64(i+T),s=t!==o;if(t>=o){if(t>o&&this.zeroOldEofTail(e,o),this.w64(i+T,t),s||n){let c=Date.now();this.w64(i+ne,c),this.w64(i+q,c),Atomics.add(this.i32,i+oe>>2,1)}return}t%4096!==0&&this.zeroInodeRange(e,t,Math.ceil(t/4096)*4096);let a=Math.ceil(t/4096);if(this.freeBlocksFrom(e,a),this.w64(i+T,t),s||n){let c=Date.now();this.w64(i+ne,c),this.w64(i+q,c),Atomics.add(this.i32,i+oe>>2,1)}}validateFileSize(e){if(!Number.isSafeInteger(e)||e<0)throw new z(Z);if(e>Qe)throw new z(bt)}validateSeekPosition(e){if(!Number.isSafeInteger(e))throw new z(Mr);if(e<0)throw new z(Z);if(e>Qe)throw new z(bt)}touchDirectoryMutation(e){let t=this.inodeOffset(e),n=Date.now();this.w64(t+ne,n),this.w64(t+q,n);let i=Atomics.add(this.i32,t+_r>>2,1)+1>>>0,o=this.dirIndexes.get(e);o&&(o.mutationSequence=i,o.size=this.r64(t+T))}dirNameKey(e){return et(e)}dirEntryNameMatches(e,t){if(this.view.getUint16(e+6,!0)!==t.length)return!1;for(let i=0;i=O&&n%4===0&&e+n<=t&&i<=n-O}inodeIsAllocated(e){let t=this.r32(_e);if(e<=0||e>=t)return!1;let n=this.r32(Fe)*4096;return(Atomics.load(this.i32,(n>>2)+(e>>5))&1<<(e&31))!==0}rebuildDirIndex(e,t,n,i){let o=new Map,s=[],a=0;for(;a4096-u&&(y=4096-u);let g=u;for(;g=O&&s.push({abs:d,recLen:p});g+=p}a+=y}let c={generation:t,mutationSequence:n,size:i,entries:o,free:s};return this.dirIndexes.set(e,c),c}getDirIndex(e){let t=this.inodeOffset(e),n=this.r64(t+T),i=this.r64(t+se),o=Atomics.load(this.i32,t+_r>>2)>>>0,s=this.dirIndexes.get(e);return s&&s.generation===i&&s.mutationSequence===o&&s.size===n?s:(s&&this.dirIndexes.delete(e),n=0;s--){let a=e.free[s];if(!(a.recLen4096-c&&(f=4096-c);let h=c;for(;hn)return-1;a=c,s+=l}return s===n?a:-1}dirAppendEntry(e,t,n,i=-1){let o=this.inodeOffset(e),s=this.r64(o+T),a=Oe(O+t.length),c=s,l=Math.floor(c/4096),u=c%4096,f=0;if(u!==0&&u+a>4096){let g=4096-u,d=0;if(g>=O){if(d=this.inodeBlockMap(e,l,!1),d<=0)return $}else if(i<0&&(i=this.findLastDirEntryInBlock(e,l,u)),i<0)return $;if(f=this.inodeBlockMap(e,l+1,!0),f<0)return f;if(g>=O){let m=d*4096+u;this.w32(m,0),this.view.setUint16(m+4,g,!0),this.view.setUint16(m+6,0,!0)}else{let p=this.view.getUint16(i+4,!0)+g;this.view.setUint16(i+4,p,!0),this.updateDirIndexRecLen(e,i,p)}c=(l+1)*4096,l++,u=0}let h;if(u===0){if(h=f||this.inodeBlockMap(e,l,!0),h<0)return h}else if(h=this.inodeBlockMap(e,l,!1),h<=0)return $;let y=h*4096+u;return this.w32(y,n),this.view.setUint16(y+4,a,!0),this.view.setUint16(y+6,t.length,!0),this.u8.set(t,y+O),this.w64(o+T,c+a),this.touchDirectoryMutation(e),this.updateDirIndexAdd(e,t,n,y,a),0}dirAddEntry(e,t,n){let i=this.getDirIndex(e);if(typeof i=="number")return i;if(i)return this.useDirIndexFreeSlot(i,e,t,n)?0:this.dirAppendEntry(e,t,n);let o=this.inodeOffset(e),s=this.r64(o+T),a=Oe(O+t.length),c=-1,l=0;for(;l4096-f&&(g=4096-f);let d=f;for(;df+g||v>w-O)return $;if(p===0&&w>=a)return this.w32(m,n),this.view.setUint16(m+6,t.length,!0),this.u8.set(t,m+O),this.touchDirectoryMutation(e),this.updateDirIndexAdd(e,t,n,m,w),0;let S=Oe(O+v),b=w-S;if(p!==0&&b>=a){this.view.setUint16(m+4,S,!0);let E=m+S;return this.w32(E,n),this.view.setUint16(E+4,b,!0),this.view.setUint16(E+6,t.length,!0),this.u8.set(t,E+O),this.touchDirectoryMutation(e),this.updateDirIndexAdd(e,t,n,E,b),0}c=m,d+=w}l+=g}return this.dirAppendEntry(e,t,n,c)}dirRemoveEntry(e,t){let n=this.getDirIndex(e);if(typeof n=="number")return n;if(n){let a=this.dirNameKey(t),c=n.entries.get(a);if(!c)return he;if(this.r32(c.abs)===c.ino&&this.view.getUint16(c.abs+4,!0)===c.recLen&&this.view.getUint16(c.abs+6,!0)===c.nameLen&&this.dirEntryNameMatches(c.abs,t))return this.w32(c.abs,0),n.entries.delete(a),n.free.push({abs:c.abs,recLen:c.recLen}),this.touchDirectoryMutation(e),0;n.entries.delete(a)}let i=this.inodeOffset(e),o=this.r64(i+T),s=0;for(;s4096-c&&(f=4096-c);let h=c;for(;h4096-l&&(h=4096-l);let y=l;for(;y4096-s&&(l=4096-s);let u=s;for(;us+l||g>y-O)throw new z($);if(h!==0){if(g===1&&this.u8[f+O]===46){u+=y;continue}if(g===2&&this.u8[f+O]===46&&this.u8[f+O+1]===46){u+=y;continue}return!1}u+=y}i+=l}return!0}dirIsAncestor(e,t){let n=t;for(let i=0;i<8*1024;i++){if(n===e)return!0;if(n===1)return!1;let o=this.dirLookup(n,Br);if(o<0||o===n)throw new z($);n=o}throw new z($)}pathResolve(e,t){if(!e.startsWith("/"))return he;let n=1,i=e.split("/").filter(s=>s.length>0),o=0;for(let s=0;s255)return Tn;let c=ae.encode(a),l;this.inodeReadLock(n);try{let h=this.inodeOffset(n);if((this.r32(h+N)&G)!==U)return Ee;l=this.dirLookup(n,c)}finally{this.inodeReadUnlock(n)}if(l<0)return l;let u=this.inodeOffset(l);if((this.r32(u+N)&G)===vt&&(!(s===i.length-1)||t)){if(++o>8)return Nr;let y=this.r64(u+T),g;if(y<=40)g=et(this.u8.subarray(u+X,u+X+y));else{let d=new Uint8Array(y);this.inodeReadData(l,0,d,y),g=xt.decode(d)}if(g.startsWith("/")){n=1;let d=g.split("/").filter(p=>p.length>0),m=i.slice(s+1);i.length=0,i.push(...d,...m),s=-1}else{let d=g.split("/").filter(p=>p.length>0),m=i.slice(s+1);i.length=s,i.push(...d,...m),s--}continue}n=l}return n}pathResolveParent(e){if(!e.startsWith("/"))throw new z(Z,"Path must be absolute");let t=e.split("/").filter(c=>c.length>0);if(t.length===0)throw new z(Z,"Cannot operate on /");let n=t.pop();if(n.length>255)throw new z(Tn);let i="/"+t.join("/"),o=this.pathResolve(i,!0);if(o<0)throw new z(o);let s=this.inodeOffset(o);if((this.r32(s+N)&G)!==U)throw new z(Ee);return{parentIno:o,name:n}}fdAlloc(e,t,n){for(let i=0;i>2;if(Atomics.compareExchange(this.i32,s,0,1)===0)return this.w32(o+Pr,e),this.w64(o+De,0),this.w32(o+Or,t),this.w32(o+Tr,n?1:0),this.inodeAddOpenRef(e)?i:(Atomics.store(this.i32,s,0),he)}return Cr}fdGet(e){if(e<0||e>=Ut)return null;let t=256+e*24;return Atomics.load(this.i32,t>>2)?{base:t,ino:this.r32(t+Pr),offset:this.r64(t+De),flags:this.r32(t+Or),isDir:this.r32(t+Tr)!==0}:null}fdFree(e){if(e>=0&&e>2,0)}}buildStat(e){let t=this.inodeOffset(e);return{ino:e,generation:this.r64(t+se),dataSequence:this.r32(t+oe),mode:this.r32(t+N),linkCount:this.r32(t+D),size:this.r64(t+T),mtime:this.r64(t+ne),ctime:this.r64(t+q),atime:this.r64(t+St),uid:this.r32(t+Lr),gid:this.r32(t+Ar)}}namespaceEntryIdentity(e){let t=this.inodeOffset(e);return{ino:e,generation:this.r64(t+se),linkCount:this.r32(t+D),mode:this.r32(t+N)}}open(e,t,n=420){return this.withNamespaceLock(()=>this.openUnlocked(e,t,n))}createLazyStub(e,t){return this.withNamespaceLock(()=>{let n=this.openUnlocked(e,kr|kt,t);try{let i=this.fdGet(n);if(!i)throw new z(Q);this.inodeWriteLock(i.ino);try{return this.inodeTruncate(i.ino,0,!0),this.buildStat(i.ino)}finally{this.inodeWriteUnlock(i.ino)}}finally{this.closeUnlocked(n)}})}replaceIfIdentity(e,t,n,i,o){return this.withNamespaceLock(()=>{let s=this.pathResolve(e,!0);if(s<0||s!==t)return!1;let a=this.inodeOffset(s);if(this.r64(a+se)!==n||this.r32(a+oe)!==i||(this.r32(a+N)&G)!==wt)return!1;this.validateFileSize(o.byteLength),this.inodeWriteLock(s);try{if(this.r64(a+se)!==n||this.r32(a+oe)!==i||this.r64(a+T)!==0)return!1;let c=this.r64(a+ne),l=this.r64(a+q);this.inodeTruncate(s,0,!0);let u=o.byteLength>0?this.inodeWriteData(s,0,o,o.byteLength):0;if(u!==o.byteLength)throw this.inodeTruncate(s,0,!0),Atomics.store(this.i32,a+oe>>2,i),this.w64(a+ne,c),this.w64(a+q,l),new z(u<0?u:Y);return!0}finally{this.inodeWriteUnlock(s)}})}replaceManyIfIdentities(e){return e.length===0?!0:this.withNamespaceLock(()=>{let t=[],n=new Set;for(let o of e){this.validateFileSize(o.data.byteLength);let s=-1;for(let a of o.paths){let c=this.pathResolve(a,!0);if(c!==o.expectedIno)continue;let l=this.inodeOffset(c);if(this.r64(l+se)===o.expectedGeneration&&this.r32(l+oe)===o.expectedDataSequence&&(this.r32(l+N)&G)===wt&&this.r64(l+T)===0){s=c;break}}if(s<0)return!1;if(n.has(s))throw new z(Z,"duplicate conditional replacement inode");n.add(s),t.push({...o,ino:s})}let i=[...n].sort((o,s)=>o-s);for(let o of i)this.inodeWriteLock(o);try{for(let a of t){let c=this.inodeOffset(a.ino);if(this.r64(c+se)!==a.expectedGeneration||this.r32(c+oe)!==a.expectedDataSequence||(this.r32(c+N)&G)!==wt||this.r64(c+T)!==0)return!1}let o=t.map(a=>{let c=this.inodeOffset(a.ino);return{ino:a.ino,dataSequence:this.r32(c+oe),mtime:this.r64(c+ne),ctime:this.r64(c+q)}}),s=0;try{for(let a of t){s++,this.inodeTruncate(a.ino,0,!0);let c=a.data.byteLength>0?this.inodeWriteData(a.ino,0,a.data,a.data.byteLength):0;if(c!==a.data.byteLength)throw new z(c<0?c:Y)}}catch(a){for(let c=s-1;c>=0;c--){let l=o[c],u=this.inodeOffset(l.ino);this.inodeTruncate(l.ino,0,!0),Atomics.store(this.i32,u+oe>>2,l.dataSequence),this.w64(u+ne,l.mtime),this.w64(u+q,l.ctime)}throw a}return!0}finally{for(let o=i.length-1;o>=0;o--)this.inodeWriteUnlock(i[o])}})}openUnlocked(e,t,n=420){let i=t&Gt,o=(t&kt)!==0,s=(t&Bn)!==0;if(o&&s){let f=this.pathResolve(e,!1);if(f>=0)throw new z(tt);if(f!==he)throw new z(f)}let a=this.pathResolve(e,!0);if(a<0&&a===he&&o){let{parentIno:f,name:h}=this.pathResolveParent(e);this.inodeWriteLock(f);try{let y=ae.encode(h),g=this.dirLookup(f,y);if(g>=0){if(s)throw new z(tt);a=g}else{let d=this.inodeAlloc();if(d<0)throw new z(Y);let m=this.inodeOffset(d);this.w32(m+N,wt|n&4095),this.w32(m+D,1),this.w64(m+T,0);let p=Date.now();this.w64(m+St,p),this.w64(m+ne,p),this.w64(m+q,p);let w=this.dirAddEntry(f,y,d);if(w<0)throw this.inodeFree(d),new z(w);a=d}}finally{this.inodeWriteUnlock(f)}}if(a<0)throw new z(a);let c=this.inodeOffset(a),l=this.r32(c+N);if((l&G)===U&&i!==qe)throw new z(Ue);if(t&bs&&(l&G)!==U)throw new z(Ee);if(t&It){if((l&G)===U)throw new z(Ue);this.inodeWriteLock(a),this.inodeTruncate(a,0,!0),this.inodeWriteUnlock(a)}let u=this.fdAlloc(a,t,!1);if(u<0)throw new z(u);return u}close(e){this.withNamespaceLock(()=>this.closeUnlocked(e))}closeUnlocked(e){let t=this.fdGet(e);if(!t)throw new z(Q);this.fdFree(e),this.inodeDropOpenRef(t.ino)}read(e,t){let n=this.fdGet(e);if(!n)throw new z(Q);let i=this.inodeOffset(n.ino);if((this.r32(i+N)&G)===U)throw new z(Ue);this.inodeReadLock(n.ino);try{let s=this.inodeReadData(n.ino,n.offset,t,t.length),a=256+e*24;return this.w64(a+De,n.offset+s),s}finally{this.inodeReadUnlock(n.ino)}}readAt(e,t,n){let i=this.fdGet(e);if(!i)throw new z(Q);let o=this.inodeOffset(i.ino);if((this.r32(o+N)&G)===U)throw new z(Ue);this.validateSeekPosition(n),this.inodeReadLock(i.ino);try{return this.inodeReadData(i.ino,n,t,t.length)}finally{this.inodeReadUnlock(i.ino)}}write(e,t){let n=this.fdGet(e);if(!n)throw new z(Q);if((n.flags&Gt)===qe)throw new z(Q);this.inodeWriteLock(n.ino);try{let o=n.offset;if(n.flags&zs){let c=this.inodeOffset(n.ino);o=this.r64(c+T)}if(!Number.isSafeInteger(o)||o<0)throw new z(Z);if(o>Qe||t.length>Qe-o)throw new z(bt);let s=this.inodeWriteData(n.ino,o,t,t.length);if(s<0)return s;let a=256+e*24;return this.w64(a+De,o+s),s}finally{this.inodeWriteUnlock(n.ino)}}writeAt(e,t,n){let i=this.fdGet(e);if(!i)throw new z(Q);if((i.flags&Gt)===qe)throw new z(Q);this.validateSeekPosition(n),this.inodeWriteLock(i.ino);try{if(n>Qe||t.length>Qe-n)throw new z(bt);return this.inodeWriteData(i.ino,n,t,t.length)}finally{this.inodeWriteUnlock(i.ino)}}lseek(e,t,n){let i=this.fdGet(e);if(!i)throw new z(Q);let o;if(n===ks)o=t;else if(n===xs)o=i.offset+t;else if(n===Is){let a=this.inodeOffset(i.ino);o=this.r64(a+T)+t}else throw new z(Z);this.validateSeekPosition(o);let s=256+e*24;return this.w64(s+De,o),o}ftruncate(e,t){let n=this.fdGet(e);if(!n)throw new z(Q);if((n.flags&Gt)===qe)throw new z(Q);this.validateFileSize(t),this.inodeWriteLock(n.ino);try{this.inodeTruncate(n.ino,t,!0)}finally{this.inodeWriteUnlock(n.ino)}}fstat(e){let t=this.fdGet(e);if(!t)throw new z(Q);this.inodeReadLock(t.ino);try{return this.buildStat(t.ino)}finally{this.inodeReadUnlock(t.ino)}}stat(e){return this.withNamespaceLock(()=>this.statUnlocked(e))}statUnlocked(e){let t=this.pathResolve(e,!0);if(t<0)throw new z(t);this.inodeReadLock(t);try{return this.buildStat(t)}finally{this.inodeReadUnlock(t)}}lstat(e){return this.withNamespaceLock(()=>this.lstatUnlocked(e))}lstatUnlocked(e){let t=this.pathResolve(e,!1);if(t<0)throw new z(t);this.inodeReadLock(t);try{return this.buildStat(t)}finally{this.inodeReadUnlock(t)}}unlink(e){return this.withNamespaceLock(()=>this.unlinkUnlocked(e))}unlinkUnlocked(e){let{parentIno:t,name:n}=this.pathResolveParent(e),i=ae.encode(n),o=e.length>1&&e.endsWith("/");this.inodeWriteLock(t);try{let s=this.dirLookup(t,i);if(s<0)throw new z(s);let a=this.inodeOffset(s),c=this.r32(a+N);if(o&&(c&G)!==U)throw new z(Ee);if((c&G)===U)throw new z(Ue);let l=this.namespaceEntryIdentity(s),u=this.dirRemoveEntry(t,i);if(u<0)throw new z(u);let f=!1;this.inodeWriteLock(s);try{f=this.inodeDropLinkRefLocked(s)}finally{this.inodeWriteUnlock(s)}return f&&this.inodeFree(s),l}finally{this.inodeWriteUnlock(t)}}rename(e,t){return this.withNamespaceLock(()=>this.renameUnlocked(e,t))}renameUnlocked(e,t){let{parentIno:n,name:i}=this.pathResolveParent(e),{parentIno:o,name:s}=this.pathResolveParent(t);if(Pn(i)||Pn(s))throw new z(Z);let a=ae.encode(i),c=ae.encode(s),l=e.length>1&&e.endsWith("/"),u=t.length>1&&t.endsWith("/"),f=Math.min(n,o),h=Math.max(n,o);this.inodeWriteLock(f),f!==h&&this.inodeWriteLock(h);try{let y=this.dirLookup(n,a);if(y<0)throw new z(y);let g=this.inodeOffset(y),m=this.r32(g+N)&G,p=this.namespaceEntryIdentity(y);if((l||u)&&m!==U)throw new z(Ee);if(m===U&&this.dirIsAncestor(y,o))throw new z(Z);let w=this.dirLookup(o,c),v=!1,S;if(w>=0){if(w===y)return{source:p,replaced:p};S=this.namespaceEntryIdentity(w);let E=this.inodeOffset(w),x=this.r32(E+N)&G;if(m===U&&x!==U)throw new z(Ee);if(m!==U&&x===U)throw new z(Ue);let I=!1,B=w===n||w===o;B||this.inodeWriteLock(w);try{if(x===U&&!this.dirIsEmpty(w))throw new z(Rn);let C=this.dirReplaceEntryIno(o,c,y);if(C<0)throw new z(C);I=x===U?this.inodeOrphanLocked(w):this.inodeDropLinkRefLocked(w)}finally{B||this.inodeWriteUnlock(w)}I&&this.inodeFree(w),v=x===U}else{let E=this.dirAddEntry(o,c,y);if(E<0)throw new z(E)}let b=this.dirRemoveEntry(n,a);if(b<0)throw new z(b);if(m===U){if(n!==o){let E=this.inodeOffset(n);this.w32(E+D,this.r32(E+D)-1);let k=this.inodeOffset(o);this.w32(k+D,this.r32(k+D)+1),this.inodeWriteLock(y);try{let x=this.dirReplaceEntryIno(y,Br,o);if(x<0)throw new z(x);this.w64(g+q,Date.now())}finally{this.inodeWriteUnlock(y)}}if(v){let E=this.inodeOffset(o);this.w32(E+D,this.r32(E+D)-1)}}else if(v){let E=this.inodeOffset(o);this.w32(E+D,this.r32(E+D)-1)}return{source:p,replaced:S}}finally{f!==h&&this.inodeWriteUnlock(h),this.inodeWriteUnlock(f)}}mkdir(e,t=493){this.withNamespaceLock(()=>this.mkdirUnlocked(e,t))}mkdirUnlocked(e,t=493){let{parentIno:n,name:i}=this.pathResolveParent(e),o=ae.encode(i);this.inodeWriteLock(n);try{if(this.dirLookup(n,o)>=0)throw new z(tt);let a=this.inodeAlloc();if(a<0)throw new z(Y);let c=this.inodeOffset(a);this.w32(c+N,U|t),this.w32(c+D,2),this.w64(c+T,0);let l=Date.now();this.w64(c+St,l),this.w64(c+ne,l),this.w64(c+q,l);let u=this.blockAllocWithGrow();if(u<0)throw this.inodeFree(a),new z(Y);this.w32(c+X,u);let f=u*4096,h=Oe(O+1),y=Oe(O+2);this.w32(f,a),this.view.setUint16(f+4,h,!0),this.view.setUint16(f+6,1,!0),this.u8[f+O]=46;let g=f+h;this.w32(g,n),this.view.setUint16(g+4,y,!0),this.view.setUint16(g+6,2,!0),this.u8[g+O]=46,this.u8[g+O+1]=46,this.w64(c+T,h+y);let d=this.dirAddEntry(n,o,a);if(d<0)throw this.blockFree(u),this.inodeFree(a),new z(d);let m=this.inodeOffset(n);this.w32(m+D,this.r32(m+D)+1)}finally{this.inodeWriteUnlock(n)}}rmdir(e){this.withNamespaceLock(()=>this.rmdirUnlocked(e))}rmdirUnlocked(e){let{parentIno:t,name:n}=this.pathResolveParent(e);if(Pn(n))throw new z(Z);let i=ae.encode(n);this.inodeWriteLock(t);try{let o=this.dirLookup(t,i);if(o<0)throw new z(o);let s=this.inodeOffset(o);if((this.r32(s+N)&G)!==U)throw new z(Ee);let c=!1;this.inodeWriteLock(o);try{if(!this.dirIsEmpty(o))throw new z(Rn);let u=this.dirRemoveEntry(t,i);if(u<0)throw new z(u);c=this.inodeOrphanLocked(o)}finally{this.inodeWriteUnlock(o)}c&&this.inodeFree(o);let l=this.inodeOffset(t);this.w32(l+D,this.r32(l+D)-1)}finally{this.inodeWriteUnlock(t)}}symlink(e,t){this.withNamespaceLock(()=>this.symlinkUnlocked(e,t))}symlinkUnlocked(e,t){let{parentIno:n,name:i}=this.pathResolveParent(t),o=ae.encode(i),s=ae.encode(e);this.inodeWriteLock(n);try{if(this.dirLookup(n,o)>=0)throw new z(tt);let c=this.inodeAlloc();if(c<0)throw new z(Y);let l=this.inodeOffset(c);if(this.w32(l+N,vt|511),this.w32(l+D,1),s.length<=40)this.u8.set(s,l+X),this.w64(l+T,s.length);else{this.w64(l+T,0);let f=this.inodeWriteData(c,0,s,s.length);if(f!==s.length)throw f>0&&this.inodeTruncate(c,0),this.inodeFree(c),new z(f<0?f:Y)}let u=this.dirAddEntry(n,o,c);if(u<0)throw s.length<=40?(this.u8.fill(0,l+X,l+X+40),this.w64(l+T,0)):this.inodeTruncate(c,0),this.inodeFree(c),new z(u)}finally{this.inodeWriteUnlock(n)}}chmod(e,t){this.withNamespaceLock(()=>this.chmodUnlocked(e,t))}chmodUnlocked(e,t){let n=this.pathResolve(e,!0);if(n<0)throw new z(n);this.inodeWriteLock(n);try{let i=this.inodeOffset(n),o=this.r32(i+N);this.w32(i+N,o&G|t&4095),this.w64(i+q,Date.now())}finally{this.inodeWriteUnlock(n)}}fchmod(e,t){let n=this.fdGet(e);if(!n)throw new z(Q);this.inodeWriteLock(n.ino);try{let i=this.inodeOffset(n.ino),o=this.r32(i+N);this.w32(i+N,o&G|t&4095),this.w64(i+q,Date.now())}finally{this.inodeWriteUnlock(n.ino)}}chown(e,t,n){this.withNamespaceLock(()=>this.chownUnlocked(e,t,n))}chownUnlocked(e,t,n){let i=this.pathResolve(e,!0);if(i<0)throw new z(i);this.inodeWriteLock(i);try{this.chownInodeUnlocked(i,t,n)}finally{this.inodeWriteUnlock(i)}}fchown(e,t,n){let i=this.fdGet(e);if(!i)throw new z(Q);this.inodeWriteLock(i.ino);try{this.chownInodeUnlocked(i.ino,t,n)}finally{this.inodeWriteUnlock(i.ino)}}lchown(e,t,n){this.withNamespaceLock(()=>this.lchownUnlocked(e,t,n))}lchownUnlocked(e,t,n){let i=this.pathResolve(e,!1);if(i<0)throw new z(i);this.inodeWriteLock(i);try{this.chownInodeUnlocked(i,t,n)}finally{this.inodeWriteUnlock(i)}}chownInodeUnlocked(e,t,n){let i=this.inodeOffset(e);t!==br&&this.w32(i+Lr,t),n!==br&&this.w32(i+Ar,n);let o=this.r32(i+N);(o&G)===wt&&(o&Ss)!==0&&this.w32(i+N,o&~(vs|Es)),this.w64(i+q,Date.now())}utimens(e,t,n,i,o){this.withNamespaceLock(()=>this.utimensUnlocked(e,t,n,i,o))}utimensUnlocked(e,t,n,i,o){let s=this.pathResolve(e,!0);if(s<0)throw new z(s);this.inodeWriteLock(s);try{let a=this.inodeOffset(s),c=1073741823,l=1073741822,u=Date.now();if(n!==l){let f=n===c?u:t*1e3+Math.floor(n/1e6);this.w64(a+St,f)}if(o!==l){let f=o===c?u:i*1e3+Math.floor(o/1e6);this.w64(a+ne,f)}this.w64(a+q,u)}finally{this.inodeWriteUnlock(s)}}link(e,t){return this.withNamespaceLock(()=>this.linkUnlocked(e,t))}linkUnlocked(e,t){let n=this.pathResolve(e,!1);if(n<0)throw new z(n);let i=this.inodeOffset(n);if((this.r32(i+N)&G)===U)throw new z(Ls);let{parentIno:s,name:a}=this.pathResolveParent(t),c=ae.encode(a);this.inodeWriteLock(s);try{if(this.dirLookup(s,c)>=0)throw new z(tt);let u=this.dirAddEntry(s,c,n);if(u<0)throw new z(u);this.inodeWriteLock(n);try{let f=this.r32(i+D);this.w32(i+D,f+1),this.w64(i+q,Date.now())}finally{this.inodeWriteUnlock(n)}return{...this.namespaceEntryIdentity(n),linkCount:this.r32(i+D)}}finally{this.inodeWriteUnlock(s)}}readlink(e){return this.withNamespaceLock(()=>this.readlinkUnlocked(e))}readlinkUnlocked(e){let t=this.pathResolve(e,!1);if(t<0)throw new z(t);let n=this.inodeOffset(t);if((this.r32(n+N)&G)!==vt)throw new z(Z);let o=this.r64(n+T);if(o<=40)return et(this.u8.subarray(n+X,n+X+o));this.inodeReadLock(t);try{let s=new Uint8Array(o);return this.inodeReadData(t,0,s,o),xt.decode(s)}finally{this.inodeReadUnlock(t)}}opendir(e){return this.withNamespaceLock(()=>this.opendirUnlocked(e))}opendirUnlocked(e){let t=this.pathResolve(e,!0);if(t<0)throw new z(t);let n=this.inodeOffset(t);if((this.r32(n+N)&G)!==U)throw new z(Ee);let o=this.fdAlloc(t,qe,!0);if(o<0)throw new z(o);return o}readdirEntry(e){return this.withNamespaceLock(()=>this.readdirEntryUnlocked(e))}readdirEntryUnlocked(e){let t=this.fdGet(e);if(!t||!t.isDir)throw new z(Q);let n=this.inodeOffset(t.ino),i=this.r64(n+T);for(;t.offset=this.r32(_e))throw new z($);let d=this.r32(Fe)*4096;if((this.r32(d+(u>>5)*4)&1<<(u&31))===0)throw new z($);let p=et(this.u8.subarray(l+O,l+O+h)),w=this.buildStat(u);return this.w64(g+De,y),t.offset=y,{name:p,stat:w}}return null}closedir(e){this.close(e)}readdir(e){let t=this.opendir(e),n=[];try{let i;for(;(i=this.readdirEntry(t))!==null;)i.name!=="."&&i.name!==".."&&n.push(i.name)}finally{this.closedir(t)}return n}writeFile(e,t){let n=typeof t=="string"?ae.encode(t):t,i=this.open(e,kr|kt|It);try{this.write(i,n)}finally{this.close(i)}}readFile(e){let t=this.open(e,qe);try{let n=this.fstat(t),i=new Uint8Array(n.size);return this.read(t,i),i}finally{this.close(t)}}readFileText(e){return xt.decode(this.readFile(e))}};function $r(r,e){let t=new Map,n=new Map;for(let s of r){if(t.has(s.path))throw new Error(`${e} duplicates path ${s.path}`);if(t.set(s.path,s),s.type==="file"){if(!s.inodeGroup)throw new Error(`${e} file ${s.path} has no inode group`);if(n.has(s.inodeGroup))throw new Error(`${e} inode group ${s.inodeGroup} has multiple files`);n.set(s.inodeGroup,s)}}let i=new Set,o=new Map;for(let s of r){if(s.type!=="hardlink"||o.has(s.path))continue;let a=[],c=s,l;for(;c.type==="hardlink";){let f=o.get(c.path);if(f){l=f;break}if(i.has(c.path))throw new Error(`${e} hardlink cycle reaches ${c.path}`);if(i.add(c.path),a.push(c),!c.target)throw new Error(`${e} hardlink ${c.path} has no target`);let h=t.get(c.target);if(!h)throw new Error(`${e} hardlink ${c.path} target ${c.target} is missing`);if(h.type!=="file"&&h.type!=="hardlink"||!c.inodeGroup||h.inodeGroup!==c.inodeGroup||h.size!==c.size||h.mode!==c.mode)throw new Error(`${e} hardlink ${c.path} has an invalid target`);c=h}l??=c.type==="file"?c:void 0;let u=n.get(s.inodeGroup??"");if(!l||l!==u)throw new Error(`${e} hardlink ${s.path} does not resolve to its inode`);for(let f=a.length-1;f>=0;f-=1){let h=a[f];if(n.get(h.inodeGroup??"")!==l)throw new Error(`${e} hardlink ${h.path} does not resolve to its inode`);i.delete(h.path),o.set(h.path,l)}}return{canonicalByGroup:n,canonicalTargetByPath:o}}var ce={maxArchiveBytes:268435456,maxExpandedBytes:268435456,maxPayloadBytes:268435456,maxEntries:1e5,maxPathBytes:4096,maxSymlinkTargetBytes:65536,maxStringBytes:8192,maxTransportsPerTree:8,maxActivationCapabilities:32,maxActivationRoots:64,maxActivationCapabilityBytes:255},me={maxArchiveBytes:512*1024*1024,maxExpandedBytes:512*1024*1024,maxPayloadBytes:512*1024*1024,maxEntries:1e5,maxGroups:512};function Fr(r,e="Deferred tree collection"){for(let[t,n]of Object.entries(r))if(!Number.isSafeInteger(n)||n<0)throw new Error(`${e} ${t} usage is invalid`);if(r.groups>me.maxGroups)throw new Error(`${e} exceeds the ${me.maxGroups}-group cap`);if(r.archiveBytes>me.maxArchiveBytes)throw new Error(`${e} exceeds the archive-byte cap`);if(r.expandedBytes>me.maxExpandedBytes)throw new Error(`${e} exceeds the expansion cap`);if(r.payloadBytes>me.maxPayloadBytes)throw new Error(`${e} exceeds the payload-byte cap`);if(r.entries>me.maxEntries)throw new Error(`${e} exceeds the entry-count cap`)}var nt="/home/linuxbrew/.linuxbrew",Gr=[["@@HOMEBREW_PREFIX@@",nt],["@@HOMEBREW_CELLAR@@",`${nt}/Cellar`],["@@HOMEBREW_REPOSITORY@@",nt],["@@HOMEBREW_LIBRARY@@",`${nt}/Library`],["@@HOMEBREW_PERL@@",`${nt}/opt/perl/bin/perl`]],Cn="@@HOMEBREW_JAVA@@",Bs=/^openjdk(?:@\d+(?:\.\d+)*)?/,rt=new TextEncoder,Cs=[...Gr.map(([r])=>r),Cn].map(r=>({placeholder:r,bytes:rt.encode(r)}));function Kr(r){let e;try{e=JSON.parse(new TextDecoder("utf-8",{fatal:!0}).decode(r))}catch(a){throw new Error("INSTALL_RECEIPT.json is not valid UTF-8 JSON: "+Fs(a))}if(typeof e!="object"||e===null||Array.isArray(e))throw new Error("INSTALL_RECEIPT.json must contain an object");let t=e,n=t.changed_files;if(n!=null&&!Array.isArray(n))throw new Error("INSTALL_RECEIPT.json changed_files must be an array or null when present");let i=Array.isArray(n)?n:[];if(i.length>1e5)throw new Error(`INSTALL_RECEIPT.json declares ${i.length} changed files, limit 100000`);let o=[],s=new Set;for(let[a,c]of i.entries()){if(typeof c!="string")throw new Error(`INSTALL_RECEIPT.json changed_files[${a}] is not a string`);if(Ms(c,"Homebrew changed file"),s.has(c))throw new Error(`INSTALL_RECEIPT.json repeats changed file ${c}`);s.add(c),o.push(c)}return{changedFiles:o,runtimeDependencies:t.runtime_dependencies}}function Zr(r,e,t){let n=r;for(let[s,a]of Gr)n=Ur(n,rt.encode(s),rt.encode(a));let i=rt.encode(Cn);if(Dr(n,i)){let s=Ns(e.runtimeDependencies);if(s===void 0)throw new Error(`Homebrew changed file ${t} uses ${Cn} without exactly one OpenJDK runtime dependency`);n=Ur(n,i,rt.encode(s))}let o=Cs.find(({bytes:s})=>Dr(n,s));if(o!==void 0)throw new Error(`Homebrew changed file ${t} retains ${o.placeholder}`);return n}function Ns(r){if(!Array.isArray(r))return;let e=[];for(let n of r){if(typeof n!="object"||n===null||Array.isArray(n))continue;let i=n,o=typeof i.full_name=="string"?i.full_name.split("/").at(-1):typeof i.name=="string"?i.name.split("/").at(-1):void 0,s=o===void 0?null:Bs.exec(o);o!==void 0&&s?.[0]===o&&e.push(o)}let t=[...new Set(e)];return t.length===1?`${nt}/opt/${t[0]}/libexec`:void 0}function Ms(r,e){if(r.length===0||r.startsWith("/")||r.includes("\\")||r.includes("\0")||$s(r)||rt.encode(r).byteLength>4096||r.split("/").some(t=>t===""||t==="."||t===".."))throw new Error(`${e} has an unsafe path segment: ${r}`)}function $s(r){for(let e=0;e57343)){if(t<=56319&&e+1=56320&&r.charCodeAt(e+1)<=57343){e+=1;continue}return!0}}return!1}function Dr(r,e){if(e.byteLength===0||e.byteLength>r.byteLength)return!1;e:for(let t=0;t<=r.byteLength-e.byteLength;t+=1){for(let n=0;ncn||r.includes("\0")||r.includes("\\"))throw new Error(`Lazy archive mount prefix must be an absolute POSIX path: ${JSON.stringify(r)}`);let e=r.replace(/\/+$/,"");if(e==="")return"/";if(e.slice(1).split("/").some(n=>n===""||n==="."||n===".."))throw new Error(`Lazy archive mount prefix is not canonical: ${JSON.stringify(r)}`);return e}function Uo(r,e,t,n){let i=ln(t),o=new Map,s=e.map(a=>{let c=a.fileName,l=`Lazy archive ${JSON.stringify(r)} member ${JSON.stringify(c)}`;if(c.length===0)throw new Error(`${l} has an empty path`);if(c.includes("\0"))throw new Error(`${l} contains a NUL byte`);if(c.includes("\\"))throw new Error(`${l} contains a backslash`);if(c.startsWith("/")||/^[A-Za-z]:\//.test(c))throw new Error(`${l} must be relative, not absolute`);if(a.isDirectory&&a.isSymlink)throw new Error(`${l} has conflicting directory and symlink types`);if(a.isDirectory!==c.endsWith("/"))throw new Error(`${l} has inconsistent directory metadata`);let u=a.isDirectory?c.slice(0,-1):c,f=u.split("/");if(u.length===0||f.some(h=>h===""||h==="."||h===".."))throw new Error(`${l} is not a canonical relative POSIX path`);if(o.has(u))throw new Error(`${l} collides with another member at ${JSON.stringify(u)}`);if(a.isSymlink&&!n?.has(c))throw new Error(`Lazy archive symlink target was not provided: ${c}`);return o.set(u,a),{entry:a,archivePath:u,vfsPath:i==="/"?`/${u}`:`${i}/${u}`}});for(let{archivePath:a}of s){let c=a.split("/");for(let l=1;lct)throw new Error(`VFS image metadata exceeds ${ct} bytes`);let e;try{e=JSON.parse(new TextDecoder().decode(r))}catch(t){let n=t instanceof Error?t.message:String(t);throw new Error(`Invalid VFS image metadata JSON: ${n}`)}return sr(e)}function Zo(r){if(r===null)return new Uint8Array(0);let e=sr(r),t=new TextEncoder().encode(JSON.stringify(e));if(t.byteLength>ct)throw new Error(`VFS image metadata exceeds ${ct} bytes`);return t}function Wo(r){return r.byteLength>=Ot.length&&r[0]===Ot[0]&&r[1]===Ot[1]&&r[2]===Ot[2]&&r[3]===Ot[3]?Yo(r):r}function Yt(r){let e=Wo(r);if(e.byteLengthQt)throw new Error(`VFS image lazy metadata exceeds ${Qt} bytes`);if(r.byteLengthen)throw new Error(`VFS image lazy archive metadata exceeds ${en} bytes`);if(r.byteLength=0?n:void 0}function qo(r,e){if(r.length===1)return r[0];let t=new Uint8Array(e),n=0;for(let i of r)t.set(i,n),n+=i.byteLength;return t}function Tt(r){if(r===void 0)return;if(typeof r!="object"||r===null||Array.isArray(r))throw new Error("Lazy archive integrity must be an object");let e=r;if(Object.keys(e).length!==2||!("sha256"in e)||!("bytes"in e))throw new Error("Lazy archive integrity has unexpected fields");if(typeof e.sha256!="string"||!Do.test(e.sha256))throw new Error("Lazy archive integrity has an invalid SHA-256 digest");if(!Number.isSafeInteger(e.bytes)||Number(e.bytes)<=0||Number(e.bytes)>pi)throw new Error(`Lazy archive integrity byte count must be between 1 and ${pi}`);return{sha256:e.sha256,bytes:Number(e.bytes)}}function lt(r,e,t){if(typeof r!="object"||r===null||Array.isArray(r))throw new Error(`${t} must be an object`);let n=r;if(Object.keys(n).length!==e.length||e.some(o=>!Object.prototype.hasOwnProperty.call(n,o)))throw new Error(`${t} has unexpected or missing fields`);return n}function nr(r,e,t,n){if(typeof r!="object"||r===null||Array.isArray(r))throw new Error(`${n} must be an object`);let i=r,o=new Set(e);if(Object.keys(i).some(s=>!o.has(s))||t.some(s=>!Object.prototype.hasOwnProperty.call(i,s)))throw new Error(`${n} has unexpected or missing fields`);return i}function ke(r,e,t,n){if(!Array.isArray(r)||r.lengthn)throw new Error(`${e} must contain ${t} to ${n} items`);return r}function Te(r,e,t){if(typeof r!="string"||r.length===0||r.includes("\0")||new TextEncoder().encode(r).byteLength>t)throw new Error(`${e} is invalid or exceeds ${t} bytes`);return r}function ie(r,e,t,n){if(!Number.isSafeInteger(r)||Number(r)n)throw new Error(`${e} must be an integer between ${t} and ${n}`);return Number(r)}function sn(r,e=1){let t=r,n=typeof t=="object"&&t!==null&&!Array.isArray(t)&&t.source!==void 0,i=lt(r,["decoder","mediaType","sha256","bytes","expandedBytes","sourceEntryCount","transports",...n?["source"]:[]],"Lazy tree content"),o=i.decoder==="zip-v1"?"application/zip":i.decoder==="homebrew-bottle-tar-gzip-v1"?"application/vnd.oci.image.layer.v1.tar+gzip":null;if(o===null||i.mediaType!==o)throw new Error("Lazy tree decoder and media type are inconsistent");let s=Tt({sha256:i.sha256,bytes:i.bytes});if(!s)throw new Error("Lazy tree integrity is required");let a=ke(i.transports,"Lazy tree transports",e,ce.maxTransportsPerTree).map((f,h)=>Te(f,`Lazy tree transport ${h}`,ir));if(new Set(a).size!==a.length)throw new Error("Lazy tree transports contain duplicates");let c=ie(i.expandedBytes,"Lazy tree expanded byte count",0,Co),l=ie(i.sourceEntryCount,"Lazy tree source entry count",1,ft),u=n?jo(i.source,i.decoder):void 0;if(u!==void 0&&u.entries.length!==l)throw new Error("Lazy tree source inventory count differs from its content");return{decoder:i.decoder,mediaType:o,sha256:s.sha256,bytes:s.bytes,expandedBytes:c,sourceEntryCount:l,transports:a,...u===void 0?{}:{source:u}}}function bi(r){let e={groups:r.length,archiveBytes:0,expandedBytes:0,payloadBytes:0,entries:0};for(let t of r)t.content===void 0||t.inventory===void 0||(e.archiveBytes+=t.content.bytes,e.expandedBytes+=t.content.expandedBytes,e.payloadBytes+=t.inventory.filter(n=>n.type==="file").reduce((n,i)=>n+i.size,0),e.entries+=t.inventory.length+(t.content.source?.entries.length??0));return e}function rr(r){Fr(r,"Serialized lazy tree collection")}function Ei(r){rr(bi(r))}function jo(r,e){if(e!=="homebrew-bottle-tar-gzip-v1")throw new Error("Lazy tree source inventory is valid only for original bottles");let t=lt(r,["schema","kind","entries"],"Lazy tree source inventory");if(t.schema!==1||t.kind!=="homebrew-bottle-tar-gzip-v1")throw new Error("Lazy tree source inventory has an unsupported identity");let n=new Map,i=ke(t.entries,"Lazy tree source entries",1,ft).map((s,a)=>{let c=s,l=typeof c=="object"&&c!==null&&!Array.isArray(c)?c.type:void 0,u=l==="directory"||l==="file"?["sourcePath","type","mode","size"]:l==="symlink"||l==="hardlink"?["sourcePath","type","mode","size","target"]:null;if(u===null)throw new Error(`Lazy tree source entry ${a} has invalid type`);let f=lt(s,u,`Lazy tree source entry ${a}`),h=fe(f.sourcePath,!1,`Lazy tree source entry ${a} path`);if(n.has(h))throw new Error(`Lazy tree source inventory duplicates ${h}`);let y=ie(f.mode,`Lazy tree source entry ${h} mode`,0,4095),g=ie(f.size,`Lazy tree source entry ${h} size`,0,tn),d;if((l==="directory"||l==="symlink"||l==="hardlink")&&g!==0)throw new Error(`Lazy tree source ${h} has payload for ${String(l)}`);l==="symlink"?d=Te(f.target,`Lazy tree source symlink ${h} target`,zi):l==="hardlink"&&(d=fe(f.target,!1,`Lazy tree source hardlink ${h} target`));let m={sourcePath:h,type:l,mode:y,size:g,...d===void 0?{}:{target:d}};return n.set(h,m),m}),o=i.map(s=>s.sourcePath);if(o.some((s,a)=>a>0&&o[a-1]>=s))throw new Error("Lazy tree source inventory is not in canonical path order");return{schema:1,kind:"homebrew-bottle-tar-gzip-v1",entries:i}}function ki(r){let e=new Map(r.map(n=>[n.sourcePath,n])),t=new Map;for(let n of r){if(n.type!=="hardlink"||t.has(n.sourcePath))continue;let i=[],o=new Set,s=n,a;for(;s.type==="hardlink"&&(a=t.get(s.sourcePath),a===void 0);){if(o.has(s.sourcePath))throw new Error(`Lazy tree source hardlink cycle includes ${s.sourcePath}`);o.add(s.sourcePath),i.push(s);let c=e.get(s.target);if(c===void 0)throw new Error(`Lazy tree source hardlink ${s.sourcePath} target is absent`);if(c.type!=="file"&&c.type!=="hardlink")throw new Error(`Lazy tree source hardlink ${s.sourcePath} target is not regular`);s=c}a===void 0&&(a=s);for(let c of i)t.set(c.sourcePath,a)}return t}function fe(r,e,t,n=!1){if(typeof r!="string"||r.length===0||new TextEncoder().encode(r).byteLength>cn||r.includes("\0")||r.includes("\\")||r.startsWith("/")!==e)throw new Error(`${t} is not a canonical ${e?"absolute":"relative"} path`);if(n&&e&&r==="/")return r;if(r.slice(e?1:0).split("/").some(o=>o===""||o==="."||o===".."))throw new Error(`${t} has an unsafe path segment`);return r}function xi(r,e,t,n,i=1){let o=sn(r,i),s=ln(t),a=lt(n,["mode","capabilities","roots"],"Lazy tree activation");if(a.mode!=="boot-prefetch"&&a.mode!=="first-use")throw new Error("Lazy tree activation mode is invalid");let c=ke(a.capabilities,"Lazy tree activation capabilities",1,$o).map((S,b)=>{let E=Te(S,`Lazy tree activation capability ${b}`,ce.maxActivationCapabilityBytes);if(!/^[a-z0-9][a-z0-9:._-]*$/.test(E))throw new Error(`Lazy tree activation capability ${b} is invalid`);return E}),l=ke(a.roots,"Lazy tree activation roots",1,Fo).map((S,b)=>fe(S,!0,`Lazy tree activation root ${b}`,!0));if(new Set(c).size!==c.length||new Set(l).size!==l.length)throw new Error("Lazy tree activation contains duplicates");let u={mode:a.mode,capabilities:c,roots:l},f=ke(e,"Lazy tree inventory",1,ft),h=[],y=new Map,g=new Map,d=o.source===void 0?void 0:new Map(o.source.entries.map(S=>[S.sourcePath,S])),m=o.source===void 0?void 0:ki(o.source.entries),p=0;for(let[S,b]of f.entries()){if(typeof b!="object"||b===null||Array.isArray(b))throw new Error(`Lazy tree entry ${S} must be an object`);let E=b.type,k=E==="directory"?["vfsPath","sourcePath","type","mode","size"]:E==="file"?["vfsPath","sourcePath","type","mode","size","inodeGroup"]:E==="symlink"?["vfsPath","sourcePath","type","mode","size","target"]:E==="hardlink"?["vfsPath","sourcePath","type","mode","size","target","inodeGroup"]:null;if(!k)throw new Error(`Lazy tree entry ${S} has an invalid type`);let x=lt(b,[...k,...d===void 0?[]:["materialization"]],`Lazy tree entry ${S}`),I=fe(x.vfsPath,!0,`Lazy tree entry ${S} VFS path`),B=fe(x.sourcePath,!1,`Lazy tree entry ${S} source path`),C=d===void 0?void 0:x.materialization;if(d!==void 0&&C!=="archive"&&C!=="archive-homebrew-relocate"&&C!=="archive-copy"&&C!=="archive-copy-mode"&&C!=="descriptor")throw new Error(`Lazy tree entry ${I} has invalid materialization provenance`);if(s!=="/"&&I!==s&&!I.startsWith(`${s}/`))throw new Error(`Lazy tree entry ${I} escapes its mount prefix`);if(y.has(I))throw new Error(`Lazy tree duplicates VFS path ${I}`);let ee=ie(x.mode,`Lazy tree entry ${I} mode`,0,4095),j=ie(x.size,`Lazy tree entry ${I} size`,0,tn),R,P;if(E==="directory"){if(j!==0)throw new Error(`Lazy tree directory ${I} has nonzero size`)}else if(E==="symlink"){if(R=Te(x.target,`Lazy tree symlink ${I} target`,zi),new TextEncoder().encode(R).byteLength!==j)throw new Error(`Lazy tree symlink ${I} size differs from its target`)}else P=Te(x.inodeGroup,`Lazy tree entry ${I} inode group`,cn),E==="hardlink"&&(R=fe(x.target,!0,`Lazy tree hardlink ${I} target`));if(E!=="hardlink"&&(p+=j,p>tn))throw new Error("Lazy tree inventory exceeds the expansion limit");let A={vfsPath:I,sourcePath:B,...C===void 0?{}:{materialization:C},type:E,mode:ee,size:j,...R===void 0?{}:{target:R},...P===void 0?{}:{inodeGroup:P}};if(d===void 0){let W=g.get(B);if(W){if(o.decoder!=="zip-v1"||A.type!=="hardlink"||W.inodeGroup!==A.inodeGroup)throw new Error(`Lazy tree duplicates source path ${B}`)}else{if(o.decoder==="zip-v1"&&A.type==="hardlink")throw new Error(`Lazy ZIP hardlink ${I} does not reuse a canonical source path`);g.set(B,A)}}else if(A.materialization==="descriptor"){if(A.type!=="directory"&&A.type!=="symlink")throw new Error(`Lazy tree descriptor entry ${I} is not structural`);if(d.has(B))throw new Error(`Lazy tree descriptor entry ${I} impersonates a source member`)}else{let W=d.get(B);if(W===void 0)throw new Error(`Lazy tree entry ${I} names absent source ${B}`);if(A.materialization==="archive-copy"||A.materialization==="archive-copy-mode"){if(A.type!=="file"||W.type!=="file"||A.materialization==="archive-copy"&&A.mode!==W.mode)throw new Error(`Lazy tree archive copy ${I} differs from its source`)}else if(A.materialization==="archive-homebrew-relocate"){if(A.type!=="file"&&A.type!=="hardlink"||W.type!==A.type||A.type==="file"&&W.mode!==A.mode)throw new Error(`Lazy tree receipt-relocated entry ${I} differs from its source`)}else if(W.type!==A.type||A.type==="symlink"&&W.target!==A.target||A.type!=="hardlink"&&W.mode!==A.mode)throw new Error(`Lazy tree archive entry ${I} differs from its source`)}h.push(A),y.set(I,A)}for(let S of h){let b=S.vfsPath.split("/").filter(Boolean);for(let E=1;E({path:S.vfsPath,type:S.type,mode:S.mode,size:S.size,target:S.target,inodeGroup:S.inodeGroup})),"Lazy tree");if(d!==void 0){let S=new Set;for(let b of h){if(b.materialization!=="archive-homebrew-relocate")continue;let E=d.get(b.sourcePath),k=E.type==="file"?E:m.get(E.sourcePath);if(k?.type!=="file")throw new Error(`Lazy tree receipt-relocated entry ${b.vfsPath} is not regular`);S.add(k.sourcePath)}for(let b of h){if(b.materialization==="descriptor"||b.type!=="file"&&b.type!=="hardlink")continue;let E=d.get(b.sourcePath),k=E.type==="file"?E:m.get(E.sourcePath);if(k?.type!=="file"||!S.has(k.sourcePath)&&b.size!==k.size)throw new Error(`Lazy tree archive entry ${b.vfsPath} differs from its source`)}for(let b of h){if(b.type!=="hardlink"||b.materialization!=="archive"&&b.materialization!=="archive-homebrew-relocate")continue;let E=d.get(b.sourcePath),k=y.get(b.target),x=m.get(E.sourcePath);if(E.target!==k?.sourcePath||x?.type!=="file"||x.mode!==b.mode||k?.mode!==b.mode)throw new Error(`Lazy tree hardlink ${b.vfsPath} differs from its source`)}}if(o.sourceEntryCount!==(d===void 0?g.size:d.size))throw new Error("Lazy tree source entry count differs from its inventory");if(o.source===void 0&&o.expandedBytesb.vfsPath===S||b.vfsPath.startsWith(`${S}/`)))throw new Error(`Lazy tree activation root ${S} is not owned by its inventory`);let v=new Map;for(let S of h)S.type==="file"&&v.set(S.inodeGroup,S);if(v.size!==w.canonicalByGroup.size)throw new Error("Lazy tree regular inode inventory is inconsistent");return{content:o,entries:h,mountPrefix:s,activation:u,canonicalByGroup:v}}function on(r){return JSON.stringify([r.sourcePath,r.type,r.inodeGroup,r.target])}function Si(r,e){let t=nr(r,["kind","content","url","mountPrefix","integrity","materialized","entries"],["url","mountPrefix","materialized","entries"],"Serialized legacy lazy archive");if(t.kind===void 0){if(!e)throw new Error("Serialized lazy archive is missing its kind discriminator")}else if(t.kind!==nn)throw new Error("Serialized legacy lazy archive has an unsupported kind");let n=Te(t.url,"Serialized legacy lazy archive URL",ir),i=ln(t.mountPrefix),o=Tt(t.integrity);if(t.content!==void 0){if(!e||t.kind!==void 0)throw new Error("Typed legacy lazy archives cannot carry generic content");let c=sn(t.content);if(c.decoder!=="zip-v1"||c.transports.length!==1||c.transports[0]!==n||!o||c.sha256!==o.sha256||c.bytes!==o.bytes)throw new Error("Untagged legacy ZIP content identity is inconsistent")}if(t.materialized!==!1)throw new Error("Serialized legacy lazy archive must describe pending content");let s=new Set,a=ke(t.entries,"Serialized legacy lazy archive entries",1,ft).map((c,l)=>{let u=nr(c,["vfsPath","ino","generation","dataSequence","size","isSymlink","deleted","materialized","archivePath","sourcePath","type","inodeGroup","target"],["vfsPath","ino","size","isSymlink","deleted"],`Serialized legacy lazy archive entry ${l}`),f=fe(u.vfsPath,!0,`Serialized legacy lazy archive entry ${l} VFS path`);if(s.has(f))throw new Error(`Serialized legacy lazy archive duplicates path ${f}`);s.add(f);let h=ie(u.ino,`Serialized legacy lazy archive entry ${f} inode`,1,Number.MAX_SAFE_INTEGER),y=u.generation===void 0?void 0:ie(u.generation,`Serialized legacy lazy archive entry ${f} generation`,0,Number.MAX_SAFE_INTEGER),g=u.dataSequence===void 0?void 0:ie(u.dataSequence,`Serialized legacy lazy archive entry ${f} data sequence`,0,Number.MAX_SAFE_INTEGER),d=ie(u.size,`Serialized legacy lazy archive entry ${f} size`,0,tn);if(u.isSymlink!==!1||u.deleted!==!1||u.materialized!==void 0&&u.materialized!==!1)throw new Error(`Serialized legacy lazy archive entry ${f} is not pending`);if(u.type!==void 0&&u.type!=="file")throw new Error(`Serialized legacy lazy archive entry ${f} has an invalid type`);let m=u.archivePath===void 0?void 0:fe(u.archivePath,!1,`Serialized legacy lazy archive entry ${f} archive path`),p=u.sourcePath===void 0?void 0:fe(u.sourcePath,!1,`Serialized legacy lazy archive entry ${f} source path`),w=u.inodeGroup===void 0?void 0:Te(u.inodeGroup,`Serialized legacy lazy archive entry ${f} inode group`,cn);if(u.target!==void 0)throw new Error(`Serialized legacy lazy archive entry ${f} has a link target`);return{vfsPath:f,ino:h,...y===void 0?{}:{generation:y},...g===void 0?{}:{dataSequence:g},size:d,isSymlink:!1,deleted:!1,materialized:!1,...m===void 0?{}:{archivePath:m},...p===void 0?{}:{sourcePath:p},type:"file",...w===void 0?{}:{inodeGroup:w}}});return{kind:nn,url:n,mountPrefix:i,...o===void 0?{}:{integrity:o},materialized:!1,entries:a}}function Xo(r,e){let t=lt(r,["kind","content","inventory","activation","url","mountPrefix","integrity","materialized","entries"],"Serialized lazy tree");if(t.kind!==e)throw new Error("Serialized lazy tree has an unsupported kind");let n=xi(t.content,t.inventory,t.mountPrefix,t.activation);if(e===rn!=(n.content.source===void 0))throw new Error(e===rn?"Serialized deferred-tree-v1 cannot contain original-bottle source metadata":"Serialized deferred-tree-v2 requires original-bottle source metadata");let i=Te(t.url,"Serialized lazy tree URL",ir);if(i!==n.content.transports[0])throw new Error("Serialized lazy tree URL differs from its primary transport");let o=Tt(t.integrity);if(!o||o.sha256!==n.content.sha256||o.bytes!==n.content.bytes)throw new Error("Serialized lazy tree integrity differs from its content");if(t.materialized!==!1)throw new Error("Serialized lazy tree must describe pending content");let s=new Map(n.entries.map(f=>[f.vfsPath,f])),a=new Map(n.entries.map(f=>[on(f),f])),c=ke(t.entries,"Serialized lazy tree entries",0,ft),l=new Set,u=c.map((f,h)=>{let y=nr(f,["vfsPath","ino","generation","dataSequence","size","isSymlink","deleted","materialized","archivePath","sourcePath","type","inodeGroup","target"],["vfsPath","ino","generation","dataSequence","size","isSymlink","deleted","materialized","archivePath","sourcePath","type","inodeGroup"],`Serialized lazy tree entry ${h}`),g=fe(y.vfsPath,!0,`Serialized lazy tree entry ${h} VFS path`);if(l.has(g))throw new Error(`Serialized lazy tree duplicates pending path ${g}`);l.add(g);let d=fe(y.sourcePath,!1,`Serialized lazy tree entry ${h} source path`),m=fe(y.archivePath,!1,`Serialized lazy tree entry ${h} archive path`),p=s.get(g),w=a.get(on({sourcePath:d,type:typeof y.type=="string"?y.type:void 0,inodeGroup:typeof y.inodeGroup=="string"?y.inodeGroup:void 0,target:typeof y.target=="string"?y.target:void 0}))??p;if(!w||w.type!=="file"&&w.type!=="hardlink"||p?.inodeGroup!==void 0&&p.inodeGroup!==w.inodeGroup)throw new Error(`Serialized lazy tree entry ${g} is absent from its inventory`);let v=n.canonicalByGroup.get(w.inodeGroup);if(y.type!==w.type||y.inodeGroup!==w.inodeGroup||y.size!==w.size||m!==v?.sourcePath||y.target!==w.target||y.isSymlink!==!1||y.deleted!==!1||y.materialized!==!1)throw new Error(`Serialized lazy tree entry ${g} disagrees with its inventory`);let S=ie(y.ino,`Serialized lazy tree entry ${g} inode`,1,Number.MAX_SAFE_INTEGER),b=ie(y.generation,`Serialized lazy tree entry ${g} generation`,0,Number.MAX_SAFE_INTEGER),E=ie(y.dataSequence,`Serialized lazy tree entry ${g} data sequence`,0,Number.MAX_SAFE_INTEGER);return{vfsPath:g,ino:S,generation:b,dataSequence:E,size:w.size,isSymlink:!1,deleted:!1,materialized:!1,archivePath:m,sourcePath:d,type:w.type,inodeGroup:w.inodeGroup,...w.target===void 0?{}:{target:w.target}}});return{kind:e,content:n.content,inventory:n.entries,activation:n.activation,url:i,mountPrefix:n.mountPrefix,integrity:o,materialized:!1,entries:u}}async function Qn(r,e,t){if(t===void 0)return;if(r.byteLength!==t.bytes)throw new Error(`Lazy ${e} byte count ${r.byteLength} does not match expected ${t.bytes}`);let n=globalThis.crypto?.subtle;if(!n)throw new Error(`Lazy ${e} integrity verification is unavailable`);let i=new Uint8Array(r.byteLength);i.set(r);let o=new Uint8Array(await n.digest("SHA-256",i)),s=Array.from(o,a=>a.toString(16).padStart(2,"0")).join("");if(s!==t.sha256)throw new Error(`Lazy ${e} SHA-256 ${s} does not match expected ${t.sha256}`)}var an=class r{fs;imageMetadata;lazyFiles=new Map;lazyArchiveGroups=[];deferredTreeMaterializationHandles=new WeakMap;lazyArchiveInodes=new Map;lazyDownloadListeners=new Set;lazyPreparations=new Map;lazyFetch=e=>globalThis.fetch(e);constructor(e,t=null){this.fs=e,this.imageMetadata=t}static inodeKey(e,t){return`${e}:${t}`}static canAdoptLegacyLazyStub(e){return(e.mode&at)===Jn&&e.size===0&&e.dataSequence<=1}reconcileLazyIdentityState(e){for(let[t,n]of this.lazyFiles){let i=e.get(t);if(!i||i.dataSequence!==n.dataSequence||i.paths.length===0){this.lazyFiles.delete(t);continue}n.paths=new Set(i.paths),n.paths.has(n.path)||(n.path=i.paths[0])}this.lazyArchiveInodes.clear();for(let t of this.lazyArchiveGroups){let n=t.content!==void 0&&t.inventory!==void 0&&!t.materialized,i=new Map;for(let s of t.entries.values()){if(s.deleted||s.materialized||s.generation===void 0)continue;let a=r.inodeKey(s.ino,s.generation);i.has(a)||i.set(a,s)}let o=new Map;for(let[s,a]of i){let c=e.get(s);if(!(!c||c.dataSequence!==(a.dataSequence??0))){for(let l of c.paths)o.set(l,{...a,ino:c.ino,generation:c.generation,dataSequence:c.dataSequence,deleted:!1,materialized:!1});c.paths.length>0&&this.lazyArchiveInodes.set(s,t)}}t.entries=o,t.materialized=o.size===0&&!n}}lazyFileForStat(e){let t=r.inodeKey(e.ino,e.generation),n=this.lazyFiles.get(t);if(n&&n.dataSequence!==e.dataSequence){this.lazyFiles.delete(t);return}return n}lazyArchiveForStat(e){let t=r.inodeKey(e.ino,e.generation),n=this.lazyArchiveInodes.get(t);if(!n)return;let i=Array.from(n.entries.values()).filter(o=>o.ino===e.ino&&o.generation===e.generation&&!o.deleted&&!o.materialized);if(i.some(o=>o.dataSequence===e.dataSequence))return n;this.lazyArchiveInodes.delete(t);for(let o of i)o.materialized=!0}lazyBackingForStat(e){let t=r.inodeKey(e.ino,e.generation),n=this.lazyFiles.get(t);if(n)return{token:n,path:n.path};let i=this.lazyArchiveInodes.get(t);if(!i)return null;let o=Array.from(i.entries.entries()).find(([,s])=>s.ino===e.ino&&s.generation===e.generation&&!s.deleted&&!s.materialized)?.[0];return o===void 0?null:{token:i,path:o}}lazyBackingForPath(e){let t=this.lazyArchiveGroups.find(n=>!n.materialized&&n.content!==void 0&&n.inventory!==void 0&&n.activation!==void 0&&Array.from(n.entries.values()).every(i=>i.deleted||i.materialized||i.isSymlink)&&n.activation.roots.some(i=>i==="/"||e===i||e.startsWith(`${i}/`)));if(t)return{token:t,path:e,directGroup:t};try{let n=this.fs.stat(e),i=this.lazyBackingForStat(n);return i?{token:i.token,path:e}:null}catch{return null}}startLazyPreparation(e){let{path:t,token:n}=e,i={status:"pending",promise:Promise.resolve(!1)},o=e.directGroup?this.ensureArchiveMaterialized(e.directGroup).then(()=>!0):this.materializePath(t);return i.promise=o.then(s=>(i.status="fulfilled",this.lazyPreparations.get(n)===i&&this.lazyPreparations.delete(n),s),s=>{throw i.status="rejected",i.error=s,s}),i.promise.catch(()=>{}),this.lazyPreparations.set(n,i),i}guardSynchronousLazyAccess(e){let t=this.lazyBackingForPath(e);if(!t)return;let n=this.lazyPreparations.get(t.token);if(n?.status==="fulfilled"){this.lazyPreparations.delete(t.token);let o=this.lazyBackingForPath(e);if(!o)return;n=this.lazyPreparations.get(o.token)??this.startLazyPreparation(o)}else if(n?.status==="rejected"){this.lazyPreparations.delete(t.token);let o=n.error instanceof Error?n.error.message:String(n.error),s=new Error(`EIO: lazy backing for ${e} failed: ${o}`);throw s.code="EIO",s.cause=n.error,s}else n||(n=this.startLazyPreparation(t));let i=new Error(`EAGAIN: lazy backing for ${e} is being prepared`);throw i.code="EAGAIN",i}invalidateLazyData(e){let t=r.inodeKey(e.ino,e.generation);this.lazyFiles.delete(t);let n=this.lazyArchiveInodes.get(t);if(n){this.lazyArchiveInodes.delete(t);for(let i of n.entries.values())i.ino===e.ino&&i.generation===e.generation&&(i.materialized=!0)}}rewriteLazyNamespacePaths(e,t,n){let i=t.length>1?t.replace(/\/+$/,""):t,o=n.length>1?n.replace(/\/+$/,""):n,s=`${i}/`,a=`${o}/`,c=r.inodeKey(e.ino,e.generation),l=(e.mode&at)===Xt,u=f=>f===i?o:l&&f.startsWith(s)?a+f.slice(s.length):f;for(let[f,h]of this.lazyFiles)!l&&f!==c||(h.paths=new Set(Array.from(h.paths,u)),h.path=u(h.path));for(let f of this.lazyArchiveGroups){let h=new Map;for(let[y,g]of f.entries){let d=g.generation===void 0?null:r.inodeKey(g.ino,g.generation);h.set(l||d===c?u(y):y,g)}f.entries=h,f.inventory&&(f.inventory=f.inventory.map(y=>({...y,vfsPath:u(y.vfsPath),...y.type==="hardlink"&&y.target!==void 0?{target:u(y.target)}:{}}))),f.activation&&(f.activation={...f.activation,roots:f.activation.roots.map(u)})}}get sharedBuffer(){return this.fs.buffer}static create(e,t){return new r(Se.mkfs(e,t))}static fromExisting(e){return new r(Se.mount(e))}rebaseToNewFileSystem(e){if(!Number.isSafeInteger(e)||e<=0)throw new Error(`Invalid MemoryFileSystem maxByteLength: ${e}`);let t=SharedArrayBuffer,{bytes:n,identities:i}=this.fs.snapshotState();this.reconcileLazyIdentityState(i);let o=this.serializeLazyEntries(),s=this.serializeLazyArchiveEntries(),a=new t(n.byteLength);new Uint8Array(a).set(n);let c=new r(Se.mount(a,{restoreImage:!0}),this.imageMetadata);c.importLazyEntries(o),c.importLazyArchiveEntries(s);let l=Math.min(e,Math.max(n.byteLength,Bo)),u=new t(l,{maxByteLength:e}),f=r.create(u,e);f.setImageMetadata(this.imageMetadata);let h=new Set(o.flatMap(g=>g.paths??[g.path])),y=new Set;for(let g of s)if(!g.materialized)for(let d of g.entries)!d.deleted&&!d.isSymlink&&y.add(d.vfsPath);return c.copyPathToFreshFileSystem("/",f,h,y,new Map),f.importLazyEntries(o.map(g=>{let d=f.fs.lstat(g.path);return{...g,ino:d.ino,generation:d.generation,dataSequence:d.dataSequence}})),f.importLazyArchiveEntries(s.map(g=>({...g,entries:g.entries.map(d=>{if(d.deleted)return{...d,ino:0,generation:void 0};let m=f.fs.lstat(d.vfsPath);return{...d,ino:m.ino,generation:m.generation,dataSequence:m.dataSequence}})}))),f}getImageMetadata(){return Go(this.imageMetadata)}setImageMetadata(e){this.imageMetadata=e===null?null:sr(e)}subscribeLazyDownloads(e){return this.lazyDownloadListeners.add(e),()=>this.lazyDownloadListeners.delete(e)}setLazyFetcher(e){this.lazyFetch=e}emitLazyDownload(e){if(this.lazyDownloadListeners.size===0)return;let t={...e,t:Ho()};for(let n of this.lazyDownloadListeners)try{n(t)}catch{}}async fetchLazyBytes(e){let t=0,n=e.integrity?.bytes??e.fallbackTotalBytes,i={id:e.id,kind:e.kind,url:e.url,path:e.path,mountPrefix:e.mountPrefix};this.emitLazyDownload({...i,status:"started",loadedBytes:t,totalBytes:n});try{let o=await this.lazyFetch(e.url);if(!o.ok)throw new Error(`HTTP ${o.status}`);if(n=Vo(o.headers)??n,e.integrity&&n!==void 0&&n!==e.integrity.bytes)throw new Error(`Lazy ${e.kind} byte count ${n} does not match expected ${e.integrity.bytes}`);if(!o.body){let l=new Uint8Array(await o.arrayBuffer());return t=l.byteLength,await Qn(l,e.kind,e.integrity),this.emitLazyDownload({...i,status:"progress",loadedBytes:t,totalBytes:n??t}),this.emitLazyDownload({...i,status:"complete",loadedBytes:t,totalBytes:n??t}),l}let s=o.body.getReader(),a=[];try{for(;;){let{done:l,value:u}=await s.read();if(l)break;if(u){if(a.push(u),t+=u.byteLength,e.integrity&&t>e.integrity.bytes)throw await s.cancel(),new Error(`Lazy ${e.kind} exceeded expected byte count ${e.integrity.bytes}`);this.emitLazyDownload({...i,status:"progress",loadedBytes:t,totalBytes:n})}}}finally{s.releaseLock()}let c=qo(a,t);return await Qn(c,e.kind,e.integrity),this.emitLazyDownload({...i,status:"complete",loadedBytes:t,totalBytes:n??t}),c}catch(o){let s=o instanceof Error?o.message:String(o);throw this.emitLazyDownload({...i,status:"error",loadedBytes:t,totalBytes:n,error:s}),o}}registerLazyFile(e,t,n,i=493){let o=e.split("/").filter(Boolean),s="";for(let c=0;c({...d})),activation:u,entries:new Map},y=d=>{let m=d.split("/").filter(Boolean),p="";for(let w=0;wm.vfsPath.split("/").length-p.vfsPath.split("/").length))if(d.type==="directory"){y(d.vfsPath);try{this.fs.mkdir(d.vfsPath,d.mode),this.fs.chmod(d.vfsPath,d.mode)}catch{if((this.fs.lstat(d.vfsPath).mode&at)!==Xt)throw new Error(`Lazy tree directory collides at ${d.vfsPath}`)}}for(let d of c){if(d.type!=="symlink")continue;y(d.vfsPath),this.fs.symlink(d.target,d.vfsPath);let m=this.fs.lstat(d.vfsPath);h.entries.set(d.vfsPath,{ino:m.ino,generation:m.generation,dataSequence:m.dataSequence,size:d.size,isSymlink:!0,deleted:!1,materialized:!0,archivePath:d.sourcePath,sourcePath:d.sourcePath,type:"symlink",target:d.target})}let g=new Map;for(let d of c){if(d.type!=="file")continue;y(d.vfsPath);let m=this.fs.createLazyStub(d.vfsPath,d.mode);this.invalidateLazyData(m),g.set(d.inodeGroup,m);let p={ino:m.ino,generation:m.generation,dataSequence:m.dataSequence,size:d.size,isSymlink:!1,deleted:!1,materialized:!1,archivePath:d.sourcePath,sourcePath:d.sourcePath,type:"file",inodeGroup:d.inodeGroup};h.entries.set(d.vfsPath,p),this.lazyArchiveInodes.set(r.inodeKey(m.ino,m.generation),h)}for(let d of c){if(d.type!=="hardlink")continue;let m=f.get(d.inodeGroup);y(d.vfsPath),this.fs.link(m.vfsPath,d.vfsPath);let p=this.fs.lstat(d.vfsPath),w=g.get(d.inodeGroup);if(p.ino!==w.ino||p.generation!==w.generation)throw new Error(`Lazy tree hardlink ${d.vfsPath} did not share its inode`);h.entries.set(d.vfsPath,{ino:p.ino,generation:p.generation,dataSequence:p.dataSequence,size:d.size,isSymlink:!1,deleted:!1,materialized:!1,archivePath:m.sourcePath,sourcePath:d.sourcePath,type:"hardlink",inodeGroup:d.inodeGroup,target:d.target})}return this.lazyArchiveGroups.push(h),h}registerLazyTreeWithMaterializationHandle(e,t,n="/",i){let o=this.registerLazyTreeInternal(e,t,n,i,!0),s=Object.freeze({[_o]:!0});return this.deferredTreeMaterializationHandles.set(s,o),s}registerLazyArchiveFromEntries(e,t,n,i,o){let s=Uo(e,t,n,i);s.some(({entry:c})=>!c.isDirectory&&!c.isSymlink)&&this.assertCanRegisterPendingLazyArchiveGroup();let a={...o?{content:sn({decoder:"zip-v1",mediaType:"application/zip",sha256:o.sha256,bytes:o.bytes,expandedBytes:s.reduce((c,l)=>c+l.entry.uncompressedSize,0),sourceEntryCount:s.length,transports:[e]})}:{},url:e,mountPrefix:n,integrity:Tt(o),materialized:!1,entries:new Map};for(let{entry:c,vfsPath:l}of s){if(c.isDirectory)continue;let u=l.split("/").filter(Boolean),f="";for(let h=0;hc.deleted||c.materialized),this.lazyArchiveGroups.push(a),a}importLazyArchiveEntries(e){this.importLazyArchiveEntriesInternal(e,!1,!0)}importLazyArchiveEntriesInternal(e,t,n){let i=ke(e,"Serialized lazy archive groups",0,Mo).map((a,c)=>{if(typeof a!="object"||a===null||Array.isArray(a))throw new Error(`Serialized lazy archive group ${c} must be an object`);let l=a.kind;if(l===rn||l===mi)return Xo(a,l);if(l===nn)return Si(a,!1);if(l!==void 0)throw new Error(`Serialized lazy archive group ${c} has an unsupported kind`);if(n)throw new Error(`Serialized lazy archive group ${c} is missing its kind discriminator`);return Si(a,!0)});Ei([...this.serializeLazyArchiveEntries(),...i]);let o=[],s=new Map;for(let a of i){let c=new Map,l=a.mountPrefix.replace(/\/+$/,""),u=a.content!==void 0&&a.inventory!==void 0&&a.activation!==void 0,f=u?new Map(a.inventory.map(p=>[p.vfsPath,p])):null,h=u?new Map(a.inventory.map(p=>[on(p),p])):null,y=new Map,g=new Map;for(let p of a.entries){let w=null,v=a.materialized||p.materialized===!0||p.isSymlink;if(!p.deleted&&!v){if((p.generation===void 0||p.dataSequence===void 0)&&!t)throw new Error("Live lazy-archive metadata requires inode generation and data sequence");try{w=this.fs.lstat(p.vfsPath)}catch{if(u)throw new Error(`Serialized lazy tree stub ${p.vfsPath} is missing from the filesystem`);continue}if(w.ino!==p.ino){if(u)throw new Error(`Serialized lazy tree stub ${p.vfsPath} has a different inode`);continue}if(p.generation!==void 0&&w.generation!==p.generation){if(u)throw new Error(`Serialized lazy tree stub ${p.vfsPath} has a different generation`);continue}if(p.dataSequence===void 0){if(!r.canAdoptLegacyLazyStub(w)){if(u)throw new Error(`Serialized lazy tree stub ${p.vfsPath} is not pristine`);continue}}else if(w.dataSequence!==p.dataSequence){if(u)throw new Error(`Serialized lazy tree stub ${p.vfsPath} has a different data sequence`);continue}if(u){let b=f.get(p.vfsPath),E=h.get(on(p))??b;if(!E||(w.mode&at)!==Jn||w.size!==0||(w.mode&4095)!==E.mode||b?.inodeGroup!==void 0&&b.inodeGroup!==E.inodeGroup)throw new Error(`Serialized lazy tree stub ${p.vfsPath} disagrees with its inventory`);let k=r.inodeKey(w.ino,w.generation),x=p.inodeGroup,I=y.get(x),B=g.get(k);if(I!==void 0&&I!==k||B!==void 0&&B!==x)throw new Error(`Serialized lazy tree inode group ${x} disagrees with the filesystem`);y.set(x,k),g.set(k,x)}}c.set(p.vfsPath,{ino:p.ino,generation:w?.generation??p.generation,dataSequence:w?.dataSequence??p.dataSequence,size:p.size,isSymlink:p.isSymlink,deleted:p.deleted,materialized:v,archivePath:p.archivePath??p.vfsPath.slice(l.length+1),sourcePath:p.sourcePath??p.archivePath??p.vfsPath.slice(l.length+1),type:p.type??(p.isSymlink?"symlink":"file"),inodeGroup:p.inodeGroup,target:p.target})}let d=a.content===void 0?void 0:sn(a.content),m={content:d,url:d?.transports[0]??a.url,mountPrefix:a.mountPrefix,integrity:d?{sha256:d.sha256,bytes:d.bytes}:Tt(a.integrity),materialized:a.materialized||!(d&&a.inventory)&&Array.from(c.values()).every(p=>p.deleted||p.materialized),inventory:a.inventory?.map(p=>({...p})),activation:a.activation?{mode:a.activation.mode,capabilities:[...a.activation.capabilities],roots:[...a.activation.roots]}:void 0,entries:c};if(o.push(m),!m.materialized){for(let[,p]of c)if(!p.deleted&&!p.materialized&&p.generation!==void 0){let w=r.inodeKey(p.ino,p.generation),v=s.get(w);if(v!==void 0&&v!==m)throw new Error(`Serialized lazy archive groups share pending inode ${w}`);if(this.lazyArchiveInodes.has(w))throw new Error(`Serialized lazy archive group collides with pending inode ${w}`);s.set(w,m)}}}this.lazyArchiveGroups.push(...o);for(let[a,c]of s)this.lazyArchiveInodes.set(a,c)}rewriteLazyArchiveUrls(e){for(let t of this.lazyArchiveGroups)t.content?(t.content={...t.content,transports:t.content.transports.map(e)},t.url=t.content.transports[0]):t.url=e(t.url)}serializeLazyArchiveEntries(){let e=[];for(let t of this.lazyArchiveGroups){let n=Array.from(t.entries,([o,s])=>({vfsPath:o,ino:s.ino,generation:s.generation,dataSequence:s.dataSequence,size:s.size,isSymlink:s.isSymlink,deleted:s.deleted,materialized:s.materialized,archivePath:s.archivePath,sourcePath:s.sourcePath,type:s.type,inodeGroup:s.inodeGroup,target:s.target})).filter(o=>!o.deleted&&!o.materialized);if(n.length===0&&!(t.content&&t.inventory&&!t.materialized))continue;let i=t.content!==void 0&&t.inventory!==void 0&&t.activation!==void 0;if(i&&t.content.transports.length===0)throw new Error("Direct-materialization tree must be materialized before serialization");e.push(i?{kind:t.content.source===void 0?rn:mi,content:t.content,inventory:t.inventory,activation:t.activation,url:t.url,mountPrefix:t.mountPrefix,integrity:t.integrity,materialized:!1,entries:n}:{kind:nn,url:t.url,mountPrefix:t.mountPrefix,integrity:t.integrity,materialized:!1,entries:n})}return e}exportLazyArchiveEntries(){return this.reconcileLazyIdentityState(this.fs.identityState()),this.serializeLazyArchiveEntries()}pendingDeferredTreeUsage(){return this.reconcileLazyIdentityState(this.fs.identityState()),bi(this.serializeLazyArchiveEntries())}assertCanAppendDeferredTreeUsage(e){rr(e);let t=this.pendingDeferredTreeUsage();rr({groups:t.groups+e.groups,archiveBytes:t.archiveBytes+e.archiveBytes,expandedBytes:t.expandedBytes+e.expandedBytes,payloadBytes:t.payloadBytes+e.payloadBytes,entries:t.entries+e.entries})}assertCanRegisterPendingLazyArchiveGroup(){if(this.reconcileLazyIdentityState(this.fs.identityState()),this.lazyArchiveGroups.filter(t=>!t.materialized&&(t.content!==void 0&&t.inventory!==void 0||Array.from(t.entries.values()).some(n=>!n.deleted&&!n.materialized))).length>=me.maxGroups)throw new Error(`Cannot register another lazy archive group: ${me.maxGroups} pending groups already exist`)}async preparePath(e){let t=!1,n=Math.max(3,this.lazyArchiveGroups.length+1);for(let i=0;i!o.materialized&&o.activation?.mode==="boot-prefetch"),t=0,n,i=Array.from({length:Math.min(e.length,No)},async()=>{for(;n===void 0;){let o=t;if(t+=1,o>=e.length)return;try{await this.prepareLazyTreeGroup(e[o])}catch(s){n??=s}}});if(await Promise.all(i),n!==void 0)throw n;return e.length}async materializeRegisteredDeferredTree(e,t){let n=this.deferredTreeMaterializationHandles.get(e);if(n===void 0)throw new Error("Deferred-tree handle was not issued by this filesystem");if(n.materialized)return!1;let i=this.lazyPreparations.get(n);if(i!==void 0)return i.promise;let o=new Uint8Array(t.byteLength);o.set(t);let s={status:"pending",promise:Promise.resolve(!1)};s.promise=Promise.resolve().then(async()=>(await Qn(o,"tree",n.integrity),await this.materializeArchiveBytes(n,o),!0)).then(a=>(s.status="fulfilled",a),a=>{throw s.status="rejected",s.error=a,a}),s.promise.catch(()=>{}),this.lazyPreparations.set(n,s);try{return await s.promise}finally{this.lazyPreparations.get(n)===s&&this.lazyPreparations.delete(n)}}async prepareLazyTreeGroup(e){if(e.materialized)return!1;let t={token:e,path:e.activation?.roots[0]??e.mountPrefix,directGroup:e},n=this.lazyPreparations.get(e)??this.startLazyPreparation(t);try{return await n.promise}finally{this.lazyPreparations.get(e)===n&&this.lazyPreparations.delete(e)}}async ensureMaterialized(e){return this.preparePath(e)}async materializePath(e){if(this.lazyFiles.size===0&&this.lazyArchiveInodes.size===0)return!1;let t;try{t=this.fs.stat(e)}catch{return!1}let n=r.inodeKey(t.ino,t.generation),i=this.lazyFiles.get(n);if(i){let s=await this.fetchLazyBytes({id:`file:${t.ino}`,kind:"file",url:i.url,path:i.path,fallbackTotalBytes:i.size});for(let a=0;a<3;a++){if(this.lazyFiles.get(n)!==i)return!1;for(let c of new Set([e,...i.paths]))if(this.fs.replaceIfIdentity(c,i.ino,i.generation,i.dataSequence,s))return i.path=c,this.lazyFiles.delete(n),!0;this.reconcileLazyIdentityState(this.fs.identityState())}throw new Error(`Lazy file kept changing names while materializing: ${e}`)}let o=this.lazyArchiveInodes.get(n);return o?(await this.ensureArchiveMaterialized(o,{path:e,ino:t.ino,generation:t.generation}),!this.lazyArchiveInodes.has(n)):!1}async decodeAndValidateLazyTree(e,t){let n=e.content,i=e.inventory;if(!n||!i)throw new Error("Lazy tree is missing its decoder or complete inventory");let o=new Map,s=new Map(i.map(f=>[f.vfsPath,f]));if(n.source!==void 0)for(let f of n.source.entries)o.set(f.sourcePath,f);else for(let f of i){if(f.type==="hardlink"){let y=s.get(f.target);if(!y)throw new Error(`Lazy tree hardlink target disappeared: ${f.target}`);if(f.sourcePath===y.sourcePath)continue}if(o.get(f.sourcePath))throw new Error(`Lazy tree inventory duplicates source member ${f.sourcePath}`);o.set(f.sourcePath,{sourcePath:f.sourcePath,type:f.type,mode:f.mode,size:f.size,...f.type==="symlink"?{target:f.target}:{},...f.type==="hardlink"?{target:s.get(f.target)?.sourcePath}:{}})}let a=new Map,c=0;if(n.decoder==="zip-v1"){let{parseZipCentralDirectory:f,extractZipEntryBounded:h}=await Promise.resolve().then(()=>(Kn(),Gn)),y=f(t);if(y.length!==n.sourceEntryCount||y.length!==o.size)throw new Error("Lazy ZIP tree decoded inventory counts differ from its descriptor");for(let g of y){let d=g.isDirectory?g.fileName.replace(/\/$/,""):g.fileName;if(a.has(d))throw new Error(`Lazy ZIP tree duplicates source member ${d}`);let m=o.get(d);if(!m)throw new Error(`Lazy ZIP tree has undeclared source member ${d}`);if(c+=g.uncompressedSize,c>n.expandedBytes||g.uncompressedSize!==m.size)throw new Error(`Lazy ZIP tree member ${d} exceeds its inventory`);if((g.isDirectory?"directory":g.isSymlink?"symlink":"file")!==m.type||(g.mode&4095)!==m.mode)throw new Error(`Lazy ZIP tree member ${d} differs from inventory`);if(g.isDirectory)a.set(d,{type:"directory",mode:g.mode});else{let w=h(t,g,m.size);if(g.isSymlink){let v;try{v=new TextDecoder("utf-8",{fatal:!0}).decode(w)}catch{throw new Error(`Lazy ZIP tree symlink ${d} is not UTF-8`)}a.set(d,{type:"symlink",mode:g.mode,target:v})}else a.set(d,{type:"file",mode:g.mode,data:w})}}}else{let{parseTarGzip:f}=await Promise.resolve().then(()=>(gi(),yi)),h=f(t,{label:`Lazy tree ${n.sha256}`,limits:{maxCompressedBytes:n.bytes,maxUncompressedBytes:n.expandedBytes,maxEntries:n.sourceEntryCount}});c=new DataView(t.buffer,t.byteOffset,t.byteLength).getUint32(t.byteLength-4,!0);for(let y of h){if(a.has(y.path))throw new Error(`Lazy TAR tree duplicates source member ${y.path}`);y.type==="file"?a.set(y.path,{type:"file",mode:y.mode,data:y.data}):y.type==="directory"?a.set(y.path,{type:"directory",mode:y.mode}):a.set(y.path,{type:y.type,mode:y.mode,target:y.linkName})}}if(a.size!==n.sourceEntryCount||a.size!==o.size||c!==n.expandedBytes)throw new Error("Lazy tree decoded inventory counts differ from its descriptor");for(let[f,h]of o){let y=a.get(f);if(!y)throw new Error(`Lazy tree is missing source member ${f}`);let g=h.type;if(y.type!==g)throw new Error(`Lazy tree member ${f} is ${y.type}, expected ${g}`);if((y.mode&4095)!==h.mode)throw new Error(`Lazy tree member ${f} mode differs from inventory`);if(g==="file"&&y.data?.byteLength!==h.size)throw new Error(`Lazy tree member ${f} size differs from inventory`);if(g==="symlink"&&y.target!==h.target)throw new Error(`Lazy tree symlink ${f} target differs from inventory`);if(g==="hardlink"&&y.target!==h.target)throw new Error(`Lazy tree hardlink ${f} target differs from inventory`)}let l=new Set(i.flatMap(f=>f.materialization==="archive-homebrew-relocate"?[f.sourcePath]:[]));if(n.source!==void 0){let f=new Map(n.source.entries.map(g=>[g.sourcePath,g])),h=ki(n.source.entries),y=n.source.entries.filter(g=>g.sourcePath==="INSTALL_RECEIPT.json"||g.sourcePath.endsWith("/INSTALL_RECEIPT.json"));if(y.length>1)throw new Error(`Lazy Homebrew bottle has ${y.length} INSTALL_RECEIPT.json source members, expected at most one`);if(y.length===0){if(l.size>0)throw new Error("Lazy Homebrew bottle marks receipt relocation without INSTALL_RECEIPT.json")}else{let g=y[0],d=g.type==="file"?g:h.get(g.sourcePath),m=d===void 0?void 0:a.get(d.sourcePath);if(d?.type!=="file"||m?.type!=="file"||m.data===void 0)throw new Error("Lazy Homebrew bottle INSTALL_RECEIPT.json is not regular");let p=Kr(m.data),w=g.sourcePath.lastIndexOf("/"),v=w<0?"":g.sourcePath.slice(0,w),S=new Set(p.changedFiles.map(E=>v.length===0?E:`${v}/${E}`));if(l.size!==S.size||[...l].some(E=>!S.has(E)))throw new Error("Lazy Homebrew bottle relocation markers differ from INSTALL_RECEIPT.json");let b=new Set;for(let E of S){let k=f.get(E),x=k?.type==="file"?k:k===void 0?void 0:h.get(k.sourcePath),I=x===void 0?void 0:a.get(x.sourcePath);if(x?.type!=="file"||I?.type!=="file"||I.data===void 0)throw new Error(`Lazy Homebrew bottle changed source ${E} is not regular`);b.has(x.sourcePath)||(I.data=Zr(I.data,p,E),b.add(x.sourcePath))}}}else if(l.size>0)throw new Error("Lazy tree receipt relocation requires original-bottle source truth");let u=new Map;for(let f of i){if(f.type!=="file"||f.materialization==="descriptor")continue;let h=a.get(f.sourcePath);if(h?.type!=="file"||!h.data)throw new Error(`Lazy tree has no file content for ${f.sourcePath}`);u.set(f.sourcePath,h.data)}return u}async ensureArchiveMaterialized(e,t){if(e.materialized)return;let n=e.content!==void 0&&e.inventory!==void 0,i=n?e.content.transports:[e.url],o=[],s=null;for(let[a,c]of i.entries())try{s=await this.fetchLazyBytes({id:`archive:${e.mountPrefix}:${e.content?.sha256??c}:${a}`,kind:n?"tree":"archive",url:c,mountPrefix:e.mountPrefix,integrity:e.integrity});break}catch(l){o.push(l instanceof Error?l.message:String(l))}if(s===null)throw new Error(`All ${i.length} lazy ${n?"tree":"archive"} transports failed: ${o.join("; ")}`);await this.materializeArchiveBytes(e,s,t)}async materializeArchiveBytes(e,t,n){if(e.materialized)return;let o=e.content!==void 0&&e.inventory!==void 0?await this.decodeAndValidateLazyTree(e,t):null,{parseZipCentralDirectory:s,extractZipEntry:a}=await Promise.resolve().then(()=>(Kn(),Gn)),c=o?[]:s(t),l=new Map;for(let y of c){if(l.has(y.fileName))throw new Error(`Lazy archive contains duplicate member: ${y.fileName}`);l.set(y.fileName,y)}let u=e.mountPrefix.replace(/\/+$/,""),f=new Map;for(let[y,g]of e.entries){if(g.deleted||g.materialized)continue;let d=g.archivePath??y.slice(u.length+1),m=o?void 0:l.get(d),p=o?.get(d);if(o){if(p===void 0||p.byteLength!==g.size)throw new Error(`Lazy tree member ${d} does not match its registered metadata`)}else if(m===void 0||m.isDirectory||m.isSymlink||m.uncompressedSize!==g.size)throw new Error(`Lazy archive member ${d} does not match its registered metadata`);if(g.generation===void 0)continue;let w=r.inodeKey(g.ino,g.generation),v=f.get(w);if(v&&v.archivePath!==d)throw new Error(`Lazy archive aliases for inode ${w} name different members`);if(!v){let S=p??a(t,m);if(S.byteLength!==g.size)throw new Error(`Lazy archive member ${d} extracted ${S.byteLength} bytes, expected ${g.size}`);f.set(w,{archivePath:d,content:S})}}let h=n?r.inodeKey(n.ino,n.generation):null;for(let y=0;y<3;y++){let g=new Map;for(let[d,m]of e.entries){if(m.deleted||m.materialized||m.generation===void 0)continue;let p=r.inodeKey(m.ino,m.generation);if(this.lazyArchiveInodes.get(p)!==e)continue;let w=f.get(p);if(!w)throw new Error(`Lazy archive has no extracted content for inode ${p}`);let v=g.get(p);v||(v={ino:m.ino,generation:m.generation,dataSequence:m.dataSequence??0,paths:new Set,content:w.content},g.set(p,v)),v.paths.add(d),n&&n.ino===m.ino&&n.generation===m.generation&&v.paths.add(n.path)}if(g.size>0&&!this.fs.replaceManyIfIdentities(Array.from(g.values(),m=>({paths:Array.from(m.paths),expectedIno:m.ino,expectedGeneration:m.generation,expectedDataSequence:m.dataSequence,data:m.content})))){if(this.reconcileLazyIdentityState(this.fs.identityState()),h&&!this.lazyArchiveInodes.has(h))return;continue}for(let[d,m]of g){this.lazyArchiveInodes.delete(d);for(let p of e.entries.values())p.ino===m.ino&&p.generation===m.generation&&(p.materialized=!0)}if(e.materialized=Array.from(e.entries.values()).every(d=>d.deleted||d.materialized),e.materialized||(this.reconcileLazyIdentityState(this.fs.identityState()),h&&!this.lazyArchiveInodes.has(h)))return}if(h&&this.lazyArchiveInodes.has(h))throw new Error(`Lazy archive member kept changing names while materializing: ${n?.path}`)}async materializeAllLazyEntries(){for(let t=0;t<3;t++){this.reconcileLazyIdentityState(this.fs.identityState());let n=this.lazyArchiveGroups.filter(s=>!s.materialized&&s.content!==void 0&&s.inventory!==void 0);if(this.lazyFiles.size===0&&this.lazyArchiveInodes.size===0&&n.length===0)return;let i=Array.from(this.lazyFiles.values(),s=>s.path);for(let s of i)await this.ensureMaterialized(s);let o=new Set(this.lazyArchiveInodes.values());for(let s of n)o.add(s);for(let s of o)await this.prepareLazyTreeGroup(s)}this.reconcileLazyIdentityState(this.fs.identityState());let e=this.lazyArchiveGroups.some(t=>!t.materialized&&t.content!==void 0&&t.inventory!==void 0);if(this.lazyFiles.size!==0||this.lazyArchiveInodes.size!==0||e)throw new Error("Cannot create a self-contained VFS image while lazy entries remain pending")}async saveImage(e){e?.materializeAll&&await this.materializeAllLazyEntries();let{bytes:t,identities:n}=this.fs.snapshotState({normalizeTimestampsMs:e?.normalizeTimestampsMs});this.reconcileLazyIdentityState(n);let i=this.serializeLazyEntries(),o=i.length>0,s=o?new TextEncoder().encode(JSON.stringify(i)):new Uint8Array(0);if(s.byteLength>Qt)throw new Error(`VFS image lazy metadata exceeds ${Qt} bytes`);let a=this.serializeLazyArchiveEntries();Ei(a);let c=a.length>0,l=c?new TextEncoder().encode(JSON.stringify(a)):new Uint8Array(0);if(l.byteLength>en)throw new Error(`VFS image lazy archive metadata exceeds ${en} bytes`);let u=e?.metadata===void 0?this.imageMetadata:e.metadata,f=Zo(u),h=f.byteLength>0,y=c?4+l.byteLength:0,g=h?4+f.byteLength:0,d=re+t.byteLength+4+s.byteLength+y+g,m=new Uint8Array(d),p=new DataView(m.buffer);p.setUint32(0,er,!0),p.setUint32(4,tr,!0),p.setUint32(8,(o?jn:0)|(c?Jt:0)|(c?Yn:0)|(h?Xn:0),!0),p.setUint32(12,t.byteLength,!0),m.set(t,re);let w=re+t.byteLength;if(p.setUint32(w,s.byteLength,!0),s.byteLength>0&&m.set(s,w+4),c){let v=w+4+s.byteLength;p.setUint32(v,l.byteLength,!0),m.set(l,v+4)}if(h){let v=w+4+s.byteLength+y;p.setUint32(v,f.byteLength,!0),m.set(f,v+4)}return m}static readImageMetadata(e){let t=Yt(e);if(!(t.flags&Xn))return null;let{metadataOffset:n}=wi(t.image,t.view,t.flags,t.sabLen);if(t.image.byteLengthct)throw new Error(`VFS image metadata exceeds ${ct} bytes`);if(t.image.byteLength0){let m=n.subarray(g+4,g+4+d),p=ke(vi(m,"VFS image lazy metadata"),"VFS image lazy entries",0,ft);y.importLazyEntriesInternal(p,!0)}if(o&Jt){let m=a.archiveOffset,p=i.getUint32(m,!0);if(p>0){let w=n.subarray(m+4,m+4+p),v=vi(w,"VFS image lazy archive metadata");y.importLazyArchiveEntriesInternal(v,!0,!!(o&Yn))}}return y}adaptStat(e){return{dev:0,ino:e.ino,mode:e.mode,nlink:e.linkCount,uid:e.uid,gid:e.gid,size:e.size,atimeMs:e.atime,mtimeMs:e.mtime,ctimeMs:e.ctime}}adaptStatWithLazySize(e){let t=this.adaptStat(e),n=this.lazyFileForStat(e);if(n)return t.size=n.size,t;let i=this.lazyArchiveForStat(e);if(i){for(let o of i.entries.values())if(o.ino===e.ino&&o.generation===e.generation&&!o.deleted){t.size=o.size;break}}return t}open(e,t,n){(t&It)===0&&!((t&kt)!==0&&(t&Bn)!==0)&&this.guardSynchronousLazyAccess(e);let i=this.fs.open(e,t,n);return(t&It)!==0&&this.invalidateLazyData(this.fs.fstat(i)),i}close(e){return this.fs.close(e),0}read(e,t,n,i){if(i>0){let o=this.lazyBackingForStat(this.fs.fstat(e));o&&(this.reconcileLazyIdentityState(this.fs.identityState()),o=this.lazyBackingForStat(this.fs.fstat(e)),o&&this.guardSynchronousLazyAccess(o.path))}return n!==null?this.fs.readAt(e,t.subarray(0,i),n):this.fs.read(e,t.subarray(0,i))}write(e,t,n,i){if(n!==null){let s=this.fs.writeAt(e,t.subarray(0,i),n);return s>0&&this.invalidateLazyData(this.fs.fstat(e)),s}let o=this.fs.write(e,t.subarray(0,i));return o>0&&this.invalidateLazyData(this.fs.fstat(e)),o}seek(e,t,n){return this.fs.lseek(e,t,n)}fstat(e){return this.adaptStatWithLazySize(this.fs.fstat(e))}fpathconf(e,t){let n=this.fstat(e);return kn(n,t,{supportsSymlinks:!0,timestampResolutionNs:1e6})}ftruncate(e,t){this.fs.ftruncate(e,t),this.invalidateLazyData(this.fs.fstat(e))}fsync(e){}fchmod(e,t){this.fs.fchmod(e,t)}fchown(e,t,n){this.fs.fchown(e,t,n)}stat(e){return this.adaptStatWithLazySize(this.fs.stat(e))}lstat(e){return this.adaptStatWithLazySize(this.fs.lstat(e))}statfs(e){this.fs.stat(e);let t=this.fs.statfs();return{type:1397114451,bsize:t.blockSize,blocks:t.totalBlocks,bfree:t.freeBlocks,bavail:t.freeBlocks,files:t.totalInodes,ffree:t.freeInodes,fsid:0,namelen:t.maxName,frsize:t.blockSize,flags:0}}pathconf(e,t){let n=this.stat(e);return kn(n,t,{supportsSymlinks:!0,timestampResolutionNs:1e6})}mkdir(e,t){this.fs.mkdir(e,t)}rmdir(e){this.fs.rmdir(e)}unlink(e){let t=this.fs.unlink(e),n=r.inodeKey(t.ino,t.generation);if(t.linkCount>1&&(this.lazyFiles.has(n)||this.lazyArchiveInodes.has(n))){this.reconcileLazyIdentityState(this.fs.identityState());return}let i=this.lazyFiles.get(n);i&&(i.paths.delete(e),t.linkCount<=1?this.lazyFiles.delete(n):i.path===e&&(i.path=i.paths.values().next().value));let o=this.lazyArchiveInodes.get(n);if(o){let s=o.entries.get(e);if(t.linkCount<=1){for(let a of o.entries.values())a.ino===t.ino&&a.generation===t.generation&&(a.deleted=!0);this.lazyArchiveInodes.delete(n)}else s&&o.entries.delete(e)}}rename(e,t){let{source:n,replaced:i}=this.fs.rename(e,t);if(i&&i.ino===n.ino&&i.generation===n.generation)return;let o=!1;if(i){let s=r.inodeKey(i.ino,i.generation);i.linkCount>1&&(this.lazyFiles.has(s)||this.lazyArchiveInodes.has(s))&&(this.reconcileLazyIdentityState(this.fs.identityState()),o=!0);let a=this.lazyFiles.get(s);!o&&a&&(a.paths.delete(t),i.linkCount<=1?this.lazyFiles.delete(s):a.path===t&&(a.path=a.paths.values().next().value));let c=this.lazyArchiveInodes.get(s);if(!o&&c){let l=c.entries.get(t);i.linkCount<=1?(l&&(l.deleted=!0),this.lazyArchiveInodes.delete(s)):l&&c.entries.delete(t)}}o||this.rewriteLazyNamespacePaths(n,e,t)}link(e,t){let n=this.fs.link(e,t),i=r.inodeKey(n.ino,n.generation),o=this.lazyFiles.get(i);o&&o.paths.add(t);let s=this.lazyArchiveInodes.get(i);if(s){let a=Array.from(s.entries.values()).find(c=>c.ino===n.ino&&c.generation===n.generation);a&&s.entries.set(t,{...a})}}symlink(e,t){this.fs.symlink(e,t)}readlink(e){return this.fs.readlink(e)}chmod(e,t){this.fs.chmod(e,t)}chown(e,t,n){this.fs.chown(e,t,n)}lchown(e,t,n){this.fs.lchown(e,t,n)}createFileWithOwner(e,t,n,i,o){let s=this.open(e,577,t);o.length>0&&this.write(s,o,null,o.length),this.close(s),this.chown(e,n,i),this.chmod(e,t)}mkdirWithOwner(e,t,n,i){this.mkdir(e,t),this.chown(e,n,i),this.chmod(e,t)}symlinkWithOwner(e,t,n,i){this.symlink(e,t),this.lchown(t,n,i)}copyPathToFreshFileSystem(e,t,n,i,o){let s=this.lstat(e),a=s.mode&at,c=s.mode&4095;if(a===Xt){e==="/"?(t.chown(e,s.uid,s.gid),t.chmod(e,c)):t.mkdirWithOwner(e,c,s.uid,s.gid);let h=this.opendir(e);try{for(;;){let y=this.readdir(h);if(!y)break;y.name==="."||y.name===".."||this.copyPathToFreshFileSystem(e==="/"?`/${y.name}`:`${e}/${y.name}`,t,n,i,o)}}finally{this.closedir(h)}r.applyTimes(t,e,s);return}let l=s.nlink>1?`${s.dev}:${s.ino}`:null,u=l?o.get(l):void 0;if(u){t.link(u,e);return}if(a===Po){t.symlinkWithOwner(this.readlink(e),e,s.uid,s.gid),l&&o.set(l,e);return}if(a!==Jn)throw new Error(`Unsupported file type while rebasing VFS: ${e}`);if(n.has(e)||i.has(e)){t.createFileWithOwner(e,c,s.uid,s.gid,new Uint8Array(0)),r.applyTimes(t,e,s),l&&o.set(l,e);return}this.copyRegularFileToFreshFileSystem(e,t,s,c),l&&o.set(l,e)}copyRegularFileToFreshFileSystem(e,t,n,i){let o=this.open(e,Oo,0),s=null;try{s=t.open(e,To,i);let a=new Uint8Array(Math.min(Ro,Math.max(1,n.size))),c=n.size;for(;c>0;){let l=Math.min(a.byteLength,c),u=this.read(o,a,null,l);if(u<=0)throw new Error(`Unexpected EOF while rebasing VFS file: ${e}`);let f=0;for(;f!e||e==="."||e===".."))throw new Error(`Binary resolver path must be a normalized portable relative path: ${JSON.stringify(r)}`);return r}var We=new Set(["wasm32","wasm64"]);function Be(r){if(aa(r),!r.startsWith("programs/"))return r;let e=r.slice(9),t=e.split("/",1)[0];return We.has(t)?r:`programs/wasm32/${e}`}function ca(r,e=F(gn(),"wasm")){let t=Be(r),n=[F(e,t)];return r==="kernel.wasm"?n.push(F(e,"kandelo-kernel.wasm")):r==="userspace.wasm"?n.push(F(e,"wasm_posix_userspace.wasm")):r==="rootfs.vfs"&&n.push(F(e,"rootfs.vfs")),n}var un=class extends Error{constructor(e){super(e),this.name="BinaryNotFoundError"}};function Ni(){let r=[],e=!1;try{let n=Ze();e=!0;for(let[i,o]of[["local-binaries",F(n,"local-binaries")],["binaries",F(n,"binaries")]])r.push({label:i,root:o,identity:i==="local-binaries"?"local-generation":"program-cache",allowRegularFileClosure:!1,candidatesFor(s){return[F(o,Be(s))]}})}catch{}let t=F(gn(),"wasm");return r.push({label:"installed package",root:t,identity:"installed-package",allowRegularFileClosure:!e,candidatesFor(n){return ca(n,t)}}),r}function dt(r,e){return new Error(`Invalid package manifest ${r}: ${e}`)}function de(r){try{return hn(r),!0}catch(e){if(e instanceof Error&&"code"in e&&e.code==="ENOENT")return!1;throw e}}function Li(r,e,t){if(r.length===0||r.startsWith("/")||r.includes("\\")||r.includes("\0")||r.split("/").some(n=>!n||n==="."||n===".."))throw dt(e,`${t} must be a normalized portable relative path`);return r}function fn(r,e,t,n=!0){if(r.length===0||r==="."||r===".."||r.includes("/")||r.includes("\\")||r.includes("\0")||!n&&r.includes("@"))throw dt(e,`${t} must be a safe single path component`);return r}var Ai="kandelo-program-packages-v2",we="program-packages.json",_i=null,dn=null,ar=0;function Mi(){if(Object.prototype.hasOwnProperty.call(process.env,"WASM_POSIX_DEPS_REGISTRY")){let r=null;return(process.env.WASM_POSIX_DEPS_REGISTRY??"").split(":").filter(Boolean).map(e=>e.startsWith("~/")&&process.env.HOME!==void 0?F(process.env.HOME,e.slice(2)):yn(e)?ge(e):(r??=Ze(),ge(r,e)))}try{return[F(Ze(),"packages","registry")]}catch{return null}}function la(){let r;try{r=Ze()}catch{return null}if(!Rt(F(r,"tools","xtask","Cargo.toml"))||!Rt(F(r,"scripts","dev-shell.sh")))return null;try{let e=pe(dr()),t=pe(r);return[F(t,"host"),F(t,"scripts")].some(i=>Rt(i)&&pr(pe(i),e))?t:null}catch{return null}}function ur(r,e,t){let n=[typeof t.stderr=="string"?t.stderr.trim():"",typeof t.stdout=="string"?t.stdout.trim():"",t.error?.message??""].filter(Boolean).join(` +`);return`${r} ${e.join(" ")} failed${t.status===null?"":` with status ${t.status}`}${n?`: +${n}`:""}`}function fa(r){let e=process.env.KANDELO_DEV_SHELL_TOOL_PATH!==void 0,t=e?"rustc":"bash",n=e?["-vV"]:[F(r,"scripts","dev-shell.sh"),"rustc","-vV"],i=fr(t,n,{cwd:r,encoding:"utf8"});if(i.status!==0)throw new Error(ur(t,n,i));let o=i.stdout.split(/\r?\n/).find(s=>s.startsWith("host: "))?.slice(6).trim();if(!o)throw new Error(`Could not determine the Rust host target for ${r}`);return o}function cr(r){try{if(hn(r).isFile())return pe(r)}catch{}throw new Error(`Prepared xtask is not a regular file: ${r}`)}function da(r){let e=process.env.WASM_POSIX_XTASK_BIN;if(e!==void 0){let l=yn(e)?ge(e):ge(r,e);return cr(l)}if(dn?.sourceRepoRoot===r)return cr(dn.xtaskPath);let t=fa(r),n=F(r,"target",t,"release",process.platform==="win32"?"xtask.exe":"xtask"),i=["build","--release","-p","xtask","--target",t,"--quiet"],o=process.env.KANDELO_DEV_SHELL_TOOL_PATH!==void 0,s=o?"cargo":"bash",a=o?i:[F(r,"scripts","dev-shell.sh"),"cargo",...i],c=fr(s,a,{cwd:r,encoding:"utf8"});if(c.status!==0)throw new Error(ur(s,a,c));return dn={sourceRepoRoot:r,xtaskPath:cr(n)},dn.xtaskPath}function ua(){let r=la();if(r===null)return;let e=Mi();if(e===null)return;if(_i){_i(r,e);return}let t=da(r),n=["build-deps","program-index-context-check"],i=fr(t,n,{cwd:r,encoding:"utf8",env:{...process.env,WASM_POSIX_DEPS_REGISTRY:e.join(":")}});if(i.status!==0)throw new Error(`Program package source projection is not current: +${ur(t,n,i)}`)}function ha(r,e){if(ar>0||!r.some(t=>t.startsWith("programs/")))return e();ar+=1;try{return ua(),e()}finally{ar-=1}}function Re(r,e){let t=Object.keys(r).sort(),n=[...e].sort();return t.length===n.length&&t.every((i,o)=>i===n[o])}function lr(r){let e;try{e=JSON.parse(Ge(r,"utf8"))}catch(s){throw new Error(`Invalid program package index ${r}: ${s instanceof Error?s.message:String(s)}`)}if(typeof e!="object"||e===null||!Re(e,["format","identities","packages"])||e.format!==Ai||typeof e.identities!="object"||e.identities===null||Array.isArray(e.identities)||typeof e.packages!="object"||e.packages===null||Array.isArray(e.packages))throw new Error(`Invalid program package index ${r}: expected ${Ai}`);let t=new Map,n=e.identities;for(let[s,a]of Object.entries(n)){if(fn(s,r,"identity package name",!1),typeof a!="object"||a===null||!Re(a,["manifestSha256","cacheKeys"])||typeof a.manifestSha256!="string"||!/^[a-f0-9]{64}$/.test(a.manifestSha256)||typeof a.cacheKeys!="object"||a.cacheKeys===null||Array.isArray(a.cacheKeys))throw new Error(`Invalid program package index ${r}: malformed identity ${JSON.stringify(s)}`);let c=a.cacheKeys;if(!Re(c,["wasm32","wasm64"])||Object.values(c).some(l=>typeof l!="string"||!/^[a-f0-9]{64}$/.test(l)))throw new Error(`Invalid program package index ${r}: identity ${JSON.stringify(s)} has invalid contextual cache keys`);t.set(s,{manifestSha256:a.manifestSha256,cacheKeys:c})}let i=new Map,o=e.packages;for(let[s,a]of Object.entries(o)){if(fn(s,r,"package name",!1),typeof a!="object"||a===null||!Re(a,["manifestSha256","arches","cacheKeys","dependencyClosures","members"])||!Array.isArray(a.arches)||typeof a.cacheKeys!="object"||a.cacheKeys===null||Array.isArray(a.cacheKeys)||typeof a.dependencyClosures!="object"||a.dependencyClosures===null||Array.isArray(a.dependencyClosures)||!Array.isArray(a.members)||typeof a.manifestSha256!="string"||!/^[a-f0-9]{64}$/.test(a.manifestSha256))throw new Error(`Invalid program package index ${r}: malformed package ${JSON.stringify(s)}`);let c=a.arches;if(c.length===0||new Set(c).size!==c.length||c.some(d=>typeof d!="string"||!We.has(d)))throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(s)} has invalid arches`);let l=a.cacheKeys;if(!Re(l,c)||Object.values(l).some(d=>typeof d!="string"||!/^[a-f0-9]{64}$/.test(d)))throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(s)} has invalid cache keys`);let u=a.dependencyClosures;if(!Re(u,c))throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(s)} has invalid dependency closure arches`);let f={};for(let d of c){let m=u[d];if(!Array.isArray(m))throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(s)} has a malformed dependency closure for ${d}`);let p=new Set;f[d]=m.map((w,v)=>{if(typeof w!="object"||w===null||!Re(w,["packageName","manifestSha256","cacheKey"])||typeof w.packageName!="string"||typeof w.manifestSha256!="string"||!/^[a-f0-9]{64}$/.test(w.manifestSha256)||typeof w.cacheKey!="string"||!/^[a-f0-9]{64}$/.test(w.cacheKey))throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(s)} dependency ${v+1} for ${d} is malformed`);let S=w;if(fn(S.packageName,r,`${s} dependency packageName`,!1),S.packageName===s||p.has(S.packageName))throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(s)} dependency closure for ${d} must contain unique dependencies other than itself`);p.add(S.packageName);let b=t.get(S.packageName);if(!b||b.manifestSha256!==S.manifestSha256||b.cacheKeys[d]!==S.cacheKey)throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(s)} dependency ${JSON.stringify(S.packageName)} for ${d} does not match the index's authoritative contextual identity`);return S})}let h=a.members.map((d,m)=>{if(typeof d!="object"||d===null||d.kind!=="output"&&d.kind!=="runtime-file"||typeof d.sourceArtifact!="string"||typeof d.mirrorPath!="string")throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(s)} member ${m+1} is malformed`);let p=d,w=p.kind==="output"?["kind","sourceArtifact","mirrorPath","outputName","forkInstrumentation"]:["kind","sourceArtifact","mirrorPath","guestPath","mode"];if(!Re(p,w))throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(s)} member ${m+1} has unknown or missing fields`);if(Li(p.sourceArtifact,r,`${s} sourceArtifact`),Li(p.mirrorPath,r,`${s} mirrorPath`),p.kind==="output"){if(typeof p.outputName!="string"||p.forkInstrumentation!=="auto"&&p.forkInstrumentation!=="disabled")throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(s)} output member lacks outputName or forkInstrumentation`);fn(p.outputName,r,`${s} outputName`)}else if(typeof p.guestPath!="string"||!p.guestPath.startsWith("/")||!Number.isInteger(p.mode)||p.mode<0||p.mode>511)throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(s)} runtime member lacks valid guestPath or mode`);return p});if(h.length===0||new Set(h.map(d=>d.sourceArtifact)).size!==h.length||new Set(h.map(d=>d.mirrorPath)).size!==h.length||h.length===1&&h[0].mirrorPath.includes("/")||h.length>1&&h.some(d=>!d.mirrorPath.startsWith(`${s}/`)))throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(s)} members are empty, collide, or violate scalar/package-directory layout`);let y=a.manifestSha256,g=t.get(s);if(!g||g.manifestSha256!==y||c.some(d=>g.cacheKeys[d]!==l[d]))throw new Error(`Invalid program package index ${r}: package ${JSON.stringify(s)} does not match its contextual package identity`);i.set(s,{manifestSha256:y,arches:c,cacheKeys:l,dependencyClosures:f,members:h})}return{identities:t,packages:i,indexPath:r}}function $i(r){return JSON.stringify({manifestSha256:r.manifestSha256,arches:r.arches,cacheKeys:Object.fromEntries(r.arches.map(e=>[e,r.cacheKeys[e]])),dependencyClosures:Object.fromEntries(r.arches.map(e=>[e,[...r.dependencyClosures[e]].sort((t,n)=>t.packageNamen.packageName?1:0)])),members:r.members.map(e=>e.kind==="output"?{kind:e.kind,sourceArtifact:e.sourceArtifact,mirrorPath:e.mirrorPath,outputName:e.outputName,forkInstrumentation:e.forkInstrumentation}:{kind:e.kind,sourceArtifact:e.sourceArtifact,mirrorPath:e.mirrorPath,guestPath:e.guestPath,mode:e.mode})})}function hr(){let r=F(gn(),"wasm",we);return de(r)?lr(r):null}function ya(r){let e=hr();if(!e)return null;let t=r.split("/");if(t[0]!=="programs"||!We.has(t[1]))return null;let n=t[1];if(t.length>=4){let o=t[2];return e.packages.get(o)?.arches.includes(n)?o:null}if(t.length!==3)return null;let i=t[2];for(let[o,s]of e.packages)if(s.arches.includes(n)&&s.members.some(a=>a.kind==="output"&&a.mirrorPath.split("/").at(-1)===i))return o;return null}function Pi(r){let e=ya(r);if(e)throw new Error(`Installed package resolver path ${JSON.stringify(r)} is owned by ${JSON.stringify(e)}, but that package is not selected by the configured program registry`)}function Fi(){let r=Mi(),e=new Map,t=new Map,n=new Map,i=new Map,o=[];if(r===null){let l=F(gn(),"wasm",we);if(!de(l))return{identities:e,unidentifiedPackages:t,packages:n,unprojectedPackages:i,physicalProgramClaims:o};let u=lr(l);for(let[f,h]of u.identities)e.set(f,{...h,packageName:f,policyPath:`${u.indexPath}#identities.${f}`});for(let[f,h]of u.packages)o.push({packageName:f,projection:h,selected:!0}),n.set(f,{...h,packageName:f,policyPath:`${u.indexPath}#${f}`});return{identities:e,unidentifiedPackages:t,packages:n,unprojectedPackages:i,physicalProgramClaims:o}}let s=new Set,a=null,c=null;for(let l of r){if(!de(l))continue;if(!Ke(l).isDirectory())throw new Error(`Program registry root is not a directory: ${l}`);let u=F(l,we);if(!de(u))throw new Error(`Program registry ${l} is missing ${we}; generate it with xtask build-deps program-index`);let f=lr(u);a??=f.identities,c??=f.packages;let h=Jo(l,{withFileTypes:!0}).filter(y=>y.isDirectory()||y.isSymbolicLink()).sort((y,g)=>y.name.localeCompare(g.name));for(let y of h){let g=y.name,d=F(l,g,"package.toml");if(!de(d))continue;let m=!1;try{m=Ke(d).isFile()}catch{m=!1}if(!m)continue;let p=f.packages.get(g),w=!s.has(g);if(p&&o.push({packageName:g,projection:p,selected:w}),!w)continue;s.add(g);let v=a.get(g);v?e.set(g,{...v,packageName:g,manifestPath:d,policyPath:d}):t.set(g,d);let S=c.get(g);if(!S){i.set(g,d);continue}n.set(g,{...S,packageName:g,manifestPath:d,policyPath:d})}}return{identities:e,unidentifiedPackages:t,packages:n,unprojectedPackages:i,physicalProgramClaims:o}}function Oi(r){if(!r.manifestPath)return;let e;try{e=Ge(r.manifestPath)}catch(n){throw new Error(`Program package identity cannot verify ${r.manifestPath}: ${n instanceof Error?n.message:String(n)}`)}if(Bi("sha256").update(e).digest("hex")!==r.manifestSha256)throw new Error(`Program package identity is stale for ${r.manifestPath}; regenerate ${we}`)}function ga(r){if(!r.manifestPath)return;let e;try{e=Ge(r.manifestPath)}catch(n){throw new Error(`Program package projection cannot verify ${r.manifestPath}: ${n instanceof Error?n.message:String(n)}`)}if(Bi("sha256").update(e).digest("hex")!==r.manifestSha256)throw new Error(`Program package projection is stale for ${r.manifestPath}; regenerate ${we}`)}function Bt(r){let e=yr(),t=e.packages.get(r);if(t)return ga(t),t;let n=e.unprojectedPackages.get(r);if(n)throw new Error(`Package ${JSON.stringify(r)} is selected at ${n} but is absent from ${we}; regenerate the registry projection`);return null}function pa(r,e){let t=r.dependencyClosures[e];if(!t)throw dt(r.policyPath,`package ${JSON.stringify(r.packageName)} lacks a dependency identity closure for ${e}`);let n=Fi(),i=n.identities.get(r.packageName);if(!i){let s=n.unidentifiedPackages.get(r.packageName);throw new Error(`Program package ${JSON.stringify(r.packageName)} has no authoritative contextual identity for ${e}${s?` at ${s}`:""}; regenerate ${we} with the exact ordered registry roots`)}Oi(i);let o=i.cacheKeys[e];if(i.manifestSha256!==r.manifestSha256||o!==r.cacheKeys[e])throw new Error(`Program package ${JSON.stringify(r.packageName)} was projected with manifest ${r.manifestSha256} and cache key ${r.cacheKeys[e]} for ${e}, but the authoritative first-hit registry context at ${i.policyPath} requires manifest ${i.manifestSha256} and cache key ${o??""}. Regenerate this program projection with the exact ordered registry roots; the highest-priority index must carry the complete combined-context projection rather than relying on a lower suffix-context build identity.`);for(let s of t){let a=n.identities.get(s.packageName);if(!a){let l=n.unidentifiedPackages.get(s.packageName);throw l?new Error(`Program package ${JSON.stringify(r.packageName)} was generated against dependency ${JSON.stringify(s.packageName)}, but the first-hit package at ${l} has no contextual identity in ${we}`):new Error(`Program package ${JSON.stringify(r.packageName)} was generated against dependency ${JSON.stringify(s.packageName)}, but that dependency is absent from the configured first-hit registry roots`)}Oi(a);let c=a.cacheKeys[e];if(a.manifestSha256!==s.manifestSha256||c!==s.cacheKey)throw new Error(`Program package ${JSON.stringify(r.packageName)} has a contextual cache identity mismatch for ${e}: its projection expects dependency ${JSON.stringify(s.packageName)} manifest ${s.manifestSha256} and cache key ${s.cacheKey}, but first-hit selection at ${a.policyPath} provides manifest ${a.manifestSha256} and cache key ${c??""}. Regenerate the program projection with the exact ordered registry roots; the complete highest-priority projection must bind every selected program to the same combined dependency context.`)}}function yr(){let r=Fi(),{physicalProgramClaims:e,...t}=r,n={...t,legacyFlatOutputs:new Map,forkInstrumentationDisabledOutputs:new Map},i=[];for(let o of r.packages.values()){let s=o.members.length>1;for(let a of o.arches)for(let c of o.members){let l=i.find(y=>y.arch===a&&(y.path===c.mirrorPath||y.path.startsWith(`${c.mirrorPath}/`)||c.mirrorPath.startsWith(`${y.path}/`)));if(l)throw new Error(`Program resolver paths programs/${a}/${l.path} and programs/${a}/${c.mirrorPath} conflict between selected packages ${JSON.stringify(l.packageName)} and ${JSON.stringify(o.packageName)}`);if(i.push({arch:a,path:c.mirrorPath,packageName:o.packageName}),c.kind!=="output")continue;let u=c.mirrorPath.split("/").at(-1),f=`${a}/${u}`,h=n.legacyFlatOutputs.get(f);h||(h={scalarOwners:new Set,packagePaths:new Map,shadowedOwners:new Set},n.legacyFlatOutputs.set(f,h)),s?h.packagePaths.set(`programs/${a}/${c.mirrorPath}`,o.packageName):h.scalarOwners.add(o.packageName),c.forkInstrumentation==="disabled"&&n.forkInstrumentationDisabledOutputs.set(`${a}/${c.mirrorPath}`,o.packageName)}}for(let{packageName:o,projection:s,selected:a}of e)if(!(a&&r.packages.has(o)))for(let c of s.arches)for(let l of s.members){if(l.kind!=="output")continue;let u=l.mirrorPath.split("/").at(-1),f=`${c}/${u}`,h=n.legacyFlatOutputs.get(f);h||(h={scalarOwners:new Set,packagePaths:new Map,shadowedOwners:new Set},n.legacyFlatOutputs.set(f,h)),h.shadowedOwners.add(o)}return n}function ma(r){let e=r.split("/");if(e.length!==3||e[0]!=="programs"||!We.has(e[1]))return null;let t=yr().legacyFlatOutputs.get(`${e[1]}/${e[2]}`);if(!t)return null;for(let n of t.scalarOwners){let i=Bt(n);if(i)return i}for(let n of t.packagePaths.values())Bt(n);if(t.packagePaths.size>0)throw new Error(`Legacy flat resolver path ${JSON.stringify(r)} belongs to a multi-member package; use ${[...t.packagePaths.keys()].sort().map(n=>JSON.stringify(n)).join(" or ")}`);for(let n of t.shadowedOwners){let i=Bt(n);if(i)return i;throw new Error(`Legacy flat resolver path ${JSON.stringify(r)} is claimed by a lower-root program package ${JSON.stringify(n)}, but its first-hit selected package does not project that program; stale scalar mirror fallback is forbidden`)}return null}function Ti(r,e,t){if(!r.arches.includes(e))throw dt(r.policyPath,`package ${JSON.stringify(r.packageName)} does not declare resolver artifacts for ${e}`);let n=r.cacheKeys[e];if(!n)throw dt(r.policyPath,`package ${JSON.stringify(r.packageName)} lacks a cache identity for ${e}`);pa(r,e);let i=$i(r),o=r.members.map(s=>({packageName:r.packageName,relPath:`programs/${e}/${s.mirrorPath}`,sourceArtifact:s.sourceArtifact,cacheKey:n,forkInstrumentation:s.kind==="output"?s.forkInstrumentation??null:null,projectionIdentity:i}));if(!o.some(s=>s.relPath===t))throw dt(r.policyPath,`resolver path ${JSON.stringify(t)} is not a declared member of package ${JSON.stringify(r.packageName)}`);return{manifestPath:r.policyPath,packageName:r.packageName,members:o}}function wa(r){let e=Be(r),t=e.split("/");if(t[0]==="programs"&&!sa()&&hr()===null)throw new Error(`Installed host package is missing wasm/${we}; program artifacts cannot be resolved without packaged policy`);if(t.length===3){let s=ma(e);return s?Ti(s,t[1],e):(Pi(e),null)}if(t.length<4||t[0]!=="programs"||!We.has(t[1]))return null;let n=t[1],i=t[2],o=Bt(i);return o?Ti(o,n,e):(Pi(e),null)}function va(r){let e=Be(r);for(let t of["programs/wasm32/","programs/wasm64/"])if(e.startsWith(t))return e.slice(t.length);return null}function Ea(r){let e=Be(r);for(let t of We){let n=`programs/${t}/`;if(e.startsWith(n)){let i=yr().forkInstrumentationDisabledOutputs.get(`${t}/${e.slice(n.length)}`);return i?Bt(i)!==null:!1}}return!1}function Sa(r){let e=Be(r);if(e==="kernel.wasm")return wr;let t=va(e);if(t&&t.endsWith(".wasm"))return ra}function za(r,e,t){if(!r.endsWith(".wasm"))return!1;try{let n=Ge(r),i=n.buffer.slice(n.byteOffset,n.byteOffset+n.byteLength),o=t===void 0?Ea(e):t==="disabled";return vr(i,{expectedAbi:41,requiredExports:Sa(e),requireForkInstrumentation:o?!1:void 0,forbidForkInstrumentation:o}).length>0}catch{return!0}}function ba(r){if(!r.endsWith(".vfs")&&!r.endsWith(".vfs.zst"))return!1;try{let t=an.readImageMetadata(Ge(r))?.kernelAbi;return t!==void 0&&t!==41}catch{return!0}}function gr(r,e,t){return za(r,e,t)||ba(r)}function Di(r,e,t){let n=r.filter(de);return n.length===0?null:n.find(i=>{try{return Ke(i).isFile()&&!gr(i,e,t)}catch{return!1}})??null}function Ui(r,e,t){try{if(!hn(r).isSymbolicLink())return r;let i=pe(r);if(!Ke(i).isFile()||gr(i,e,t))throw new Error("canonical target is not an accepted regular file");if(Be(e).startsWith("programs/")&&ka(i))throw new Error("resolver-owned program generation has no matching selected package projection");return i}catch(n){throw new Error(`Binary changed or became invalid while pinning ${e}: ${n instanceof Error?n.message:String(n)}`)}}function ka(r){let e=[Ci()];try{e.push(F(Ze(),"local-binaries",".kandelo-local-generations"))}catch{}return e.some(t=>{try{return de(t)&&pr(pe(t),r)}catch{return!1}})}function pr(r,e){let t=ea(r,e);return t===""||t!==".."&&!t.startsWith(`..${ta}`)&&!yn(t)}function xa(r,e){let t=e.split("/"),n=r;for(let i=0;ia.packageName!==o))return"declared package members do not share a valid program namespace";if(!Ke(e).isDirectory())return"shared package generation root is not a directory";let s=t[0].cacheKey;if(!/^[a-f0-9]{64}$/.test(s)||t.some(a=>a.cacheKey!==s))return"declared package members do not share one valid cache identity";if(r.identity==="local-generation"){let a=F(r.root,".kandelo-local-generations",i,o,s);if(!de(a))return"local mirror targets are not one direct immutable local generation";let c=pe(a);return Ct(e)===c?null:"local mirror targets are not one direct immutable local generation"}if(r.identity==="program-cache"){let a=Ci();if(!de(a))return"fetched mirror targets are not one canonical program-cache generation";let c=pe(a),l=Qo(e),u=l.startsWith(`${o}-`)&&new RegExp(`-rev[0-9]+-${i}-${s}$`).test(l);return Ct(e)===c&&u?null:"fetched mirror targets are not one canonical program-cache generation"}return"installed-package symlink closures are not an immutable installed identity"}function La(r,e,t){if(e.length!==t.length)return{failure:"internal member/path count mismatch"};try{let n=e.map(l=>{let u=hn(l);return u.isSymbolicLink()?"symlink":u.isFile()?"file":"other"});if(n.includes("other"))return{failure:"a selected mirror member is neither a regular file nor a symlink"};let i=n.every(l=>l==="symlink"),o=n.every(l=>l==="file");if(!i&&!o)return{failure:"regular files and symlinks cannot share one package identity"};if(o){if(!r.allowRegularFileClosure)return{failure:"a mutable source-checkout wasm tree is not an installed package identity"};let l=t[0].packageName,u=t[0].projectionIdentity;if(t.some(d=>d.packageName!==l||d.projectionIdentity!==u))return{failure:"declared members do not share one selected package projection"};let h=hr()?.packages.get(l);if(!h||$i(h)!==u)return{failure:"installed bytes do not match the selected package projection"};let y=pe(r.root),g=[];for(let d of e){let m=pe(d);if(!pr(y,m)||!Ke(m).isFile())return{failure:"an installed-package member escapes its immutable wasm tree"};g.push(m)}return{paths:g}}let s=null,a=[];for(let l=0;lAa(r))}function Aa(r){let e=Be(r),t=wa(e);if(t){let s=_a(t.members.map(a=>a.relPath),t.members);if(s)return s[t.members.findIndex(a=>a.relPath===e)];throw new un(`Package artifacts not found for ${t.packageName}: ${e}`)}let n=[],i=[];for(let s of Ni())for(let a of s.candidatesFor(r))n.push(a),i.push(a);let o=Di(i,r);if(o)return Ui(o,r);throw i.some(de)?new Error(`Binary exists but was rejected by artifact policy: ${r} +`+n.map(s=>` checked: ${s}`).join(` +`)):new un(`Binary not found: ${r} +`+n.map(s=>` checked: ${s}`).join(` +`)+` + Run scripts/fetch-binaries.sh, place a file at local-binaries/${e}, or install a package that includes wasm/${r}.`)}function _a(r,e){if(r.length===0)return[];let t=!1,n=[];for(let i of Ni()){let o=[],s=[];if(e){let[a,c,l]=e[0].relPath.split("/");a==="programs"&&c&&l&&(t||=de(F(i.root,a,c,l)))}for(let[a,c]of r.entries()){let l=i.candidatesFor(c),u=l.filter(de);t||=u.length>0;let f=Di(l,c,e?.[a]?.forkInstrumentation);f?o.push(f):u.length>0?s.push(`${c} (rejected by artifact policy)`):s.push(`${c} (missing)`)}if(s.length===0&&e){let a=La(i,o,e);if("failure"in a)s.push(`shared package identity rejected: ${a.failure}`);else{let c=a.paths.flatMap((l,u)=>gr(l,r[u],e[u].forkInstrumentation)?[r[u]]:[]);if(c.length>0)s.push(`pinned package generation rejected by artifact policy: ${c.join(", ")}`);else return a.paths}}if(s.length===0)return o.map((a,c)=>Ui(a,r[c],e?.[c]?.forkInstrumentation));n.push(` ${i.label} (${i.root}): ${s.join(", ")}`)}if(!t)return null;throw new Error(`Package artifact closure is incomplete: no single provenance tier contains every accepted artifact, and tiers will not be mixed. +`+n.join(` +`))}var[Ki,...Pa]=process.argv.slice(2);(!Ki||Pa.length>0)&&(console.error("usage: scripts/resolve-binary.sh "),process.exit(2));try{process.stdout.write(`${Gi(Ki)} +`)}catch(r){console.error(r instanceof Error?r.message:String(r)),process.exit(1)} diff --git a/scripts/resolve-binary.sh b/scripts/resolve-binary.sh index fc86eb3b42..0082215137 100755 --- a/scripts/resolve-binary.sh +++ b/scripts/resolve-binary.sh @@ -1,265 +1,64 @@ #!/usr/bin/env bash # -# Resolve a binary relative to the binaries/ tree. Priority: -# 1. $REPO/local-binaries/ (user override unless it is a legacy -# fork artifact and fetched release is fresh) -# 2. $REPO/binaries/ (fetched release) -# -# Prints the absolute path on stdout, or prints a helpful error to -# stderr and exits 1. -# -# This is the shell-script equivalent of host/src/binary-resolver.ts. -# Keep them in sync. +# Resolve one repository artifact through the same TypeScript resolver used by +# Node and browser build tooling. The Rust-generated program package projection +# therefore governs shell, TypeScript, external registries, and installed host +# packages without a second manifest parser. # # Usage: -# $(scripts/resolve-binary.sh kernel.wasm) -# $(scripts/resolve-binary.sh programs/dash.wasm) -# $(scripts/resolve-binary.sh vfs/shell.vfs.zst) +# scripts/resolve-binary.sh kernel.wasm +# scripts/resolve-binary.sh programs/dash.wasm +# scripts/resolve-binary.sh programs/cpython/python.wasm set -euo pipefail script_dir="$(cd "$(dirname "$0")" && pwd)" -source "$script_dir/wasm-artifact-guards.sh" - if [ $# -ne 1 ] || [ "$1" = "-h" ] || [ "$1" = "--help" ]; then - sed -n '3,18p' "$0" + sed -n '3,12p' "$0" exit 0 fi -rel="$1" -repo_root="$(git rev-parse --show-toplevel 2>/dev/null || true)" -if [ -z "$repo_root" ]; then - # Fall back to walking up from $PWD looking for the workspace - # Cargo.toml + abi/snapshot.json (a unique pair only present at - # this repo's root). - dir="$(pwd)" - while [ "$dir" != "/" ]; do - if [ -f "$dir/Cargo.toml" ] && [ -f "$dir/abi/snapshot.json" ]; then - repo_root="$dir" - break - fi - dir="$(dirname "$dir")" - done -fi -if [ -z "$repo_root" ]; then - echo "ERROR: could not find repo root" >&2 - exit 1 -fi - -# Default-arch shim: callers historically pass `programs/` without -# an arch segment (run.sh has 30+ `has_resolvable programs/.wasm` -# checks). After the per-arch layout refactor, those files live at -# `programs/wasm32/` (or `wasm64/`). Inject `wasm32/` when the -# caller's path starts with `programs/` and the next segment isn't -# already `wasm32` or `wasm64`. Mirrors the same shim in -# host/src/binary-resolver.ts (applyDefaultArch). -adjusted="$rel" -case "$rel" in - programs/wasm32/*|programs/wasm64/*) ;; # explicit arch — pass through - programs/*) - adjusted="programs/wasm32/${rel#programs/}" - ;; -esac - -local_path="$repo_root/local-binaries/$adjusted" -fetched_path="$repo_root/binaries/$adjusted" -current_abi="$(wasm_current_abi_version "$repo_root" || true)" - -fork_instrumentation_for_rel() { - local rel="$1" - case "$rel" in - programs/wasm32/*) rel="${rel#programs/wasm32/}" ;; - programs/wasm64/*) rel="${rel#programs/wasm64/}" ;; - programs/*) rel="${rel#programs/}" ;; - *) echo none; return 0 ;; - esac - - local manifest policy - for manifest in "$repo_root"/packages/registry/*/package.toml; do - [ -f "$manifest" ] || continue - policy="$(awk -v target="$rel" ' - function val(s) { - sub(/^[^=]*=[ \t]*"/, "", s) - sub(/".*$/, "", s) - return s - } - function ext(path, parts, n, base, dot) { - n = split(path, parts, "/") - base = parts[n] - dot = index(base, ".") - return dot ? substr(base, dot) : "" - } - function flush() { - if (!in_output) return - count++ - output_name[count] = out_name - output_wasm[count] = out_wasm - output_policy[count] = out_policy - out_name = "" - out_wasm = "" - out_policy = "" - in_output = 0 - } - $0 ~ /^\[\[outputs\]\]/ { - flush() - in_output = 1 - in_root = 0 - next - } - in_output && $0 ~ /^\[/ { - flush() - in_root = 0 - next - } - !in_output && $0 ~ /^\[/ { - in_root = 0 - next - } - BEGIN { - in_root = 1 - } - in_root && $0 ~ /^kind[ \t]*=/ { kind = val($0); next } - in_root && $0 ~ /^name[ \t]*=/ { pkg = val($0); next } - in_output && $0 ~ /^name[ \t]*=/ { out_name = val($0); next } - in_output && $0 ~ /^wasm[ \t]*=/ { out_wasm = val($0); next } - in_output && $0 ~ /^fork_instrumentation[ \t]*=/ { out_policy = val($0); next } - END { - flush() - if (kind != "program" || pkg == "") exit - for (i = 1; i <= count; i++) { - if (output_name[i] == "" || output_wasm[i] == "") continue - dest = output_name[i] ext(output_wasm[i]) - if (count > 1) dest = pkg "/" dest - if (dest == target && output_policy[i] == "disabled") { - print "disabled" - exit - } - } - } - ' "$manifest")" - if [ "$policy" = "disabled" ]; then - echo disabled - return 0 - fi - done - echo auto -} - -fork_instrumentation="$(fork_instrumentation_for_rel "$adjusted")" - -kernel_required_exports=( - __abi_version - kernel_alloc_scratch - kernel_create_process - kernel_create_process_with_stdio - kernel_get_parent_pid - kernel_get_process_state - kernel_handle_channel - kernel_has_sa_nocldstop - kernel_host_adapter_manifest_len - kernel_host_adapter_manifest_ptr - kernel_mark_process_signaled - kernel_reap_exited_child - kernel_remove_process - kernel_wait_child_poll -) - -executable_program_required_exports=( - __abi_version - _start -) - -is_executable_program_wasm_rel() { - case "$adjusted" in - programs/wasm32/*.wasm|programs/wasm32/*/*.wasm|programs/wasm64/*.wasm|programs/wasm64/*/*.wasm) - return 0 - ;; - *) - return 1 - ;; - esac -} - -is_stale_wasm_artifact() { - wasm_has_legacy_asyncify "$1" || - wasm_has_stale_abi "$1" "$current_abi" || - { [ "$adjusted" = "kernel.wasm" ] && wasm_has_missing_exports "$1" "${kernel_required_exports[@]}"; } || - { is_executable_program_wasm_rel && wasm_has_missing_exports "$1" "${executable_program_required_exports[@]}"; } || - case "$fork_instrumentation" in - none) - false - ;; - disabled) - wasm_has_any_wpk_fork_export "$1" - ;; - *) - wasm_has_missing_fork_instrumentation "$1" - ;; - esac -} - -has_stale_vfs_abi() { - local status=0 - [ -n "$current_abi" ] || return 0 - command -v node >/dev/null 2>&1 || return 0 - node "$script_dir/vfs-has-stale-abi.mjs" "$1" "$current_abi" \ - >/dev/null 2>&1 || status=$? - - # Status 1 is the only accepted result: it means the image either matches - # or carries no ABI declaration (legacy/data-only VFS). Explicit mismatches - # and uninspectable metadata both fail closed. - [ "$status" -ne 1 ] -} - -has_artifact_policy_failures() { - case "$adjusted" in - *.wasm) - # A Wasm-named artifact must remain fail-closed even when its bytes - # are malformed or the structural decoder cannot inspect it. - is_stale_wasm_artifact "$1" - ;; - *.vfs|*.vfs.zst) - has_stale_vfs_abi "$1" - ;; - *) - # Fetched archives, Wasm side modules, and declared runtime data - # are authenticated by package materialization before entering - # binaries/. They have no executable Wasm ABI/export contract; - # local-binaries/ remains the user's explicit override tier. - false - ;; - esac -} - -if [ -e "$local_path" ]; then - if [ -e "$fetched_path" ] \ - && has_artifact_policy_failures "$local_path" \ - && ! has_artifact_policy_failures "$fetched_path"; then - echo "$fetched_path" - exit 0 +repo_root="$(cd "$script_dir/.." && pwd)" +cd "$repo_root" + +# Source checkouts verify their generated program-package projection with the +# exact Rust manifest/cache-key implementation before Node consumes it. Prepare +# the release xtask once and pass its path into the bundled resolver; installed +# host packages and deliberately minimal resolver fixtures have no xtask source +# tree and continue to use their pack-time-verified bundled projection. +checker_root="${WASM_POSIX_BINARY_RESOLVER_REPO_ROOT:-$repo_root}" +if [[ "$1" == programs/* ]] && + [ -z "${WASM_POSIX_XTASK_BIN:-}" ] && + [ -f "$checker_root/tools/xtask/Cargo.toml" ] && + [ -f "$checker_root/scripts/dev-shell.sh" ]; then + if [ -n "${KANDELO_DEV_SHELL_TOOL_PATH:-}" ]; then + host_target="$(rustc -vV | awk '/^host:/ {print $2}')" + else + host_target="$( + bash "$checker_root/scripts/dev-shell.sh" rustc -vV | + awk '/^host:/ {print $2}' + )" fi - if has_artifact_policy_failures "$local_path"; then - echo "ERROR: stale or invalid artifact ignored: $local_path" >&2 - echo " Rebuild it for ABI ${current_abi:-current}, fetch a fresh release, or remove the stale local override." >&2 + if [ -z "$host_target" ]; then + echo "resolve-binary: could not determine the Rust host target" >&2 exit 1 fi - echo "$local_path" - exit 0 -fi -if [ -e "$fetched_path" ]; then - if has_artifact_policy_failures "$fetched_path"; then - echo "ERROR: stale or invalid artifact ignored: $fetched_path" >&2 - echo " Rebuild it for ABI ${current_abi:-current} or fetch a fresh release." >&2 - exit 1 + WASM_POSIX_XTASK_BIN="$checker_root/target/$host_target/release/xtask" + # An existing path may belong to an older source state. Cargo's + # incremental no-op is the exact preparation check for this invocation. + if [ -n "${KANDELO_DEV_SHELL_TOOL_PATH:-}" ]; then + ( + cd "$checker_root" + cargo build --release -p xtask --target "$host_target" --quiet + ) + else + ( + cd "$checker_root" + bash scripts/dev-shell.sh \ + cargo build --release -p xtask --target "$host_target" --quiet + ) >&2 fi - echo "$fetched_path" - exit 0 + export WASM_POSIX_XTASK_BIN fi -cat >&2 < 0) { + console.error("usage: scripts/resolve-binary.sh "); + process.exit(2); +} + +try { + process.stdout.write(`${resolveBinary(relPath)}\n`); +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +} diff --git a/scripts/run-browser-mariadb-tests.sh b/scripts/run-browser-mariadb-tests.sh index 5a0f1224d1..c0beff9431 100755 --- a/scripts/run-browser-mariadb-tests.sh +++ b/scripts/run-browser-mariadb-tests.sh @@ -18,8 +18,8 @@ KERNEL_WASM="$("$REPO_ROOT/scripts/resolve-binary.sh" kernel.wasm)" VFS_IMAGE="$REPO_ROOT/apps/browser-demos/public/mariadb-test.vfs.zst" RUNNER="$REPO_ROOT/scripts/browser-mariadb-test-runner.ts" -# ── Curated tests (from full browser triage of all 1184 tests) ── -# 185 tests verified to pass in headless Chromium with MariaDB on kandelo. +# ── Curated tests (from full browser triage of all 1183 tests) ── +# 184 tests verified to pass in headless Chromium with MariaDB on kandelo. # Excludes: 230 connect-command tests (deadlock with no-threads), 339 timeouts, # 143 self-skipping, 287 other failures. CURATED_TESTS=( @@ -69,7 +69,7 @@ CURATED_TESTS=( set_statement_notembedded show_create_user show_function_with_pad_char_to_full_length show_row_order-9226 signal_demo1 signal_demo2 signal_demo3 - signal_sqlmode simple_select single_delete_update + signal_sqlmode single_delete_update skip_log_bin sp-bugs2 sp-condition-handler sp-destruct sp-memory-leak sp-no-code sp-no-valgrind sp-ucs2 sp-vars sp_gis sp_missing_4665 sql_mode_pad_char_to_full_length diff --git a/scripts/test-homebrew-main-shell-closure.sh b/scripts/test-homebrew-main-shell-closure.sh index 4a1f737c98..e4735b5c19 100755 --- a/scripts/test-homebrew-main-shell-closure.sh +++ b/scripts/test-homebrew-main-shell-closure.sh @@ -92,6 +92,11 @@ for required_path in \ "scripts/install-local-binary.sh" \ "scripts/install-overlay-headers.sh" \ "scripts/resolve-binary.sh" \ + "scripts/resolve-binary.ts" \ + "scripts/resolve-binary.bundle.mjs" \ + "scripts/resolve-binary.bundle.LICENSES.txt" \ + "scripts/build-resolve-binary-bundle.sh" \ + "scripts/test-resolve-binary-bundle.sh" \ "scripts/recover-homebrew-bottle-mirror.ts" \ "scripts/run-wasm-fork-instrument.sh" \ "scripts/verify-homebrew-main-shell-artifact-lock.sh" \ @@ -191,8 +196,42 @@ grep -Fq -- '--lazy-shell \' "$WORKFLOW" || fail "candidate proof must explicitly opt into lazy shell composition" grep -Fq 'scripts/build-homebrew-main-shell-closure.sh \' "$WORKFLOW" || fail "candidate proof must invoke the strict shell composer" -grep -Fq 'cp "$CANDIDATE_PATH" "$installed"' "$WORKFLOW" || - fail "candidate proof must install the exact candidate bytes for browser resolution" +candidate_install_workflow_block="$(sed -n \ + "/- name: Install the candidate's exact shell bytes/,/- name: Recover the exact bottle mirror/p" \ + "$WORKFLOW")" +grep -Fq 'WASM_POSIX_LOCAL_INSTALL_SOURCE="$1"' \ + <<<"$candidate_install_workflow_block" || + fail "candidate proof must give the exact candidate to the package installer" +grep -Fq 'WASM_POSIX_LOCAL_INSTALL_SESSION="$2"' \ + <<<"$candidate_install_workflow_block" || + fail "candidate proof must give the package installer an explicit session" +grep -Fq 'bash "$CANDIDATE_PATH" "$install_session"' \ + <<<"$candidate_install_workflow_block" || + fail "candidate and session must be passed into the installer shell as isolated arguments" +grep -Fq '${GITHUB_RUN_ID}' <<<"$candidate_install_workflow_block" && + grep -Fq '${GITHUB_RUN_ATTEMPT}' <<<"$candidate_install_workflow_block" && + grep -Fq '${GITHUB_JOB}' <<<"$candidate_install_workflow_block" || + fail "candidate package-install session must be unique to one workflow job attempt" +grep -Fq 'build-deps --arch wasm32 --binaries-dir local-binaries \' \ + <<<"$candidate_install_workflow_block" || + fail "candidate proof must publish through the wasm32 local package installer" +grep -Fq 'install-local-artifact shell shell.vfs.zst' \ + <<<"$candidate_install_workflow_block" || + fail "candidate proof must install shell.vfs.zst as a declared shell artifact" +grep -Fq 'resolved=$(bash scripts/resolve-binary.sh programs/shell.vfs.zst)' \ + <<<"$candidate_install_workflow_block" || + fail "candidate proof must resolve the canonical installed shell artifact" +grep -Fq 'cmp "$CANDIDATE_PATH" "$resolved"' \ + <<<"$candidate_install_workflow_block" || + fail "candidate proof must compare the canonical installed artifact with the candidate" +grep -Fq 'cp "$CANDIDATE_PATH" "$browser_copy"' \ + <<<"$candidate_install_workflow_block" || + fail "candidate proof must retain a separate browser-public copy" +[ "$(grep -Fc 'local-binaries' <<<"$candidate_install_workflow_block")" -eq 1 ] || + fail "candidate proof must access local-binaries only through the package installer" +grep -Eq '(^|[[:space:]])(cp|mv|install|ln)[[:space:]].*(local-binaries|\$installed)' \ + <<<"$candidate_install_workflow_block" && + fail "candidate proof must not write or copy directly into local-binaries" grep -Fq -- '--image "${{ steps.candidate.outputs.image }}"' "$WORKFLOW" || fail "Node proof must boot the exact candidate bytes directly" grep -Fq -- '--migration-lock homebrew/main-shell-migration-lock.json' "$WORKFLOW" || diff --git a/scripts/test-install-local-binary-sealed.sh b/scripts/test-install-local-binary-sealed.sh index 534ee736c7..2d4c557a6a 100755 --- a/scripts/test-install-local-binary-sealed.sh +++ b/scripts/test-install-local-binary-sealed.sh @@ -37,6 +37,29 @@ for tool in rustc cargo; do 'exit 97' > "$fake_bin/$tool" chmod +x "$fake_bin/$tool" done +export SEALED_REAL_LN +SEALED_REAL_LN="$(command -v ln)" +printf '%s\n' \ + '#!/usr/bin/env bash' \ + 'set -euo pipefail' \ + 'case "${SEALED_LN_ATTACK:-}" in' \ + ' fail)' \ + ' exit 91' \ + ' ;;' \ + ' replace-stage)' \ + ' "$SEALED_REAL_LN" "$@"' \ + ' rm -f "$1"' \ + ' printf "replacement-stage\\n" >"$1"' \ + ' ;;' \ + ' modify-backup)' \ + ' "$SEALED_REAL_LN" "$@"' \ + ' printf "changed-backup-contents\\n" >"$(dirname "$1")/backup"' \ + ' ;;' \ + ' *)' \ + ' exec "$SEALED_REAL_LN" "$@"' \ + ' ;;' \ + 'esac' >"$fake_bin/ln" +chmod +x "$fake_bin/ln" chmod -R a-w "$fake_repo" "$source_dir" @@ -79,4 +102,151 @@ if ( fi [ ! -e "$work/escape.zip" ] +# Nested destination ancestors are never followed. +attack_out="$work/attack-output" +outside="$work/outside" +mkdir -p "$attack_out" "$outside" +printf 'outside-sentinel\n' >"$outside/sentinel" +ln -s "$outside" "$attack_out/share" +if ( + source "$fake_repo/scripts/install-local-binary.sh" + WASM_POSIX_DEP_OUT_DIR="$attack_out" \ + WASM_POSIX_INSTALL_LOCAL_MIRROR=0 \ + install_local_runtime_file \ + cpython "$source_dir/python-runtime.zip" share/runtime.zip +) >/dev/null 2>&1; then + echo "test-install-local-binary-sealed.sh: followed a nested destination symlink" >&2 + exit 1 +fi +[ "$(cat "$outside/sentinel")" = "outside-sentinel" ] +[ ! -e "$outside/runtime.zip" ] + +# The authorized root itself may not be a symlink. +linked_out="$work/linked-output" +ln -s "$outside" "$linked_out" +if ( + source "$fake_repo/scripts/install-local-binary.sh" + WASM_POSIX_DEP_OUT_DIR="$linked_out" \ + WASM_POSIX_INSTALL_LOCAL_MIRROR=0 \ + install_local_runtime_file \ + cpython "$source_dir/python-runtime.zip" runtime.zip +) >/dev/null 2>&1; then + echo "test-install-local-binary-sealed.sh: accepted a symlink output root" >&2 + exit 1 +fi +[ ! -e "$outside/runtime.zip" ] + +# The packaging-only shell path is intentionally scoped to a caller-owned, +# single-writer scratch tree. A group/other-writable output root is rejected +# before a transaction or destination can be created. +shared_out="$work/shared-output" +mkdir "$shared_out" +chmod 0777 "$shared_out" +if ( + source "$fake_repo/scripts/install-local-binary.sh" + PATH="$fake_bin:$PATH" \ + WASM_POSIX_DEP_OUT_DIR="$shared_out" \ + WASM_POSIX_INSTALL_LOCAL_MIRROR=0 \ + install_local_runtime_file \ + cpython "$source_dir/python-runtime.zip" runtime.zip +) >/dev/null 2>&1; then + echo "test-install-local-binary-sealed.sh: accepted a shared-writer output root" >&2 + exit 1 +fi +[ ! -e "$shared_out/runtime.zip" ] +[ -z "$(find "$shared_out" -mindepth 1 -maxdepth 1 -print -quit)" ] +chmod 0700 "$shared_out" + +# A failed create-once publication restores an unchanged previous destination. +# The failed transaction remains as evidence instead of using recursive or +# unverified cleanup. +rollback_out="$work/rollback-output" +mkdir "$rollback_out" +printf 'previous-runtime\n' >"$rollback_out/runtime.zip" +if ( + source "$fake_repo/scripts/install-local-binary.sh" + export PATH="$fake_bin:$PATH" + export SEALED_LN_ATTACK=fail + export WASM_POSIX_DEP_OUT_DIR="$rollback_out" + export WASM_POSIX_INSTALL_LOCAL_MIRROR=0 + install_local_runtime_file \ + cpython "$source_dir/python-runtime.zip" runtime.zip +) >/dev/null 2>&1; then + echo "test-install-local-binary-sealed.sh: ignored a publication failure" >&2 + exit 1 +fi +[ "$(cat "$rollback_out/runtime.zip")" = "previous-runtime" ] +rollback_transactions=("$rollback_out"/.kandelo-install.*) +[ "${#rollback_transactions[@]}" -eq 1 ] +[ -f "${rollback_transactions[0]}/stage" ] + +# If another same-user process violates the single-writer contract and swaps +# the staged pathname after linking it, identity checks refuse to delete the +# replacement. This is defense in depth around the documented contract. +stage_attack_out="$work/stage-attack-output" +mkdir "$stage_attack_out" +if ( + source "$fake_repo/scripts/install-local-binary.sh" + export PATH="$fake_bin:$PATH" + export SEALED_LN_ATTACK=replace-stage + export WASM_POSIX_DEP_OUT_DIR="$stage_attack_out" + export WASM_POSIX_INSTALL_LOCAL_MIRROR=0 + install_local_runtime_file \ + cpython "$source_dir/python-runtime.zip" runtime.zip +) >/dev/null 2>&1; then + echo "test-install-local-binary-sealed.sh: accepted a substituted staged path" >&2 + exit 1 +fi +stage_attack_transactions=("$stage_attack_out"/.kandelo-install.*) +[ "${#stage_attack_transactions[@]}" -eq 1 ] +[ "$(cat "${stage_attack_transactions[0]}/stage")" = "replacement-stage" ] +cmp "$source_dir/python-runtime.zip" "$stage_attack_out/runtime.zip" + +# Content checks are independent of inode checks: mutating a quarantined +# regular file in place is detected and the changed bytes are preserved. +backup_attack_out="$work/backup-attack-output" +mkdir "$backup_attack_out" +printf 'previous-runtime\n' >"$backup_attack_out/runtime.zip" +if ( + source "$fake_repo/scripts/install-local-binary.sh" + export PATH="$fake_bin:$PATH" + export SEALED_LN_ATTACK=modify-backup + export WASM_POSIX_DEP_OUT_DIR="$backup_attack_out" + export WASM_POSIX_INSTALL_LOCAL_MIRROR=0 + install_local_runtime_file \ + cpython "$source_dir/python-runtime.zip" runtime.zip +) >/dev/null 2>&1; then + echo "test-install-local-binary-sealed.sh: deleted a changed quarantine" >&2 + exit 1 +fi +backup_attack_transactions=("$backup_attack_out"/.kandelo-install.*) +[ "${#backup_attack_transactions[@]}" -eq 1 ] +[ "$(cat "${backup_attack_transactions[0]}/backup")" = "changed-backup-contents" ] +cmp "$source_dir/python-runtime.zip" "$backup_attack_out/runtime.zip" + +# Sealed executable installs require an explicit reviewed policy. +if ( + source "$fake_repo/scripts/install-local-binary.sh" + WASM_POSIX_DEP_OUT_DIR="$out_dir" \ + WASM_POSIX_INSTALL_LOCAL_MIRROR=0 \ + install_local_binary cpython "$source_dir/python.wasm" +) >/dev/null 2>&1; then + echo "test-install-local-binary-sealed.sh: accepted an implicit sealed fork policy" >&2 + exit 1 +fi + +# Source symlinks are rejected before any destination is touched. +linked_source="$work/linked-source.wasm" +ln -s "$source_dir/python.wasm" "$linked_source" +if ( + source "$fake_repo/scripts/install-local-binary.sh" + WASM_POSIX_DEP_OUT_DIR="$out_dir" \ + WASM_POSIX_INSTALL_LOCAL_MIRROR=0 \ + WASM_POSIX_INSTALL_FORK_INSTRUMENTATION=auto \ + install_local_binary cpython "$linked_source" +) >/dev/null 2>&1; then + echo "test-install-local-binary-sealed.sh: accepted a symlink source" >&2 + exit 1 +fi + echo "test-install-local-binary-sealed.sh: ok" diff --git a/scripts/test-install-local-generation.sh b/scripts/test-install-local-generation.sh new file mode 100755 index 0000000000..f65fdb5991 --- /dev/null +++ b/scripts/test-install-local-generation.sh @@ -0,0 +1,214 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +HOST_TARGET="$(rustc -vV | awk '/^host/ {print $2}')" +[ -n "$HOST_TARGET" ] || { + echo "test-install-local-generation.sh: rustc did not report a host target" >&2 + exit 1 +} + +# This test is also invoked from package-build tests that intentionally export +# caller-owned output variables. Its direct-local scenarios must not inherit +# those sealed-build settings. +unset WASM_POSIX_DEP_OUT_DIR +unset WASM_POSIX_DEP_TARGET_ARCH +unset WASM_POSIX_INSTALL_LOCAL_MIRROR +unset WASM_POSIX_LOCAL_INSTALL_SESSION + +work="$(mktemp -d)" +cleanup() { + chmod -R u+w "$work" 2>/dev/null || true + rm -rf "$work" +} +trap cleanup EXIT + +fail() { + echo "test-install-local-generation.sh: $*" >&2 + exit 1 +} + +registry="$work/registry" +package_dir="$registry/local-python" +mirror="$work/local-binaries" +fetched="$work/fetched-cache" +source_dir="$work/build-output" +mkdir -p "$package_dir" "$mirror/programs/wasm32/local-python" \ + "$fetched/bin" "$fetched/share" "$source_dir" + +cat >"$package_dir/package.toml" <<'EOF' +kind = "program" +name = "local-python" +version = "1.0" +depends_on = [] + +[source] +url = "https://example.test/local-python.tar.gz" +sha256 = "0000000000000000000000000000000000000000000000000000000000000000" + +[license] +spdx = "MIT" + +[[outputs]] +name = "python" +wasm = "bin/python.wasm" + +[[runtime_files]] +artifact = "share/python-runtime.zip" +guest_path = "/usr/share/local-python/python-runtime.zip" +EOF + +# Minimal executables export the two normal program entry points. Distinct +# custom sections make fetched and local bytes observably different while +# retaining valid Wasm. +printf '\000asm\001\000\000\000\001\005\001\140\000\001\177\003\002\001\000\007\032\002\015__abi_version\000\000\006_start\000\000\012\006\001\004\000\101\000\013\000\006\005fetch' \ + >"$fetched/bin/python.wasm" +printf '\000asm\001\000\000\000\001\005\001\140\000\001\177\003\002\001\000\007\032\002\015__abi_version\000\000\006_start\000\000\012\006\001\004\000\101\000\013\000\006\005local' \ + >"$source_dir/python.wasm" +printf 'FETCHED-RUNTIME\n' >"$fetched/share/python-runtime.zip" +printf 'LOCAL-RUNTIME\n' >"$source_dir/python-runtime.zip" + +ln -s "$fetched/bin/python.wasm" \ + "$mirror/programs/wasm32/local-python/python.wasm" +mkdir -p "$mirror/programs/wasm32/local-python/share" +ln -s "$fetched/share/python-runtime.zip" \ + "$mirror/programs/wasm32/local-python/share/python-runtime.zip" +fetched_wasm_before="$(shasum -a 256 "$fetched/bin/python.wasm" | awk '{print $1}')" +fetched_runtime_before="$(shasum -a 256 "$fetched/share/python-runtime.zip" | awk '{print $1}')" + +run_install() { + local artifact="$1" + local source="$2" + ( + cd "$REPO_ROOT" + WASM_POSIX_DEPS_REGISTRY="$registry" \ + WASM_POSIX_LOCAL_INSTALL_SOURCE="$source" \ + WASM_POSIX_LOCAL_INSTALL_SESSION=direct-build-one \ + cargo run -p xtask --target "$HOST_TARGET" --quiet -- \ + build-deps --arch wasm32 --binaries-dir "$mirror" \ + install-local-artifact local-python "$artifact" + ) +} + +cache_key="$( + cd "$REPO_ROOT" + WASM_POSIX_DEPS_REGISTRY="$registry" \ + cargo run -p xtask --target "$HOST_TARGET" --quiet -- \ + build-deps --arch wasm32 sha local-python +)" +printf '%s\n' "$cache_key" | grep -Eq '^[0-9a-f]{64}$' || + fail "resolver did not return a full local generation cache identity" + +first_log="$work/first.log" +run_install python.wasm "$source_dir/python.wasm" >"$first_log" +grep -F 'waiting for 1 declared package artifact' "$first_log" >/dev/null || + fail "first closure member was reported as fully installed" +cmp "$fetched/bin/python.wasm" \ + "$mirror/programs/wasm32/local-python/python.wasm" >/dev/null || + fail "incomplete local generation changed the live executable" +cmp "$fetched/share/python-runtime.zip" \ + "$mirror/programs/wasm32/local-python/share/python-runtime.zip" >/dev/null || + fail "incomplete local generation changed the live runtime file" + +generation="$mirror/.kandelo-local-generations/wasm32/local-python/$cache_key/direct-build-one" +cmp "$source_dir/python.wasm" "$generation/bin/python.wasm" >/dev/null || + fail "local output was not collected at its exact declared suffix" +[ ! -e "$generation/share/python-runtime.zip" ] || + fail "incomplete generation synthesized a missing runtime file" + +second_log="$work/second.log" +run_install share/python-runtime.zip "$source_dir/python-runtime.zip" >"$second_log" +grep -F 'from complete local generation' "$second_log" >/dev/null || + fail "complete closure was not reported as published" +cmp "$source_dir/python.wasm" \ + "$mirror/programs/wasm32/local-python/python.wasm" >/dev/null || + fail "complete local generation did not publish the executable" +cmp "$source_dir/python-runtime.zip" \ + "$mirror/programs/wasm32/local-python/share/python-runtime.zip" >/dev/null || + fail "complete local generation did not publish the runtime file" +generation_physical="$(cd "$generation" && pwd -P)" +[ "$(readlink "$mirror/programs/wasm32/local-python/python.wasm")" = \ + "$generation_physical/bin/python.wasm" ] || + fail "live executable does not target the exact declared generation suffix" +[ "$(readlink "$mirror/programs/wasm32/local-python/share/python-runtime.zip")" = \ + "$generation_physical/share/python-runtime.zip" ] || + fail "live runtime file does not target the exact declared generation suffix" +[ "$fetched_wasm_before" = \ + "$(shasum -a 256 "$fetched/bin/python.wasm" | awk '{print $1}')" ] || + fail "direct build overwrote fetched canonical executable bytes" +[ "$fetched_runtime_before" = \ + "$(shasum -a 256 "$fetched/share/python-runtime.zip" | awk '{print $1}')" ] || + fail "direct build overwrote fetched canonical runtime bytes" + +# Undeclared package names must fail before changing either source bytes or a +# destination. A fake repo makes the negative path disposable. +fake_repo="$work/fake-repo" +fake_bin="$work/fake-bin" +legacy_canonical="$work/legacy-canonical.wasm" +legacy_source="$work/legacy-source.wasm" +mkdir -p "$fake_repo/scripts" \ + "$fake_repo/local-binaries/programs/wasm32" "$fake_bin" +cp "$REPO_ROOT/scripts/install-local-binary.sh" "$fake_repo/scripts/" +cp "$REPO_ROOT/scripts/wasm-artifact-guards.sh" "$fake_repo/scripts/" +cp "$fetched/bin/python.wasm" "$legacy_canonical" +cp "$source_dir/python.wasm" "$legacy_source" +ln -s "$legacy_canonical" \ + "$fake_repo/local-binaries/programs/wasm32/legacy-alias.wasm" +cat >"$fake_bin/rustc" <<'EOF' +#!/usr/bin/env bash +printf 'host: fake-test-target\n' +EOF +cat >"$fake_bin/cargo" <<'EOF' +#!/usr/bin/env bash +printf 'fixture manifest lookup failed\n' >&2 +exit 19 +EOF +chmod +x "$fake_bin/rustc" "$fake_bin/cargo" +legacy_before="$(shasum -a 256 "$legacy_canonical" | awk '{print $1}')" +legacy_source_before="$(shasum -a 256 "$legacy_source" | awk '{print $1}')" +legacy_err="$work/legacy.err" +if ( + PATH="$fake_bin:$PATH" + WASM_POSIX_INSTALL_FORK_INSTRUMENTATION=disabled + export PATH WASM_POSIX_INSTALL_FORK_INSTRUMENTATION + # shellcheck source=/dev/null + source "$fake_repo/scripts/install-local-binary.sh" + install_local_binary legacy-alias "$legacy_source" +) 2>"$legacy_err"; then + fail "undeclared package was installed through a guessed path" +fi +legacy_dest="$fake_repo/local-binaries/programs/wasm32/legacy-alias.wasm" +[ -L "$legacy_dest" ] || + fail "failed package lookup changed its existing mirror" +[ "$legacy_before" = "$(shasum -a 256 "$legacy_canonical" | awk '{print $1}')" ] || + fail "failed package lookup mutated canonical cache bytes" +[ "$legacy_source_before" = "$(shasum -a 256 "$legacy_source" | awk '{print $1}')" ] || + fail "failed package lookup mutated source bytes before resolving policy" +grep -F "does not uniquely declare output" "$legacy_err" >/dev/null || + fail "undeclared package lookup failure was not explained" + +# A selected malformed package also fails at the same pre-mutation lookup. +mkdir -p "$fake_repo/packages/registry/registered" +printf 'malformed = [\n' >"$fake_repo/packages/registry/registered/package.toml" +registered_dest="$fake_repo/local-binaries/programs/wasm32/registered.wasm" +ln -s "$legacy_canonical" "$registered_dest" +registered_err="$work/registered.err" +if ( + PATH="$fake_bin:$PATH" + WASM_POSIX_INSTALL_FORK_INSTRUMENTATION=disabled + export PATH WASM_POSIX_INSTALL_FORK_INSTRUMENTATION + # shellcheck source=/dev/null + source "$fake_repo/scripts/install-local-binary.sh" + install_local_binary registered "$legacy_source" +) 2>"$registered_err"; then + fail "registered package lookup failure fell through to the legacy copy path" +fi +grep -F "package 'registered' does not uniquely declare output" \ + "$registered_err" >/dev/null || + fail "registered package lookup failure was not explained" +[ -L "$registered_dest" ] || + fail "registered package lookup failure changed its existing mirror" +[ "$legacy_before" = "$(shasum -a 256 "$legacy_canonical" | awk '{print $1}')" ] || + fail "registered package lookup failure mutated canonical cache bytes" + +echo "test-install-local-generation.sh: ok" diff --git a/scripts/test-package-build-roots.sh b/scripts/test-package-build-roots.sh index 5aeff9d74a..5cdbcc1257 100755 --- a/scripts/test-package-build-roots.sh +++ b/scripts/test-package-build-roots.sh @@ -3,6 +3,12 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +HOST_TARGET="$(rustc -vV | awk '/^host/ {print $2}')" +cargo run -p xtask --target "$HOST_TARGET" --quiet -- \ + build-deps program-index-check \ + "$REPO_ROOT/packages/registry" \ + "$REPO_ROOT/packages/registry/program-packages.json" + TMP_ROOT="$(mktemp -d)" cleanup() { chmod -R u+w "$TMP_ROOT" 2>/dev/null || true @@ -323,6 +329,7 @@ grep -F "artifact must be a portable relative path" "$err" >/dev/null || fail "unsafe caller-owned runtime artifact escaped its output root" bash "$REPO_ROOT/scripts/test-graphics-pkgconfig.sh" +bash "$REPO_ROOT/scripts/test-install-local-generation.sh" # Every exact-shell registry recipe must enter through this tested root # contract. Their real package builds remain separate bottle/dry-run evidence. diff --git a/scripts/test-pages-deployment-contract.sh b/scripts/test-pages-deployment-contract.sh index 218fdeb50d..0b160281a6 100755 --- a/scripts/test-pages-deployment-contract.sh +++ b/scripts/test-pages-deployment-contract.sh @@ -80,6 +80,21 @@ expect_mutation_rejected \ "does not watch docs-site/**" \ 's/^ - "docs-site\/\*\*"\n//m' +expect_mutation_rejected \ + "missing browser package scanner trigger" \ + "does not watch scripts/browser-binary-package-roots.mjs" \ + 's/^ - "scripts\/browser-binary-package-roots\.mjs"\n//m' + +expect_mutation_rejected \ + "missing package-registry trigger" \ + "does not watch packages/registry/**" \ + 's/^ - "packages\/registry\/\*\*"\n//m' + +expect_mutation_rejected \ + "bypassed package projection check" \ + "must verify the generated package projection" \ + 's/build-deps program-index-check/build-deps parse/' + expect_mutation_rejected \ "checkout of a different ref" \ "checkout must use the workflow event source SHA" \ diff --git a/scripts/test-resolve-binary-bundle.sh b/scripts/test-resolve-binary-bundle.sh new file mode 100755 index 0000000000..e5b250d653 --- /dev/null +++ b/scripts/test-resolve-binary-bundle.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "$0")/.." && pwd)" +committed="$repo_root/scripts/resolve-binary.bundle.mjs" +generated_dir="$(mktemp -d "${TMPDIR:-/tmp}/kandelo-resolver-bundle.XXXXXX")" +generated="$generated_dir/resolve-binary.bundle.mjs" +trap 'rm -rf "$generated_dir"' EXIT + +if [ ! -f "$committed" ] || [ -L "$committed" ]; then + echo "test-resolve-binary-bundle: committed bundle must be a regular non-symlink file" >&2 + exit 1 +fi + +bash "$repo_root/scripts/build-resolve-binary-bundle.sh" "$generated" >/dev/null +if ! cmp "$committed" "$generated" >/dev/null; then + echo "test-resolve-binary-bundle: scripts/resolve-binary.bundle.mjs is stale" >&2 + echo " regenerate it with: bash scripts/build-resolve-binary-bundle.sh" >&2 + exit 1 +fi +load_err="${generated}.err" +if node "$generated" 2>"$load_err"; then + echo "test-resolve-binary-bundle: no-argument standalone bundle unexpectedly succeeded" >&2 + exit 1 +fi +if ! grep -F "usage: scripts/resolve-binary.sh" "$load_err" >/dev/null; then + echo "test-resolve-binary-bundle: standalone copy could not load without node_modules" >&2 + sed -n '1,20p' "$load_err" >&2 + exit 1 +fi + +echo "test-resolve-binary-bundle: ok" diff --git a/tests/package-system/browser-binary-dependencies.test.ts b/tests/package-system/browser-binary-dependencies.test.ts index d9dbb6961f..4017d92156 100644 --- a/tests/package-system/browser-binary-dependencies.test.ts +++ b/tests/package-system/browser-binary-dependencies.test.ts @@ -1,11 +1,25 @@ -import { existsSync, readdirSync, readFileSync, statSync } from "node:fs"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { createHash } from "node:crypto"; +import { tmpdir } from "node:os"; import { dirname, join, relative, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { describe, expect, it } from "vitest"; import { + browserBinariesImports, browserBinaryPackageRoots, + configuredProgramRegistryRoots, firstTomlString, inspectBrowserBinaryDependencies, + packageOutputOwners, registryPackagesWithoutBuildToml, } from "../../scripts/browser-binary-package-roots.mjs"; @@ -18,6 +32,85 @@ function registryPackageDirs(): string[] { .filter((path) => existsSync(join(path, "package.toml"))); } +interface ContextRegistryEntry { + manifest: string; + cacheKeys: { wasm32: string; wasm64: string }; + projection?: { + dependencyClosures: Record>; + mirrorPath: string; + }; +} + +function contextDependencyIdentity( + packageName: string, + manifest: string, + cacheKey: string, +) { + return { + packageName, + manifestSha256: createHash("sha256").update(manifest).digest("hex"), + cacheKey, + }; +} + +function writeContextRegistry( + root: string, + entries: Record, + contextualEntries: Record = {}, +): void { + const identities: Record = {}; + const packages: Record = {}; + const addProjection = ( + packageName: string, + entry: ContextRegistryEntry, + ) => { + const manifestSha256 = createHash("sha256") + .update(entry.manifest) + .digest("hex"); + identities[packageName] = { + manifestSha256, + cacheKeys: entry.cacheKeys, + }; + if (entry.projection) { + packages[packageName] = { + manifestSha256, + arches: ["wasm32"], + cacheKeys: { wasm32: entry.cacheKeys.wasm32 }, + dependencyClosures: entry.projection.dependencyClosures, + members: [{ + kind: "output", + sourceArtifact: entry.projection.mirrorPath, + mirrorPath: entry.projection.mirrorPath, + outputName: entry.projection.mirrorPath.replace(/\.wasm$/, ""), + forkInstrumentation: "auto", + }], + }; + } + }; + for (const [packageName, entry] of Object.entries(contextualEntries)) { + addProjection(packageName, entry); + } + for (const [packageName, entry] of Object.entries(entries)) { + const packageDir = join(root, packageName); + mkdirSync(packageDir, { recursive: true }); + writeFileSync(join(packageDir, "package.toml"), entry.manifest); + writeFileSync(join(packageDir, "build.toml"), "revision = 1\n"); + addProjection(packageName, entry); + } + writeFileSync( + join(root, "program-packages.json"), + `${JSON.stringify({ + format: "kandelo-program-packages-v2", + identities, + packages, + }, null, 2)}\n`, + ); +} + describe("browser binary dependencies", () => { it("requires a build.toml sidecar for every fetchable registry package", () => { const missingBuildToml = registryPackageDirs() @@ -42,6 +135,834 @@ describe("browser binary dependencies", () => { expect(unfetchableOwners).toEqual([]); }); + it("discovers syntax-level imports without treating generated source strings as imports", () => { + const fixtureRoot = mkdtempSync(join(tmpdir(), "kandelo-browser-imports-")); + try { + const browserRoot = join(fixtureRoot, "apps", "browser-demos"); + mkdirSync(browserRoot, { recursive: true }); + writeFileSync( + join(browserRoot, "imports.ts"), + [ + "const generated = `import ignored from \"@binaries/programs/ignored.wasm?url\";`;", + "import actual from \"@binaries/programs/actual.wasm?url\";", + "const dynamic = import(\"@binaries/programs/dynamic.wasm?url\");", + "void generated; void actual; void dynamic;", + ].join("\n"), + ); + expect(browserBinariesImports(fixtureRoot)).toEqual([ + "programs/wasm32/actual.wasm", + "programs/wasm32/dynamic.wasm", + ]); + } finally { + rmSync(fixtureRoot, { recursive: true, force: true }); + } + }); + + it("uses repo-anchored external registries with first-hit browser ownership", () => { + const fixtureRoot = mkdtempSync( + join(tmpdir(), "kandelo-browser-external-registry-"), + ); + try { + const browserRoot = join(fixtureRoot, "apps", "browser-demos"); + const externalRoot = join(fixtureRoot, "third-party", "registry"); + const fallbackRoot = join(fixtureRoot, "packages", "registry"); + const packageName = "external-runtime"; + const manifest = (version: string) => `kind = "program" +name = "${packageName}" +version = "${version}" +depends_on = [] +[source] +url = "https://example.test/${packageName}-${version}.tar.gz" +sha256 = "${"0".repeat(64)}" +[license] +spdx = "MIT" +[[outputs]] +name = "${version === "external" ? "external-command" : "fallback-command"}" +wasm = "${version}.wasm" +`; + const writeRegistryPackage = ( + root: string, + version: string, + mirrorPath: string, + ) => { + const packageDir = join(root, packageName); + mkdirSync(packageDir, { recursive: true }); + const text = manifest(version); + writeFileSync(join(packageDir, "package.toml"), text); + writeFileSync(join(packageDir, "build.toml"), 'revision = 1\n'); + writeFileSync( + join(root, "program-packages.json"), + `${JSON.stringify({ + format: "kandelo-program-packages-v2", + identities: { + [packageName]: { + manifestSha256: createHash("sha256").update(text).digest("hex"), + cacheKeys: { + wasm32: "a".repeat(64), + wasm64: "b".repeat(64), + }, + }, + }, + packages: { + [packageName]: { + manifestSha256: createHash("sha256").update(text).digest("hex"), + arches: ["wasm32"], + cacheKeys: { wasm32: "a".repeat(64) }, + dependencyClosures: { wasm32: [] }, + members: [{ + kind: "output", + sourceArtifact: `${version}.wasm`, + mirrorPath, + outputName: mirrorPath.replace(/\.wasm$/, ""), + forkInstrumentation: "auto", + }], + }, + }, + }, null, 2)}\n`, + ); + }; + mkdirSync(browserRoot, { recursive: true }); + writeFileSync( + join(browserRoot, "entry.ts"), + 'import binary from "@binaries/programs/wasm32/external-command.wasm?url";\nvoid binary;\n', + ); + writeRegistryPackage(externalRoot, "external", "external-command.wasm"); + writeRegistryPackage(fallbackRoot, "fallback", "fallback-command.wasm"); + const registryPath = "third-party/registry:packages/registry"; + + expect(configuredProgramRegistryRoots(fixtureRoot, registryPath)).toEqual([ + externalRoot, + fallbackRoot, + ]); + const audit = inspectBrowserBinaryDependencies(fixtureRoot, { + registryPath, + }); + expect(audit.missingOwners).toEqual([]); + expect(audit.unfetchableOwners).toEqual([]); + expect(audit.packageNames).toEqual([packageName]); + const owners = packageOutputOwners(fixtureRoot, { registryPath }); + expect(owners.get("programs/wasm32/external-command.wasm")).toMatchObject({ + packageName, + hasBuildToml: true, + }); + expect(owners.has("programs/wasm32/fallback-command.wasm")).toBe(false); + } finally { + rmSync(fixtureRoot, { recursive: true, force: true }); + } + }); + + it("does not expose a lower program when a higher non-program package owns its name", () => { + const fixtureRoot = mkdtempSync( + join(tmpdir(), "kandelo-browser-non-program-shadow-"), + ); + try { + const browserRoot = join(fixtureRoot, "apps", "browser-demos"); + const upperRoot = join(fixtureRoot, "external"); + const lowerRoot = join(fixtureRoot, "main"); + const packageName = "shadowed-program"; + const outputPath = "shadowed-command.wasm"; + const upperManifest = + `kind = "library"\nname = "${packageName}"\nversion = "2.0.0"\n`; + const lowerManifest = + `kind = "program"\nname = "${packageName}"\nversion = "1.0.0"\n`; + writeContextRegistry(upperRoot, { + [packageName]: { + manifest: upperManifest, + cacheKeys: { + wasm32: "1".repeat(64), + wasm64: "2".repeat(64), + }, + }, + }); + writeContextRegistry(lowerRoot, { + [packageName]: { + manifest: lowerManifest, + cacheKeys: { + wasm32: "3".repeat(64), + wasm64: "4".repeat(64), + }, + projection: { + dependencyClosures: { wasm32: [] }, + mirrorPath: outputPath, + }, + }, + }); + mkdirSync(browserRoot, { recursive: true }); + writeFileSync( + join(browserRoot, "entry.ts"), + `import command from "@binaries/programs/wasm32/${outputPath}?url";\nvoid command;\n`, + ); + + const audit = inspectBrowserBinaryDependencies(fixtureRoot, { + registryPath: "external:main", + }); + expect(audit.missingOwners).toEqual([ + `programs/wasm32/${outputPath}`, + ]); + expect(() => + browserBinaryPackageRoots(fixtureRoot, { + registryPath: "external:main", + }) + ).toThrow(/browser @binaries imports without registry owners/); + } finally { + rmSync(fixtureRoot, { recursive: true, force: true }); + } + }); + + it("accepts identical first-hit identities and order-insensitive external dependency closures", () => { + const fixtureRoot = mkdtempSync( + join(tmpdir(), "kandelo-browser-context-safe-"), + ); + try { + const browserRoot = join(fixtureRoot, "apps", "browser-demos"); + const upperRoot = join(fixtureRoot, "external"); + const lowerRoot = join(fixtureRoot, "main"); + const sharedName = "z-shared-dependency"; + const auxiliaryName = "a-auxiliary-dependency"; + const externalName = "external-context-program"; + const lowerName = "lower-context-program"; + const sharedManifest = + `kind = "library"\nname = "${sharedName}"\nversion = "1.0.0"\n`; + const auxiliaryManifest = + `kind = "source"\nname = "${auxiliaryName}"\nversion = "1.0.0"\n`; + const externalManifest = + `kind = "program"\nname = "${externalName}"\nversion = "1.0.0"\n`; + const lowerManifest = + `kind = "program"\nname = "${lowerName}"\nversion = "1.0.0"\n`; + const sharedKeys = { + wasm32: "1".repeat(64), + wasm64: "2".repeat(64), + }; + const auxiliaryKeys = { + wasm32: "3".repeat(64), + wasm64: "3".repeat(64), + }; + const sharedIdentity = contextDependencyIdentity( + sharedName, + sharedManifest, + sharedKeys.wasm32, + ); + const auxiliaryIdentity = contextDependencyIdentity( + auxiliaryName, + auxiliaryManifest, + auxiliaryKeys.wasm32, + ); + writeContextRegistry( + upperRoot, + { + [sharedName]: { manifest: sharedManifest, cacheKeys: sharedKeys }, + [auxiliaryName]: { + manifest: auxiliaryManifest, + cacheKeys: auxiliaryKeys, + }, + [externalName]: { + manifest: externalManifest, + cacheKeys: { + wasm32: "4".repeat(64), + wasm64: "5".repeat(64), + }, + projection: { + // Deliberately reverse lexical order: closure order is not policy. + dependencyClosures: { + wasm32: [sharedIdentity, auxiliaryIdentity], + }, + mirrorPath: "external-context.wasm", + }, + }, + }, + { + [lowerName]: { + manifest: lowerManifest, + cacheKeys: { + wasm32: "6".repeat(64), + wasm64: "7".repeat(64), + }, + projection: { + dependencyClosures: { wasm32: [sharedIdentity] }, + mirrorPath: "lower-context.wasm", + }, + }, + }, + ); + writeContextRegistry(lowerRoot, { + [sharedName]: { manifest: sharedManifest, cacheKeys: sharedKeys }, + [lowerName]: { + manifest: lowerManifest, + cacheKeys: { + wasm32: "6".repeat(64), + wasm64: "7".repeat(64), + }, + projection: { + dependencyClosures: { wasm32: [sharedIdentity] }, + mirrorPath: "lower-context.wasm", + }, + }, + }); + mkdirSync(browserRoot, { recursive: true }); + writeFileSync( + join(browserRoot, "entry.ts"), + [ + 'import external from "@binaries/programs/wasm32/external-context.wasm?url";', + 'import lower from "@binaries/programs/wasm32/lower-context.wasm?url";', + "void external; void lower;", + ].join("\n"), + ); + const registryPath = "external:main"; + + const audit = inspectBrowserBinaryDependencies(fixtureRoot, { + registryPath, + }); + expect(audit.missingOwners).toEqual([]); + expect(audit.unfetchableOwners).toEqual([]); + expect(audit.packageNames).toEqual([externalName, lowerName]); + } finally { + rmSync(fixtureRoot, { recursive: true, force: true }); + } + }); + + it("accepts an external browser program generated against a changed transitive external:main context", () => { + const fixtureRoot = mkdtempSync( + join(tmpdir(), "kandelo-browser-external-transitive-"), + ); + try { + const browserRoot = join(fixtureRoot, "apps", "browser-demos"); + const upperRoot = join(fixtureRoot, "external"); + const lowerRoot = join(fixtureRoot, "main"); + const leafName = "external-transitive-leaf"; + const middleName = "external-transitive-middle"; + const externalName = "external-transitive-program"; + const lowerLeafManifest = + `kind = "source" +name = "${leafName}" +version = "1.0.0" +depends_on = [] +[source] +url = "https://example.test/${leafName}.tar.gz" +sha256 = "${"0".repeat(64)}" +[license] +spdx = "MIT" +`; + const upperLeafManifest = lowerLeafManifest.replace( + "0".repeat(64), + "1".repeat(64), + ); + const middleManifest = + `kind = "library"\nname = "${middleName}"\nversion = "1.0.0"\ndepends_on = ["${leafName}@1.0.0"]\n`; + const externalManifest = + `kind = "program"\nname = "${externalName}"\nversion = "1.0.0"\ndepends_on = ["${middleName}@1.0.0"]\n`; + const upperLeafKeys = { + wasm32: "1".repeat(64), + wasm64: "2".repeat(64), + }; + const combinedMiddleKeys = { + wasm32: "3".repeat(64), + wasm64: "4".repeat(64), + }; + writeContextRegistry( + upperRoot, + { + [leafName]: { + manifest: upperLeafManifest, + cacheKeys: upperLeafKeys, + }, + [externalName]: { + manifest: externalManifest, + cacheKeys: { + wasm32: "5".repeat(64), + wasm64: "6".repeat(64), + }, + projection: { + dependencyClosures: { + wasm32: [ + contextDependencyIdentity( + leafName, + upperLeafManifest, + upperLeafKeys.wasm32, + ), + contextDependencyIdentity( + middleName, + middleManifest, + combinedMiddleKeys.wasm32, + ), + ], + }, + mirrorPath: "external-transitive.wasm", + }, + }, + }, + { + [middleName]: { + manifest: middleManifest, + cacheKeys: combinedMiddleKeys, + }, + }, + ); + writeContextRegistry(lowerRoot, { + [leafName]: { + manifest: lowerLeafManifest, + cacheKeys: { + wasm32: "7".repeat(64), + wasm64: "8".repeat(64), + }, + }, + [middleName]: { + manifest: middleManifest, + cacheKeys: { + wasm32: "9".repeat(64), + wasm64: "a".repeat(64), + }, + }, + }); + mkdirSync(browserRoot, { recursive: true }); + writeFileSync( + join(browserRoot, "entry.ts"), + 'import external from "@binaries/programs/wasm32/external-transitive.wasm?url";\nvoid external;\n', + ); + + const audit = inspectBrowserBinaryDependencies(fixtureRoot, { + registryPath: "external:main", + }); + expect(audit.missingOwners).toEqual([]); + expect(audit.unfetchableOwners).toEqual([]); + expect(audit.packageNames).toEqual([externalName]); + + writeFileSync( + join(lowerRoot, middleName, "package.toml"), + `${middleManifest}build_input = "changed-after-projection"\n`, + ); + expect(() => + inspectBrowserBinaryDependencies(fixtureRoot, { + registryPath: "external:main", + }) + ).toThrow(/stale generated package identity.*external-transitive-middle/); + } finally { + rmSync(fixtureRoot, { recursive: true, force: true }); + } + }); + + it("uses the complete top projection for a lower browser program with a direct dependency override", () => { + const fixtureRoot = mkdtempSync( + join(tmpdir(), "kandelo-browser-context-direct-"), + ); + try { + const upperRoot = join(fixtureRoot, "external"); + const lowerRoot = join(fixtureRoot, "main"); + const dependencyName = "direct-shadow-dependency"; + const programName = "direct-shadow-program"; + const lowerDependencyManifest = + `kind = "library" +name = "${dependencyName}" +version = "1.0.0" +depends_on = [] +[source] +url = "https://example.test/${dependencyName}.tar.gz" +sha256 = "${"0".repeat(64)}" +[license] +spdx = "MIT" +`; + const upperDependencyManifest = lowerDependencyManifest.replace( + "0".repeat(64), + "1".repeat(64), + ); + const programManifest = + `kind = "program"\nname = "${programName}"\nversion = "1.0.0"\ndepends_on = ["${dependencyName}@1.0.0"]\n`; + const upperDependencyIdentity = contextDependencyIdentity( + dependencyName, + upperDependencyManifest, + "8".repeat(64), + ); + writeContextRegistry( + upperRoot, + { + [dependencyName]: { + manifest: upperDependencyManifest, + cacheKeys: { + wasm32: "8".repeat(64), + wasm64: "9".repeat(64), + }, + }, + }, + { + [programName]: { + manifest: programManifest, + cacheKeys: { + wasm32: "e".repeat(64), + wasm64: "f".repeat(64), + }, + projection: { + dependencyClosures: { + wasm32: [upperDependencyIdentity], + }, + mirrorPath: "direct-shadow.wasm", + }, + }, + }, + ); + writeContextRegistry(lowerRoot, { + [dependencyName]: { + manifest: lowerDependencyManifest, + cacheKeys: { + wasm32: "a".repeat(64), + wasm64: "b".repeat(64), + }, + }, + [programName]: { + manifest: programManifest, + cacheKeys: { + wasm32: "c".repeat(64), + wasm64: "d".repeat(64), + }, + projection: { + dependencyClosures: { + wasm32: [contextDependencyIdentity( + dependencyName, + lowerDependencyManifest, + "a".repeat(64), + )], + }, + mirrorPath: "direct-shadow.wasm", + }, + }, + }); + + const combinedOwners = packageOutputOwners(fixtureRoot, { + registryPath: "external:main", + }); + expect( + combinedOwners.get("programs/wasm32/direct-shadow.wasm"), + ).toMatchObject({ packageName: programName }); + + const fallbackOwners = packageOutputOwners(fixtureRoot, { + registryPath: "main", + }); + expect( + fallbackOwners.get("programs/wasm32/direct-shadow.wasm"), + ).toMatchObject({ packageName: programName }); + } finally { + rmSync(fixtureRoot, { recursive: true, force: true }); + } + }); + + it("uses the complete top projection for a lower browser program with a transitive dependency override", () => { + const fixtureRoot = mkdtempSync( + join(tmpdir(), "kandelo-browser-context-transitive-"), + ); + try { + const upperRoot = join(fixtureRoot, "external"); + const lowerRoot = join(fixtureRoot, "main"); + const leafName = "transitive-shadow-source"; + const intermediateName = "transitive-shadow-library"; + const programName = "transitive-shadow-program"; + const lowerLeafManifest = + `kind = "source" +name = "${leafName}" +version = "1.0.0" +depends_on = [] +[source] +url = "https://example.test/${leafName}.tar.gz" +sha256 = "${"0".repeat(64)}" +[license] +spdx = "MIT" +`; + const upperLeafManifest = lowerLeafManifest.replace( + "0".repeat(64), + "1".repeat(64), + ); + const intermediateManifest = + `kind = "library"\nname = "${intermediateName}"\nversion = "1.0.0"\ndepends_on = ["${leafName}@1.0.0"]\n`; + const programManifest = + `kind = "program"\nname = "${programName}"\nversion = "1.0.0"\ndepends_on = ["${intermediateName}@1.0.0"]\n`; + const leafIdentity = contextDependencyIdentity( + leafName, + lowerLeafManifest, + "e".repeat(64), + ); + const intermediateIdentity = contextDependencyIdentity( + intermediateName, + intermediateManifest, + "f".repeat(64), + ); + const combinedLeafIdentity = contextDependencyIdentity( + leafName, + upperLeafManifest, + "0".repeat(64), + ); + const combinedIntermediateIdentity = contextDependencyIdentity( + intermediateName, + intermediateManifest, + "4".repeat(64), + ); + writeContextRegistry( + upperRoot, + { + [leafName]: { + manifest: upperLeafManifest, + cacheKeys: { + wasm32: "0".repeat(64), + wasm64: "0".repeat(64), + }, + }, + }, + { + [intermediateName]: { + manifest: intermediateManifest, + cacheKeys: { + wasm32: "4".repeat(64), + wasm64: "5".repeat(64), + }, + }, + [programName]: { + manifest: programManifest, + cacheKeys: { + wasm32: "6".repeat(64), + wasm64: "7".repeat(64), + }, + projection: { + dependencyClosures: { + wasm32: [ + combinedIntermediateIdentity, + combinedLeafIdentity, + ], + }, + mirrorPath: "transitive-shadow.wasm", + }, + }, + }, + ); + writeContextRegistry(lowerRoot, { + [leafName]: { + manifest: lowerLeafManifest, + cacheKeys: { + wasm32: leafIdentity.cacheKey, + wasm64: leafIdentity.cacheKey, + }, + }, + [intermediateName]: { + manifest: intermediateManifest, + cacheKeys: { + wasm32: intermediateIdentity.cacheKey, + wasm64: "1".repeat(64), + }, + }, + [programName]: { + manifest: programManifest, + cacheKeys: { + wasm32: "2".repeat(64), + wasm64: "3".repeat(64), + }, + projection: { + dependencyClosures: { + wasm32: [intermediateIdentity, leafIdentity], + }, + mirrorPath: "transitive-shadow.wasm", + }, + }, + }); + + const owners = packageOutputOwners(fixtureRoot, { + registryPath: "external:main", + }); + expect( + owners.get("programs/wasm32/transitive-shadow.wasm"), + ).toMatchObject({ packageName: programName }); + } finally { + rmSync(fixtureRoot, { recursive: true, force: true }); + } + }); + + it("fails closed for absent, stale, or widened external projections", () => { + const fixtureRoot = mkdtempSync( + join(tmpdir(), "kandelo-browser-invalid-projection-"), + ); + try { + const browserRoot = join(fixtureRoot, "apps", "browser-demos"); + const registry = join(fixtureRoot, "registry"); + const packageName = "external-command-package"; + const packageDir = join(registry, packageName); + const manifest = `kind = "program" +name = "${packageName}" +version = "1.0.0" +[[outputs]] +name = "external-command" +wasm = "external-command.wasm" +`; + mkdirSync(browserRoot, { recursive: true }); + mkdirSync(packageDir, { recursive: true }); + writeFileSync( + join(browserRoot, "entry.ts"), + 'import command from "@binaries/programs/wasm32/external-command.wasm?url";\nvoid command;\n', + ); + writeFileSync(join(packageDir, "package.toml"), manifest); + writeFileSync(join(packageDir, "build.toml"), "revision = 1\n"); + const projection = { + manifestSha256: createHash("sha256").update(manifest).digest("hex"), + arches: ["wasm32"], + cacheKeys: { wasm32: "a".repeat(64) }, + dependencyClosures: { wasm32: [] }, + members: [{ + kind: "output", + sourceArtifact: "external-command.wasm", + mirrorPath: "external-command.wasm", + outputName: "external-command", + forkInstrumentation: "auto", + }], + }; + const writeIndex = (packageProjection?: Record) => { + writeFileSync( + join(registry, "program-packages.json"), + `${JSON.stringify({ + format: "kandelo-program-packages-v2", + identities: { + [packageName]: { + manifestSha256: + (packageProjection?.manifestSha256 as string | undefined) + ?? createHash("sha256").update(manifest).digest("hex"), + cacheKeys: { + wasm32: + (packageProjection?.cacheKeys as Record | undefined) + ?.wasm32 ?? "a".repeat(64), + wasm64: "b".repeat(64), + }, + }, + }, + packages: packageProjection + ? { [packageName]: packageProjection } + : {}, + }, null, 2)}\n`, + ); + }; + + writeIndex(); + expect( + inspectBrowserBinaryDependencies(fixtureRoot, { + registryPath: "registry", + }).missingOwners, + ).toEqual(["programs/wasm32/external-command.wasm"]); + + writeIndex({ ...projection, manifestSha256: "b".repeat(64) }); + expect(() => + inspectBrowserBinaryDependencies(fixtureRoot, { + registryPath: "registry", + }) + ).toThrow(/stale generated package identity/); + + writeIndex({ + ...projection, + members: [{ + ...projection.members[0], + unexpectedPolicy: true, + }], + }); + expect(() => + inspectBrowserBinaryDependencies(fixtureRoot, { + registryPath: "registry", + }) + ).toThrow(/invalid generated program package projection member/); + } finally { + rmSync(fixtureRoot, { recursive: true, force: true }); + } + }); + + it("rejects cross-package file and directory ownership conflicts", () => { + const fixtureRoot = mkdtempSync( + join(tmpdir(), "kandelo-browser-owner-conflict-"), + ); + try { + const registry = join(fixtureRoot, "registry"); + mkdirSync(registry, { recursive: true }); + const manifests: Record = { + "scalar-owner": `kind = "program" +name = "scalar-owner" +version = "1.0.0" +[[outputs]] +name = "shared" +wasm = "shared" +`, + shared: `kind = "program" +name = "shared" +version = "1.0.0" +[[outputs]] +name = "child" +wasm = "child.wasm" +[[runtime_files]] +artifact = "data" +guest_path = "/usr/share/data" +`, + }; + for (const [name, manifest] of Object.entries(manifests)) { + const packageDir = join(registry, name); + mkdirSync(packageDir, { recursive: true }); + writeFileSync(join(packageDir, "package.toml"), manifest); + writeFileSync(join(packageDir, "build.toml"), "revision = 1\n"); + } + const base = (name: string) => ({ + manifestSha256: createHash("sha256") + .update(manifests[name]!) + .digest("hex"), + arches: ["wasm32"], + cacheKeys: { wasm32: "c".repeat(64) }, + dependencyClosures: { wasm32: [] }, + }); + writeFileSync( + join(registry, "program-packages.json"), + `${JSON.stringify({ + format: "kandelo-program-packages-v2", + identities: Object.fromEntries( + Object.entries(manifests).map(([name, manifest]) => [ + name, + { + manifestSha256: createHash("sha256") + .update(manifest) + .digest("hex"), + cacheKeys: { + wasm32: "c".repeat(64), + wasm64: "d".repeat(64), + }, + }, + ]), + ), + packages: { + "scalar-owner": { + ...base("scalar-owner"), + members: [{ + kind: "output", + sourceArtifact: "shared", + mirrorPath: "shared", + outputName: "shared", + forkInstrumentation: "auto", + }], + }, + shared: { + ...base("shared"), + members: [ + { + kind: "output", + sourceArtifact: "child.wasm", + mirrorPath: "shared/child.wasm", + outputName: "child", + forkInstrumentation: "auto", + }, + { + kind: "runtime-file", + sourceArtifact: "data", + mirrorPath: "shared/data", + guestPath: "/usr/share/data", + mode: 0o644, + }, + ], + }, + }, + }, null, 2)}\n`, + ); + + expect(() => + packageOutputOwners(fixtureRoot, { registryPath: "registry" }) + ).toThrow(/outputs .* conflict between/); + } finally { + rmSync(fixtureRoot, { recursive: true, force: true }); + } + }); + it("derives the exact package roots needed to bundle the browser app", () => { const audit = inspectBrowserBinaryDependencies(repoRoot); const roots = browserBinaryPackageRoots(repoRoot, { diff --git a/tests/package-system/host-package-projection-contract.test.ts b/tests/package-system/host-package-projection-contract.test.ts new file mode 100644 index 0000000000..676b9d20ec --- /dev/null +++ b/tests/package-system/host-package-projection-contract.test.ts @@ -0,0 +1,34 @@ +import { readFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { describe, expect, it } from "vitest"; + +const repoRoot = resolve(import.meta.dirname, "../.."); + +describe("standalone host package projection contract", () => { + it("checks the generated projection before copying it into the npm package", () => { + const script = readFileSync( + join(repoRoot, "scripts", "prepare-host-package.sh"), + "utf8", + ); + const check = script.indexOf("build-deps program-index-check \\"); + const copy = script.indexOf( + 'cp \\\n "$REPO_ROOT/packages/registry/program-packages.json"', + ); + + expect(check).toBeGreaterThan(-1); + expect(copy).toBeGreaterThan(check); + expect(script).toContain( + '"$REPO_ROOT/packages/registry/program-packages.json"', + ); + }); + + it("runs the checked preparation script on every npm pack", () => { + const packageJson = JSON.parse( + readFileSync(join(repoRoot, "host", "package.json"), "utf8"), + ) as { scripts?: Record }; + + expect(packageJson.scripts?.prepack).toContain( + "bash ../scripts/prepare-host-package.sh", + ); + }); +}); diff --git a/tests/package-system/installed-host-package.test.ts b/tests/package-system/installed-host-package.test.ts new file mode 100644 index 0000000000..cd31b9736e --- /dev/null +++ b/tests/package-system/installed-host-package.test.ts @@ -0,0 +1,494 @@ +import { afterAll, describe, expect, it } from "vitest"; +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + cpSync, + mkdirSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { tmpdir } from "node:os"; + +const repoRoot = resolve(import.meta.dirname, "../.."); +const fixtureRoots: string[] = []; + +afterAll(() => { + for (const root of fixtureRoots) { + rmSync(root, { recursive: true, force: true }); + } +}); + +describe("installed host package binary policy", () => { + it("binds installed scalar and multi-member bytes to packaged projection identity", () => { + execFileSync("npm", ["--prefix", "host", "run", "build"], { + cwd: repoRoot, + stdio: "pipe", + }); + + const root = mkdtempSync(join(tmpdir(), "kandelo-packed-host-")); + fixtureRoots.push(root); + const staging = join(root, "staging"); + const wasmRoot = join(staging, "wasm"); + const multiName = "packed-runtime"; + const scalarName = "packed-command-package"; + const dependencyName = "packed-runtime-dependency"; + const auxiliaryDependencyName = "packed-runtime-auxiliary"; + const scalarOutput = "packed-command.zip"; + const imageRel = `programs/wasm32/${multiName}/image.zip`; + const runtimeRel = + `programs/wasm32/${multiName}/share/runtime.dat`; + const scalarRel = `programs/wasm32/${scalarOutput}`; + const multiManifest = `kind = "program" +name = "${multiName}" +version = "1.0.0" +depends_on = ["${dependencyName}@1.0.0", "${auxiliaryDependencyName}@1.0.0"] +[[outputs]] +name = "image" +wasm = "artifacts/image.zip" +[[runtime_files]] +artifact = "share/runtime.dat" +guest_path = "/usr/share/packed-runtime/runtime.dat" +`; + const scalarManifest = `kind = "program" +name = "${scalarName}" +version = "1.0.0" +[[outputs]] +name = "packed-command" +wasm = "bin/${scalarOutput}" +`; + const dependencyManifest = `kind = "library" +name = "${dependencyName}" +version = "1.0.0" +`; + const auxiliaryDependencyManifest = `kind = "source" +name = "${auxiliaryDependencyName}" +version = "1.0.0" +`; + const dependencyIdentity = { + packageName: dependencyName, + manifestSha256: createHash("sha256") + .update(dependencyManifest) + .digest("hex"), + cacheKey: "d".repeat(64), + }; + const auxiliaryDependencyIdentity = { + packageName: auxiliaryDependencyName, + manifestSha256: createHash("sha256") + .update(auxiliaryDependencyManifest) + .digest("hex"), + cacheKey: "c".repeat(64), + }; + const multiProjection = { + manifestSha256: createHash("sha256").update(multiManifest).digest("hex"), + arches: ["wasm32"], + cacheKeys: { wasm32: "1".repeat(64) }, + dependencyClosures: { + wasm32: [dependencyIdentity, auxiliaryDependencyIdentity], + }, + members: [ + { + kind: "output", + sourceArtifact: "artifacts/image.zip", + mirrorPath: `${multiName}/image.zip`, + outputName: "image", + forkInstrumentation: "auto", + }, + { + kind: "runtime-file", + sourceArtifact: "share/runtime.dat", + mirrorPath: `${multiName}/share/runtime.dat`, + guestPath: "/usr/share/packed-runtime/runtime.dat", + mode: 0o644, + }, + ], + }; + const scalarProjection = { + manifestSha256: createHash("sha256").update(scalarManifest).digest("hex"), + arches: ["wasm32"], + cacheKeys: { wasm32: "2".repeat(64) }, + dependencyClosures: { wasm32: [] }, + members: [{ + kind: "output", + sourceArtifact: `bin/${scalarOutput}`, + mirrorPath: scalarOutput, + outputName: "packed-command", + forkInstrumentation: "auto", + }], + }; + const bundledPackages = { + [multiName]: multiProjection, + [scalarName]: scalarProjection, + }; + const bundledIdentities: Record = Object.fromEntries( + Object.entries(bundledPackages).map(([packageName, projection]) => [ + packageName, + { + manifestSha256: projection.manifestSha256, + cacheKeys: { + wasm32: projection.cacheKeys.wasm32, + wasm64: createHash("sha256") + .update(`${packageName}:wasm64`) + .digest("hex"), + }, + }, + ]), + ); + bundledIdentities[dependencyName] = { + manifestSha256: dependencyIdentity.manifestSha256, + cacheKeys: { + wasm32: dependencyIdentity.cacheKey, + wasm64: "e".repeat(64), + }, + }; + bundledIdentities[auxiliaryDependencyName] = { + manifestSha256: auxiliaryDependencyIdentity.manifestSha256, + cacheKeys: { + wasm32: auxiliaryDependencyIdentity.cacheKey, + wasm64: "b".repeat(64), + }, + }; + + mkdirSync(join(wasmRoot, dirname(imageRel)), { recursive: true }); + mkdirSync(join(wasmRoot, dirname(runtimeRel)), { recursive: true }); + mkdirSync(join(wasmRoot, dirname(scalarRel)), { recursive: true }); + cpSync(join(repoRoot, "host", "dist"), join(staging, "dist"), { + recursive: true, + }); + cpSync(join(repoRoot, "host", "package.json"), join(staging, "package.json")); + writeFileSync(join(wasmRoot, imageRel), "packed image"); + writeFileSync(join(wasmRoot, runtimeRel), "packed runtime"); + writeFileSync(join(wasmRoot, scalarRel), "packed scalar"); + writeFileSync( + join(wasmRoot, "program-packages.json"), + `${JSON.stringify({ + format: "kandelo-program-packages-v2", + identities: bundledIdentities, + packages: bundledPackages, + }, null, 2)}\n`, + ); + + const packed = JSON.parse( + execFileSync("npm", ["pack", "--ignore-scripts", "--json"], { + cwd: staging, + encoding: "utf8", + }), + ) as Array<{ filename: string }>; + const archive = join(staging, packed[0]!.filename); + const listing = execFileSync("tar", ["-tzf", archive], { + encoding: "utf8", + }); + expect(listing).toContain("package/wasm/program-packages.json"); + expect(listing).toContain(`package/wasm/${imageRel}`); + expect(listing).toContain(`package/wasm/${runtimeRel}`); + expect(listing).toContain(`package/wasm/${scalarRel}`); + + const consumer = join(root, "consumer"); + const modules = join(consumer, "node_modules"); + mkdirSync(modules, { recursive: true }); + execFileSync("tar", ["-xzf", archive], { cwd: root }); + cpSync(join(root, "package"), join(modules, "wasm-posix-host"), { + recursive: true, + }); + for (const dependency of ["fflate", "fzstd"]) { + symlinkSync( + join(repoRoot, "node_modules", dependency), + join(modules, dependency), + "dir", + ); + } + writeFileSync( + join(consumer, "package.json"), + '{"name":"isolated-host-consumer","private":true,"type":"module"}\n', + ); + const baseEnv = { ...process.env }; + delete baseEnv.WASM_POSIX_DEPS_REGISTRY; + delete baseEnv.WASM_POSIX_BINARY_RESOLVER_REPO_ROOT; + // Installed projections are verified when the package is assembled, not + // by reaching back into a source checkout at runtime. A deliberately + // unusable checker path proves installed resolution never consults it. + baseEnv.WASM_POSIX_XTASK_BIN = join(root, "must-not-run-xtask"); + + const writeRegistry = ( + name: string, + packages: Record; + cacheKeys?: Record; + }>, + contextualPackages: Record; + projection?: Record; + }> = {}, + ): string => { + const registry = join(root, name); + mkdirSync(registry, { recursive: true }); + const identities: Record = {}; + const projections: Record = {}; + for ( + const [packageName, entry] of Object.entries(contextualPackages) + ) { + identities[packageName] = { + manifestSha256: createHash("sha256") + .update(entry.manifest) + .digest("hex"), + cacheKeys: entry.cacheKeys, + }; + if (entry.projection) projections[packageName] = entry.projection; + } + for (const [packageName, entry] of Object.entries(packages)) { + const packageDir = join(registry, packageName); + mkdirSync(packageDir, { recursive: true }); + writeFileSync(join(packageDir, "package.toml"), entry.manifest); + if (entry.projection) projections[packageName] = entry.projection; + const projection = entry.projection as { + manifestSha256?: string; + cacheKeys?: Record; + } | undefined; + const manifestSha256 = createHash("sha256") + .update(entry.manifest) + .digest("hex"); + const cacheKeys = entry.cacheKeys ?? projection?.cacheKeys ?? {}; + identities[packageName] = { + manifestSha256, + cacheKeys: { + wasm32: cacheKeys.wasm32 + ?? createHash("sha256").update(`${packageName}:wasm32`).digest("hex"), + wasm64: cacheKeys.wasm64 + ?? createHash("sha256").update(`${packageName}:wasm64`).digest("hex"), + }, + }; + } + writeFileSync( + join(registry, "program-packages.json"), + `${JSON.stringify({ + format: "kandelo-program-packages-v2", + identities, + packages: projections, + }, null, 2)}\n`, + ); + return registry; + }; + const equivalentRegistry = writeRegistry("registry-equivalent", { + [multiName]: { + manifest: multiManifest, + projection: { + ...multiProjection, + // Runtime identity is a unique package-identity set. Reverse the + // deterministic generator order to prove serialization order alone + // cannot reject the same installed package generation. + dependencyClosures: { + wasm32: [ + auxiliaryDependencyIdentity, + dependencyIdentity, + ], + }, + }, + }, + [scalarName]: { + manifest: scalarManifest, + projection: scalarProjection, + }, + [dependencyName]: { + manifest: dependencyManifest, + cacheKeys: { + wasm32: dependencyIdentity.cacheKey, + wasm64: "e".repeat(64), + }, + }, + [auxiliaryDependencyName]: { + manifest: auxiliaryDependencyManifest, + cacheKeys: { + wasm32: auxiliaryDependencyIdentity.cacheKey, + wasm64: "b".repeat(64), + }, + }, + }); + const mismatchedRegistry = writeRegistry("registry-mismatched", { + [multiName]: { + manifest: multiManifest, + projection: { + ...multiProjection, + cacheKeys: { wasm32: "3".repeat(64) }, + }, + }, + [scalarName]: { + manifest: scalarManifest, + projection: { + ...scalarProjection, + cacheKeys: { wasm32: "4".repeat(64) }, + }, + }, + [dependencyName]: { + manifest: dependencyManifest, + cacheKeys: { + wasm32: dependencyIdentity.cacheKey, + wasm64: "e".repeat(64), + }, + }, + [auxiliaryDependencyName]: { + manifest: auxiliaryDependencyManifest, + cacheKeys: { + wasm32: auxiliaryDependencyIdentity.cacheKey, + wasm64: "b".repeat(64), + }, + }, + }); + const contextMismatchedRegistry = writeRegistry( + "registry-context-mismatch", + { + [dependencyName]: { + manifest: dependencyManifest.replace("1.0.0", "2.0.0"), + cacheKeys: { + wasm32: "f".repeat(64), + wasm64: "0".repeat(64), + }, + }, + }, + { + [multiName]: { + manifest: multiManifest, + cacheKeys: { + wasm32: "9".repeat(64), + wasm64: "a".repeat(64), + }, + projection: { + ...multiProjection, + cacheKeys: { wasm32: "9".repeat(64) }, + dependencyClosures: { + wasm32: [{ + packageName: dependencyName, + manifestSha256: createHash("sha256") + .update(dependencyManifest.replace("1.0.0", "2.0.0")) + .digest("hex"), + cacheKey: "f".repeat(64), + }], + }, + }, + }, + [scalarName]: { + manifest: scalarManifest, + cacheKeys: { + wasm32: scalarProjection.cacheKeys.wasm32, + wasm64: "b".repeat(64), + }, + projection: scalarProjection, + }, + }, + ); + const unrelatedName = "unrelated-runtime"; + const unrelatedManifest = `kind = "program" +name = "${unrelatedName}" +version = "1.0.0" +[[outputs]] +name = "unrelated" +wasm = "unrelated.wasm" +`; + const unrelatedRegistry = writeRegistry("registry-unrelated", { + [unrelatedName]: { + manifest: unrelatedManifest, + projection: { + manifestSha256: createHash("sha256") + .update(unrelatedManifest) + .digest("hex"), + arches: ["wasm32"], + cacheKeys: { wasm32: "5".repeat(64) }, + dependencyClosures: { wasm32: [] }, + members: [{ + kind: "output", + sourceArtifact: "unrelated.wasm", + mirrorPath: "unrelated.wasm", + outputName: "unrelated", + forkInstrumentation: "auto", + }], + }, + }, + }); + + const runResolver = ( + registry?: string, + ): Record => { + const env = { ...baseEnv }; + if (registry === undefined) { + delete env.WASM_POSIX_DEPS_REGISTRY; + } else { + env.WASM_POSIX_DEPS_REGISTRY = registry; + } + return JSON.parse( + execFileSync( + process.execPath, + [ + "--input-type=module", + "-e", + `import { resolveBinary } from "wasm-posix-host"; +const result = {}; +for (const [name, rel] of Object.entries(${JSON.stringify({ + multi: imageRel, + scalar: scalarRel, + })})) { + try { + result[name] = { path: resolveBinary(rel) }; + } catch (error) { + result[name] = { error: error instanceof Error ? error.message : String(error) }; + } +} +process.stdout.write(JSON.stringify(result));`, + ], + { + cwd: consumer, + encoding: "utf8", + env, + }, + ), + ) as Record; + }; + + for (const result of [runResolver(), runResolver(equivalentRegistry)]) { + expect(result.multi?.error).toBeUndefined(); + expect(result.scalar?.error).toBeUndefined(); + expect(readFileSync(result.multi!.path!, "utf8")).toBe("packed image"); + expect(readFileSync(result.scalar!.path!, "utf8")).toBe("packed scalar"); + } + const installedWasmRoot = realpathSync( + join(modules, "wasm-posix-host", "wasm"), + ); + const resolved = runResolver(equivalentRegistry); + expect(resolved.multi!.path!.startsWith(`${installedWasmRoot}/`)).toBe(true); + expect(resolved.scalar!.path!.startsWith(`${installedWasmRoot}/`)).toBe(true); + + for (const entry of Object.values(runResolver(mismatchedRegistry))) { + expect(entry.error).toMatch( + /installed bytes do not match the selected package projection/, + ); + } + const contextMismatch = runResolver( + `${contextMismatchedRegistry}:${equivalentRegistry}`, + ); + expect(contextMismatch.multi?.error).toMatch( + /installed bytes do not match the selected package projection/, + ); + expect(contextMismatch.scalar?.error).toBeUndefined(); + expect(readFileSync(contextMismatch.scalar!.path!, "utf8")).toBe( + "packed scalar", + ); + for (const entry of Object.values(runResolver(unrelatedRegistry))) { + expect(entry.error).toMatch( + /owned by .*but that package is not selected/, + ); + } + + rmSync( + join(modules, "wasm-posix-host", "wasm", "program-packages.json"), + ); + for (const entry of Object.values(runResolver())) { + expect(entry.error).toMatch( + /missing wasm\/program-packages\.json/, + ); + } + }, 120_000); +}); diff --git a/tests/package-system/package-source-publish-contract.test.ts b/tests/package-system/package-source-publish-contract.test.ts index 45e5739d21..237f4d4706 100644 --- a/tests/package-system/package-source-publish-contract.test.ts +++ b/tests/package-system/package-source-publish-contract.test.ts @@ -11,6 +11,24 @@ const script = readFileSync( ); describe("package-source publication contract", () => { + it("rejects a stale runtime projection in the exact publish registry order", () => { + const sync = script.indexOf('"$KANDELO_ROOT/scripts/sync-package-source.sh"'); + const registry = script.indexOf( + 'export WASM_POSIX_DEPS_REGISTRY="$PACKAGE_SOURCE_ROOT/packages:$KANDELO_ROOT/packages/registry"', + ); + const projectionCheck = script.indexOf( + "build-deps program-index-context-check", + ); + const packageLoop = script.indexOf("while IFS= read -r pkg; do"); + + expect(sync).toBeGreaterThan(-1); + expect(registry).toBeGreaterThan(-1); + expect(registry).toBeLessThan(sync); + expect(projectionCheck).toBeGreaterThan(registry); + expect(projectionCheck).toBeLessThan(sync); + expect(packageLoop).toBeGreaterThan(sync); + }); + it("materializes declared program dependencies for source builds", () => { const lines = script.split(/\r?\n/); const archiveStage = lines.findIndex((line) => line.trim() === "archive-stage \\"); diff --git a/tests/package-system/program-resolver-literals.test.ts b/tests/package-system/program-resolver-literals.test.ts new file mode 100644 index 0000000000..af66db0f03 --- /dev/null +++ b/tests/package-system/program-resolver-literals.test.ts @@ -0,0 +1,265 @@ +import { + existsSync, + readFileSync, + readdirSync, +} from "node:fs"; +import { basename, extname, join, resolve } from "node:path"; +import { describe, expect, it } from "vitest"; + +const repoRoot = resolve(import.meta.dirname, "../.."); +const projectionPath = join( + repoRoot, + "packages", + "registry", + "program-packages.json", +); +const fixtureInventoryPath = join( + repoRoot, + "tests", + "test-artifacts", + "kernel-test-programs.json", +); + +type ProgramMember = { + kind: "output" | "runtime-file"; + mirrorPath: string; +}; + +type ProgramProjection = { + arches: Array<"wasm32" | "wasm64">; + members: ProgramMember[]; +}; + +type ProgramPackageIndex = { + packages: Record; +}; + +type FixtureInventory = { + fixtures: Array<{ + source: string; + binary: string; + resolver_path: string; + }>; +}; + +type StalePathOwner = { + packageName: string; + replacement: string; +}; + +const sourceRoots = [ + ".cargo", + ".github", + "abi", + "apps", + "benchmarks", + "crates", + "docs", + "docs-site", + "examples", + "homebrew", + "host", + "images", + "packages", + "programs", + "scripts", + "sdk", + "tests", + "tools", + "web-libs", +]; + +const topLevelSources = [ + "README.md", + "TODO-kernel-refactoring.md", + "build.sh", + "flake.nix", + "package.json", + "run.sh", +]; + +const excludedDirectories = new Set([ + ".git", + "build", + "dist", + "node_modules", + "target", + "test-results", +]); + +const excludedFiles = new Set([ + "packages/registry/program-packages.json", + "scripts/resolve-binary.bundle.LICENSES.txt", + "scripts/resolve-binary.bundle.mjs", +]); + +function readJson(path: string): T { + return JSON.parse(readFileSync(path, "utf8")) as T; +} + +function directProgramSources(): string[] { + return readdirSync(join(repoRoot, "programs")) + .filter((name) => [".c", ".cpp"].includes(extname(name))) + .map((name) => `programs/${name}`) + .sort(); +} + +function legitimateFlatResolverPaths(inventory: FixtureInventory): Set { + const paths = new Set(); + const inventoriedSources = new Set( + inventory.fixtures.map((fixture) => fixture.source), + ); + + for (const fixture of inventory.fixtures) { + paths.add(fixture.binary); + paths.add(fixture.resolver_path); + } + + for (const source of directProgramSources()) { + // The inventory is authoritative for architecture-specific exceptions + // such as hello64.c. Every other direct program source is produced by + // scripts/build-programs.sh in the ordinary wasm32 fixture namespace. + if (inventoriedSources.has(source)) continue; + const name = basename(source, extname(source)); + paths.add(`programs/${name}.wasm`); + paths.add(`programs/wasm32/${name}.wasm`); + } + + return paths; +} + +function staleFlatPackagePaths( + index: ProgramPackageIndex, + legitimateFlatPaths: ReadonlySet, +): Map { + const candidates = new Map(); + + function add(path: string, owner: StalePathOwner): void { + if (legitimateFlatPaths.has(path)) return; + const owners = candidates.get(path) ?? []; + owners.push(owner); + candidates.set(path, owners); + } + + for (const [packageName, projection] of Object.entries(index.packages)) { + for (const member of projection.members) { + if (member.kind !== "output" || !member.mirrorPath.includes("/")) { + continue; + } + const outputBasename = member.mirrorPath.split("/").at(-1)!; + for (const arch of projection.arches) { + add(`programs/${arch}/${outputBasename}`, { + packageName, + replacement: `programs/${arch}/${member.mirrorPath}`, + }); + if (arch === "wasm32") { + add(`programs/${outputBasename}`, { + packageName, + replacement: `programs/${member.mirrorPath}`, + }); + } + } + } + } + + return candidates; +} + +function sourceFilesUnder(relPath: string): string[] { + const absolute = join(repoRoot, relPath); + if (!existsSync(absolute)) return []; + return readdirSync(absolute, { withFileTypes: true }).flatMap((entry) => { + if (entry.isSymbolicLink()) return []; + const child = join(relPath, entry.name); + if (entry.isDirectory()) { + if ( + excludedDirectories.has(entry.name) + || child === "docs/plans" + ) { + return []; + } + return sourceFilesUnder(child); + } + if (!entry.isFile() || excludedFiles.has(child)) return []; + return [child]; + }); +} + +function auditedSourceFiles(): string[] { + return [ + ...sourceRoots.flatMap(sourceFilesUnder), + ...topLevelSources.filter((path) => existsSync(join(repoRoot, path))), + ].sort(); +} + +function staleLiteralFailures( + candidates: ReadonlyMap, +): string[] { + const failures: string[] = []; + for (const relPath of auditedSourceFiles()) { + const bytes = readFileSync(join(repoRoot, relPath)); + if (bytes.includes(0)) continue; + const lines = bytes.toString("utf8").split("\n"); + for (const [stalePath, owners] of candidates) { + lines.forEach((line, index) => { + if (!line.includes(stalePath)) return; + const replacements = [...new Set(owners.map((owner) => owner.replacement))] + .sort() + .map((replacement) => JSON.stringify(replacement)) + .join(" or "); + const packages = [...new Set(owners.map((owner) => owner.packageName))] + .sort() + .map((packageName) => JSON.stringify(packageName)) + .join(", "); + failures.push( + `${relPath}:${index + 1}: ${JSON.stringify(stalePath)} is a stale flat ` + + `resolver path owned by package ${packages}; use ${replacements}`, + ); + }); + } + } + return failures; +} + +describe("program resolver source literals", () => { + it("keeps direct program fixtures outside package directory policy", () => { + const inventory = readJson(fixtureInventoryPath); + const legitimatePaths = legitimateFlatResolverPaths(inventory); + expect(legitimatePaths).toContain("programs/sh.wasm"); + expect(legitimatePaths).toContain("programs/wasm32/sh.wasm"); + + const syntheticIndex: ProgramPackageIndex = { + packages: { + "synthetic-shell-tools": { + arches: ["wasm32"], + members: [ + { + kind: "output", + mirrorPath: "synthetic-shell-tools/sh.wasm", + }, + ], + }, + }, + }; + const candidates = staleFlatPackagePaths(syntheticIndex, legitimatePaths); + expect(candidates.has("programs/sh.wasm")).toBe(false); + expect(candidates.has("programs/wasm32/sh.wasm")).toBe(false); + }); + + it("does not name package-directory outputs through obsolete flat paths", () => { + const index = readJson(projectionPath); + const inventory = readJson(fixtureInventoryPath); + const candidates = staleFlatPackagePaths( + index, + legitimateFlatResolverPaths(inventory), + ); + const failures = staleLiteralFailures(candidates); + + expect( + failures, + failures.length === 0 + ? undefined + : `Stale package resolver literals:\n${failures.join("\n")}`, + ).toEqual([]); + }); +}); diff --git a/tests/package-system/resolve-binary.test.ts b/tests/package-system/resolve-binary.test.ts index 81437ca29d..c19cd4392f 100644 --- a/tests/package-system/resolve-binary.test.ts +++ b/tests/package-system/resolve-binary.test.ts @@ -1,9 +1,10 @@ import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { spawnSync } from "node:child_process"; import { - copyFileSync, + chmodSync, mkdirSync, mkdtempSync, + readFileSync, realpathSync, rmSync, writeFileSync, @@ -120,9 +121,15 @@ function writeCandidate( } function resolveBinary(relPath: string) { - return spawnSync("bash", [join(fakeRepoRoot, "scripts", "resolve-binary.sh"), relPath], { + const env = { + ...process.env, + WASM_POSIX_BINARY_RESOLVER_REPO_ROOT: fakeRepoRoot, + }; + delete env.WASM_POSIX_DEPS_REGISTRY; + return spawnSync("bash", [join(repoRoot, "scripts", "resolve-binary.sh"), relPath], { cwd: fakeRepoRoot, encoding: "utf8", + env, }); } @@ -130,22 +137,16 @@ beforeAll(() => { fakeRepoRoot = realpathSync( mkdtempSync(join(tmpdir(), "kandelo-resolve-binary-")), ); - mkdirSync(join(fakeRepoRoot, "scripts"), { recursive: true }); - mkdirSync(join(fakeRepoRoot, "abi"), { recursive: true }); - mkdirSync(join(fakeRepoRoot, "crates", "shared", "src"), { recursive: true }); + mkdirSync(join(fakeRepoRoot, "packages", "registry"), { recursive: true }); writeFileSync(join(fakeRepoRoot, "Cargo.toml"), "[workspace]\nmembers = []\n"); - writeFileSync(join(fakeRepoRoot, "abi", "snapshot.json"), "{}\n"); writeFileSync( - join(fakeRepoRoot, "crates", "shared", "src", "lib.rs"), - `pub const ABI_VERSION: u32 = ${ABI_VERSION};\n`, + join(fakeRepoRoot, "package.json"), + "{\"name\":\"kandelo\",\"private\":true}\n", + ); + writeFileSync( + join(fakeRepoRoot, "packages", "registry", "program-packages.json"), + '{"format":"kandelo-program-packages-v2","identities":{},"packages":{}}\n', ); - for (const script of [ - "resolve-binary.sh", - "wasm-artifact-guards.sh", - "vfs-has-stale-abi.mjs", - ]) { - copyFileSync(join(repoRoot, "scripts", script), join(fakeRepoRoot, "scripts", script)); - } }); afterAll(() => { @@ -153,6 +154,97 @@ afterAll(() => { }); describe("shell binary resolver artifact policy", () => { + it("incrementally rebuilds an existing source checker before exporting it", () => { + const sourceRoot = mkdtempSync( + join(tmpdir(), "kandelo-resolve-binary-checker-source-"), + ); + const toolBin = join(sourceRoot, "test-tools"); + const hostTarget = "test-checker-host"; + const xtaskPath = join( + sourceRoot, + "target", + hostTarget, + "release", + "xtask", + ); + const buildRecord = join(sourceRoot, "cargo-build-record"); + mkdirSync(join(sourceRoot, "tools", "xtask"), { recursive: true }); + mkdirSync(join(sourceRoot, "scripts"), { recursive: true }); + mkdirSync(dirname(xtaskPath), { recursive: true }); + mkdirSync(toolBin, { recursive: true }); + writeFileSync(join(sourceRoot, "tools", "xtask", "Cargo.toml"), ""); + writeFileSync( + join(sourceRoot, "scripts", "dev-shell.sh"), + `#!/bin/sh +printf '%s\\n' 'dev-shell setup chatter' +exec "$@" +`, + ); + writeFileSync(xtaskPath, "#!/bin/sh\nexit 99\n"); + chmodSync(xtaskPath, 0o755); + writeFileSync( + join(toolBin, "rustc"), + `#!/bin/sh +printf 'rustc 1.0\\nhost: ${hostTarget}\\n' +`, + ); + writeFileSync( + join(toolBin, "cargo"), + `#!/bin/sh +printf '%s\\n' "$*" > "$CHECKER_BUILD_RECORD" +`, + ); + writeFileSync( + join(toolBin, "node"), + `#!/bin/sh +printf '%s\\n' "$WASM_POSIX_XTASK_BIN" +`, + ); + for (const tool of ["rustc", "cargo", "node"]) { + chmodSync(join(toolBin, tool), 0o755); + } + + try { + const result = spawnSync( + "bash", + [ + join(repoRoot, "scripts", "resolve-binary.sh"), + "programs/wasm32/checker/checker.wasm", + ], + { + cwd: sourceRoot, + encoding: "utf8", + env: { + ...process.env, + PATH: `${toolBin}:${process.env.PATH ?? ""}`, + KANDELO_DEV_SHELL_TOOL_PATH: "", + WASM_POSIX_BINARY_RESOLVER_REPO_ROOT: sourceRoot, + CHECKER_BUILD_RECORD: buildRecord, + WASM_POSIX_XTASK_BIN: "", + }, + }, + ); + + expect(result.status, result.stderr).toBe(0); + expect(result.stdout.trim()).toBe(xtaskPath); + expect(result.stderr).toContain("dev-shell setup chatter"); + expect(readFileSync(buildRecord, "utf8").trim()).toBe( + `build --release -p xtask --target ${hostTarget} --quiet`, + ); + } finally { + rmSync(sourceRoot, { recursive: true, force: true }); + } + }); + + it("ships a standalone resolver bundle generated from the shared TypeScript source", () => { + const result = spawnSync( + "bash", + [join(repoRoot, "scripts", "test-resolve-binary-bundle.sh")], + { cwd: repoRoot, encoding: "utf8" }, + ); + expect(result.status, result.stderr || result.stdout).toBe(0); + }); + it("resolves a ZIP archive without applying Wasm policy", () => { const relPath = "programs/wasm32/__resolve_binary_test__/runtime.zip"; const localPath = writeCandidate( @@ -229,7 +321,7 @@ describe("shell binary resolver artifact policy", () => { const result = resolveBinary(relPath); expect(result.status).toBe(1); - expect(result.stderr).toContain("stale or invalid artifact ignored"); + expect(result.stderr).toContain("exists but was rejected by artifact policy"); }); it("keeps an uninspectable .wasm artifact fail-closed", () => { @@ -243,7 +335,7 @@ describe("shell binary resolver artifact policy", () => { const result = resolveBinary(relPath); expect(result.status).toBe(1); - expect(result.stderr).toContain("stale or invalid artifact ignored"); + expect(result.stderr).toContain("exists but was rejected by artifact policy"); }); it("falls back from an uninspectable local .wasm to a valid fetched candidate", () => { diff --git a/tools/xtask/Cargo.toml b/tools/xtask/Cargo.toml index d20ff63c89..2ef04ee94c 100644 --- a/tools/xtask/Cargo.toml +++ b/tools/xtask/Cargo.toml @@ -58,6 +58,7 @@ xz2 = "0.1" # substring out of ` --version` output. `regex` is a default- # featured, well-cached crate; no non-default features needed. regex = "1" +rustix = { version = "1", features = ["fs"] } [dev-dependencies] # Used by source_extract + build_deps integration tests for ergonomic diff --git a/tools/xtask/src/build_deps.rs b/tools/xtask/src/build_deps.rs index 5ab41223bc..dcecc9bf78 100644 --- a/tools/xtask/src/build_deps.rs +++ b/tools/xtask/src/build_deps.rs @@ -40,7 +40,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::os::fd::AsFd; -use std::path::{Path, PathBuf}; +use std::path::{Component, Path, PathBuf}; use std::process::{Command, Stdio}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Mutex, OnceLock}; @@ -51,17 +51,25 @@ use crate::host_tool_probe::{self, ProbeFailure}; use crate::index_toml::{self, EntryStatus}; use crate::pkg_manifest::{ BinarySource, BuildToml, DepRef, DepsManifest, ForkInstrumentationPolicy, GitBuildInput, - HostTool, ManifestKind, TargetArch, remove_cache_provenance, validate_cache_provenance, - write_cache_provenance, + HostTool, ManifestKind, TargetArch, file_paths_conflict, remove_cache_provenance, + validate_cache_provenance, write_cache_provenance, }; use crate::remote_fetch; use crate::repo_root; use crate::source_extract; -/// Root directory of the per-user lib cache. Honors `XDG_CACHE_HOME`, -/// else `$HOME/.cache`. Matches the pattern other tools in the repo use. +/// Root directory of the package cache. `WASM_POSIX_BINARY_CACHE_ROOT` is the +/// explicit cross-language override shared with the TypeScript resolver. +/// Otherwise honors `XDG_CACHE_HOME`, then `$HOME/.cache`. pub fn default_cache_root() -> PathBuf { - if let Some(xdg) = std::env::var_os("XDG_CACHE_HOME") { + if let Some(explicit) = std::env::var_os("WASM_POSIX_BINARY_CACHE_ROOT") { + let explicit = PathBuf::from(explicit); + if explicit.is_absolute() { + explicit + } else { + repo_root().join(explicit) + } + } else if let Some(xdg) = std::env::var_os("XDG_CACHE_HOME") { PathBuf::from(xdg).join("kandelo") } else if let Some(home) = std::env::var_os("HOME") { PathBuf::from(home).join(".cache").join("kandelo") @@ -72,6 +80,19 @@ pub fn default_cache_root() -> PathBuf { } } +#[cfg(unix)] +fn create_private_transaction_directory(path: &Path) -> std::io::Result<()> { + use std::os::unix::fs::DirBuilderExt; + let mut builder = std::fs::DirBuilder::new(); + builder.mode(0o700); + builder.create(path) +} + +#[cfg(not(unix))] +fn create_private_transaction_directory(path: &Path) -> std::io::Result<()> { + std::fs::create_dir(path) +} + /// Registry search path. Later entries have lower priority. pub struct Registry { pub roots: Vec, @@ -85,7 +106,7 @@ impl Registry { let roots = env .split(':') .filter(|s| !s.is_empty()) - .map(|s| expand_tilde(s)) + .map(|s| resolve_registry_root(repo, s)) .collect(); return Self { roots }; } @@ -149,8 +170,30 @@ impl Registry { if !toml.is_file() { continue; } - let m = - DepsManifest::load(&toml).map_err(|e| format!("{}: {e}", toml.display()))?; + // Match `Registry::load`: build.toml owns the package revision + // used by cache-key computation, and a package.pr.toml may + // replace only binary fetch metadata. Using the base manifest + // parser here would silently project revision 1 identities + // for packages whose published revision is newer. + let m = DepsManifest::load_with_overlay(&path) + .map_err(|e| format!("{}: {e}", toml.display()))?; + let directory_name = + path.file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| { + format!( + "registry package directory is not valid UTF-8: {}", + path.display() + ) + })?; + if m.name != directory_name { + return Err(format!( + "{}: package name {:?} does not match registry directory {:?}", + toml.display(), + m.name, + directory_name + )); + } // First-root-wins, mirrors `find()`. out.entry(m.name.clone()).or_insert(m); } @@ -159,4262 +202,9049 @@ impl Registry { } } -/// Subset of [`Registry::walk_all`] containing only `kind = "program"` -/// manifests. Used by `bundle-program` and `archive-stage` to look -/// up source + license decoration for release artifacts. -pub fn programs_by_name(registry: &Registry) -> Result, String> { - Ok(registry - .walk_all()? - .into_iter() - .filter(|(_, m)| matches!(m.kind, ManifestKind::Program)) - .collect()) +const PROGRAM_PACKAGE_INDEX_FORMAT: &str = "kandelo-program-packages-v2"; +const PROGRAM_PACKAGE_CONTEXT_ARCHES: [TargetArch; 2] = [TargetArch::Wasm32, TargetArch::Wasm64]; + +/// Runtime-facing projection of the program-package contract. +/// +/// `package.toml` remains the only authored source. Rust's complete manifest +/// parser emits this deliberately small, versioned index so Node, browser +/// tooling, shell scripts, external registry roots, and the standalone host +/// package all consume exactly the same closure and artifact policy without +/// growing independent TOML parsers. +#[derive(Debug, serde::Serialize)] +#[serde(rename_all = "camelCase")] +struct ProgramPackageIndex { + format: &'static str, + identities: BTreeMap, + packages: BTreeMap, } -fn expand_tilde(s: &str) -> PathBuf { - if let Some(rest) = s.strip_prefix("~/") { - if let Some(home) = std::env::var_os("HOME") { - return PathBuf::from(home).join(rest); - } +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "camelCase")] +struct ProgramPackageIdentity { + manifest_sha256: String, + cache_keys: BTreeMap, +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "camelCase")] +struct ProgramPackageProjection { + manifest_sha256: String, + arches: Vec, + cache_keys: BTreeMap, + dependency_closures: BTreeMap>, + members: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "camelCase")] +struct ProgramDependencyIdentity { + package_name: String, + manifest_sha256: String, + cache_key: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "camelCase")] +struct ProgramPackageProjectionMember { + kind: &'static str, + source_artifact: String, + mirror_path: String, + #[serde(skip_serializing_if = "Option::is_none")] + output_name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + fork_instrumentation: Option, + #[serde(skip_serializing_if = "Option::is_none")] + guest_path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + mode: Option, +} + +fn package_manifest_sha256(manifest_path: &Path) -> Result { + let bytes = std::fs::read(manifest_path).map_err(|e| { + format!( + "read {} for package identity digest: {e}", + manifest_path.display() + ) + })?; + Ok(hex(&Sha256::digest(bytes))) +} + +fn package_context_cache_keys( + manifest: &DepsManifest, + registry: &Registry, +) -> Result, String> { + package_context_cache_keys_with_global_toolchain_inputs(manifest, registry, None) +} + +fn package_context_cache_keys_with_global_toolchain_inputs( + manifest: &DepsManifest, + registry: &Registry, + global_toolchain_inputs: Option<&[BuildInputDigest]>, +) -> Result, String> { + let mut cache_keys = BTreeMap::new(); + let mut memo = BTreeMap::new(); + for arch in PROGRAM_PACKAGE_CONTEXT_ARCHES { + let cache_key = compute_sha_with_global_toolchain_inputs( + manifest, + registry, + arch, + current_abi_version(), + &mut memo, + &mut Vec::new(), + global_toolchain_inputs, + )?; + cache_keys.insert(arch.as_str().to_string(), hex(&cache_key)); } - PathBuf::from(s) + Ok(cache_keys) } -/// Cache-key sha for a manifest. Recursively hashes transitive deps -/// so any change in the tree invalidates every downstream consumer. -/// The hash domain and inputs differ by manifest kind: -/// -/// Library / program kind (arch- and ABI-specific artifacts): -/// domain `"wasm-posix-pkg\n"`, then -/// `name`, `version`, `revision`, `target_arch`, `abi_version`, -/// `source.url`, `source.sha256`, declared build input content -/// digests, global package build/toolchain content digests, optional -/// fork-instrument tool content digests for program outputs that use -/// that post-processor, then for each dep (sorted by name): -/// `dep.name`, `dep.version`, hex(dep_sha). -/// -/// Source kind (raw upstream archive, arch- and ABI-agnostic): -/// domain `"wasm-posix-pkg-source\n"`, then -/// `name`, `version`, `revision`, `source.url`, `source.sha256`, -/// declared build input content digests, then the same per-dep -/// tail. `target_arch` and `abi_version` are intentionally omitted -/// — a source tarball does not change when the kernel ABI bumps or -/// when we cross-compile for a new arch. -/// -/// ABI-bump propagation: a kernel ABI bump shifts every library and -/// program leaf sha (because `abi_version` is in their input set), -/// and those shifts ripple up to their consumers via the per-dep -/// `hex(dep_sha)` tail. Source-kind leaf shas stay stable, but a -/// library or program that consumes a source-kind dep still -/// invalidates correctly because its own `abi_version` input changes. -/// -/// Note: the `abi_version` parameter here is the **consumer's** target -/// ABI. Archives separately advertise a `Vec` of compatible ABIs -/// via `[compatibility].abi_versions`; Task A.9 verifies the -/// consumer's value is in that set during remote-fetch. -/// -/// Cycle detection via `chain`: a manifest may not transitively -/// depend on itself. -pub fn compute_sha( +fn collect_program_dependency_identities( target: &DepsManifest, registry: &Registry, arch: TargetArch, - abi_version: u32, + identities: &mut BTreeMap, + visiting: &mut Vec, memo: &mut BTreeMap, - chain: &mut Vec, -) -> Result<[u8; 32], String> { - if chain.iter().any(|s| s == &target.name) { +) -> Result<(), String> { + if visiting.iter().any(|name| name == &target.name) { return Err(format!( - "cycle in dep graph: {} -> {}", - chain.join(" -> "), + "cycle in projected dependency graph: {} -> {}", + visiting.join(" -> "), target.name )); } - // Memo key MUST include arch + abi: a single resolve chain can - // legitimately need the same package at multiple arches (e.g. a - // wasm64 program that transitively pulls a wasm32-only sibling - // via the wasm32-fallback path) and at multiple ABIs (rare today - // but the field is part of the sha input). Without these, a - // memo'd wasm64 sha bleeds into a later wasm32 lookup, producing - // a canonical cache path with wasm32 in the dir but the wasm64 - // sha in the suffix — which then can't possibly be satisfied by - // either archive. - let memo_key = format!("{}|{}|{}", target.spec(), arch.as_str(), abi_version); - if let Some(cached) = memo.get(&memo_key) { - return Ok(*cached); - } + visiting.push(target.name.clone()); - chain.push(target.name.clone()); - - // Resolve deps first; sort by name so iteration order is stable. - let mut dep_shas: Vec<(DepRef, [u8; 32])> = Vec::with_capacity(target.depends_on.len()); - for dref in &target.depends_on { - let child = registry.load(&dref.name)?; - if child.version != dref.version { + for dependency_ref in &target.depends_on { + if visiting.iter().any(|name| name == &dependency_ref.name) { + return Err(format!( + "cycle in projected dependency graph: {} -> {}", + visiting.join(" -> "), + dependency_ref.name + )); + } + let manifest_path = registry.find(&dependency_ref.name).ok_or_else(|| { + format!( + "{} depends on {}@{}, but that package is absent from the selected registry roots", + target.spec(), + dependency_ref.name, + dependency_ref.version + ) + })?; + let dependency = registry.load(&dependency_ref.name)?; + if dependency.version != dependency_ref.version { return Err(format!( "{} depends on {}@{}, but registry has {}", target.spec(), - dref.name, - dref.version, - child.spec() + dependency_ref.name, + dependency_ref.version, + dependency.spec() )); } - let child_sha = compute_sha(&child, registry, arch, abi_version, memo, chain)?; - dep_shas.push((dref.clone(), child_sha)); + let cache_key = hex(&compute_sha( + &dependency, + registry, + arch, + current_abi_version(), + memo, + &mut Vec::new(), + )?); + let identity = ProgramDependencyIdentity { + package_name: dependency.name.clone(), + manifest_sha256: package_manifest_sha256(&manifest_path)?, + cache_key, + }; + let should_recurse = match identities.get(&dependency.name) { + Some(previous) if previous != &identity => { + return Err(format!( + "{} resolves dependency {:?} to conflicting identities while projecting {}", + target.spec(), + dependency.name, + arch.as_str() + )); + } + Some(_) => false, + None => { + identities.insert(dependency.name.clone(), identity); + true + } + }; + if should_recurse { + collect_program_dependency_identities( + &dependency, + registry, + arch, + identities, + visiting, + memo, + )?; + } } - dep_shas.sort_by(|a, b| a.0.name.cmp(&b.0.name)); - chain.pop(); + visiting.pop(); + Ok(()) +} - let build_inputs = build_input_digests(target, registry)?; - let global_toolchain_inputs = match target.kind { - ManifestKind::Library | ManifestKind::Program => global_package_toolchain_digests()?, - ManifestKind::Source => Vec::new(), - }; - let fork_instrument_tool_inputs = if package_uses_fork_instrument_tool(target) { - fork_instrument_tool_digests()? - } else { - Vec::new() - }; +fn program_dependency_closure( + target: &DepsManifest, + registry: &Registry, + arch: TargetArch, +) -> Result, String> { + let mut identities = BTreeMap::new(); + collect_program_dependency_identities( + target, + registry, + arch, + &mut identities, + &mut Vec::new(), + &mut BTreeMap::new(), + )?; + Ok(identities.into_values().collect()) +} - let mut h = Sha256::new(); - match target.kind { - ManifestKind::Source => { - h.update(b"wasm-posix-pkg-source\n"); - h.update(target.name.as_bytes()); - h.update(b"\n"); - h.update(target.version.as_bytes()); - h.update(b"\n"); - h.update(target.revision.to_le_bytes()); - h.update(b"\n"); - // No target_arch, no abi_version — sources are arch/ABI-agnostic. - h.update(target.source.url.as_bytes()); - h.update(b"\n"); - h.update(target.source.sha256.as_bytes()); - h.update(b"\n"); - } - ManifestKind::Library | ManifestKind::Program => { - h.update(b"wasm-posix-pkg\n"); - h.update(target.name.as_bytes()); - h.update(b"\n"); - h.update(target.version.as_bytes()); - h.update(b"\n"); - h.update(target.revision.to_le_bytes()); - h.update(b"\n"); - h.update(arch.as_str().as_bytes()); - h.update(b"\n"); - h.update(abi_version.to_le_bytes()); - h.update(b"\n"); - h.update(target.source.url.as_bytes()); - h.update(b"\n"); - h.update(target.source.sha256.as_bytes()); - h.update(b"\n"); - // Fold in declared outputs so changing what a build is - // expected to produce invalidates the cache. Without this, - // renaming a program's `wasm = "..."` (or any library - // libs/headers/pkgconfig/files path) leaves cache_key_sha - // unchanged — the resolver then serves a canonical - // directory that doesn't match the new declaration and - // archive-stage packs broken archives. Bug discovered in - // PR #384 (lamp.vfs → lamp.vfs.zst). - // - // Ordering: hashed in authored Vec order (no sort). That - // matches how consumers like `mirror_program_outputs` - // iterate, and re-ordering is a real semantic change - // worth invalidating on. `b"|"` separators keep - // adjacent strings unambiguous (e.g. lib `"a"` + `"bc"` ≠ - // lib `"ab"` + `"c"`). A section tag (`"libs:"`, etc.) - // before each list prevents cross-section collisions. - h.update(b"outputs.libs:\n"); - for s in &target.outputs.libs { - h.update(s.as_bytes()); - h.update(b"|"); - } - h.update(b"\n"); - h.update(b"outputs.headers:\n"); - for s in &target.outputs.headers { - h.update(s.as_bytes()); - h.update(b"|"); - } - h.update(b"\n"); - h.update(b"outputs.pkgconfig:\n"); - for s in &target.outputs.pkgconfig { - h.update(s.as_bytes()); - h.update(b"|"); - } - h.update(b"\n"); - // Preserve every existing package's cache key: the additive files - // field participates only when authored. A universally empty - // section would invalidate the entire package registry merely for - // learning a new output kind. - if !target.outputs.files.is_empty() { - h.update(b"outputs.files:v1\n"); - for s in &target.outputs.files { - h.update((s.len() as u64).to_le_bytes()); - h.update(s.as_bytes()); - } +fn program_package_index_for_root_once( + root: &Path, + registry: &Registry, +) -> Result { + let canonical_root = std::fs::canonicalize(root) + .map_err(|e| format!("resolve program registry root {}: {e}", root.display()))?; + let mut first_existing_root = None; + for candidate in ®istry.roots { + match std::fs::metadata(candidate) { + Ok(metadata) => { + first_existing_root = Some((candidate, metadata)); + break; } - h.update(b"program_outputs:\n"); - for out in &target.program_outputs { - h.update(out.name.as_bytes()); - h.update(b"|"); - h.update(out.wasm.as_bytes()); - if out.fork_instrumentation != ForkInstrumentationPolicy::Auto { - h.update(b"|fork_instrumentation="); - h.update(out.fork_instrumentation.as_str().as_bytes()); - } - h.update(b"\n"); - } - // Additive program runtime closure. Keep the section absent for - // existing manifests so learning this schema does not invalidate - // every historical package cache key. - if !target.runtime_files.is_empty() { - h.update(b"runtime_files:v1\n"); - for runtime_file in &target.runtime_files { - for field in [ - runtime_file.artifact.as_bytes(), - runtime_file.guest_path.as_bytes(), - ] { - h.update((field.len() as u64).to_le_bytes()); - h.update(field); - } - h.update(runtime_file.mode.to_le_bytes()); - } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(error) => { + return Err(format!( + "inspect configured program registry root {}: {error}", + candidate.display() + )); } } } - if !build_inputs.is_empty() { - h.update(b"build-inputs:\n"); - for input in &build_inputs { - h.update(input.label.as_bytes()); - h.update(b"\n"); - h.update(input.digest); - h.update(b"\n"); - } + let (first_existing_root, first_existing_metadata) = first_existing_root.ok_or_else(|| { + format!( + "{}: no configured program registry root exists", + root.display() + ) + })?; + if !first_existing_metadata.is_dir() { + return Err(format!( + "configured program registry root is not a directory: {}", + first_existing_root.display() + )); } - if !global_toolchain_inputs.is_empty() { - h.update(b"global-toolchain-inputs:\n"); - for input in &global_toolchain_inputs { - h.update(input.label.as_bytes()); - h.update(b"\n"); - h.update(input.digest); - h.update(b"\n"); - } + let canonical_first = std::fs::canonicalize(first_existing_root).map_err(|e| { + format!( + "resolve configured program registry root {}: {e}", + first_existing_root.display() + ) + })?; + if canonical_root != canonical_first { + return Err(format!( + "{} is not the highest-priority existing configured registry root {}; generate each index with its owning root first in the ordered registry context", + root.display(), + first_existing_root.display(), + )); } - if !fork_instrument_tool_inputs.is_empty() { - h.update(b"fork-instrument-tool-inputs:\n"); - for input in &fork_instrument_tool_inputs { - h.update(input.label.as_bytes()); - h.update(b"\n"); - h.update(input.digest); - h.update(b"\n"); + + // The first generated index in an ordered registry path is the + // authoritative view of that complete first-hit context. Include both + // identities and program projections from lower roots: a dependency-only + // override can change a lower program's cache key without changing that + // program's physical manifest or members. Lower suffix indexes remain + // self-contained fallbacks when their root becomes the first existing one. + let selected_manifests = registry.walk_all()?; + let mut identities = BTreeMap::new(); + for (selected_name, manifest) in &selected_manifests { + if &manifest.name != selected_name { + return Err(format!( + "{}: selected registry key {:?} does not match manifest package name {:?}", + root.display(), + selected_name, + manifest.name + )); + } + let manifest_path = manifest.dir.join("package.toml"); + let identity = ProgramPackageIdentity { + manifest_sha256: package_manifest_sha256(&manifest_path)?, + cache_keys: package_context_cache_keys(&manifest, registry)?, + }; + if identities.insert(manifest.name.clone(), identity).is_some() { + return Err(format!( + "{}: duplicate selected package identity {:?}", + root.display(), + manifest.name + )); } } - for (dref, dsha) in &dep_shas { - h.update(dref.name.as_bytes()); - h.update(b"@"); - h.update(dref.version.as_bytes()); - h.update(b":"); - h.update(hex(dsha).as_bytes()); - h.update(b"\n"); - } - - let out: [u8; 32] = h.finalize().into(); - memo.insert(memo_key, out); - Ok(out) -} - -#[derive(Clone, Debug)] -struct BuildInputDigest { - label: String, - digest: [u8; 32], -} - -const GLOBAL_PACKAGE_TOOLCHAIN_INPUTS: &[&str] = &[ - "flake.nix", - "flake.lock", - "rust-toolchain.toml", - "scripts/dev-shell.sh", - "scripts/build-musl.sh", - "scripts/install-overlay-headers.sh", - ".github/actions/package-archive-build", - ".github/actions/package-toolchain", - ".github/actions/fetch-submodules", - ".github/actions/download-run-artifacts", - "libc/glue", - "libc/musl-overlay", - "libc/musl", - "sdk/activate.sh", - "sdk/bin", - "sdk/config.site", - "sdk/package.json", - "sdk/package-lock.json", - "sdk/src", -]; - -const FORK_INSTRUMENT_TOOL_INPUTS: &[&str] = &[ - "Cargo.toml", - "crates/fork-instrument/Cargo.toml", - "crates/fork-instrument/src", - "scripts/build-fork-instrument-tool.sh", - "scripts/run-wasm-fork-instrument.sh", -]; - -static GLOBAL_PACKAGE_TOOLCHAIN_DIGESTS: OnceLock, String>> = - OnceLock::new(); -static FORK_INSTRUMENT_TOOL_DIGESTS: OnceLock, String>> = - OnceLock::new(); -fn global_package_toolchain_digests() -> Result, String> { - GLOBAL_PACKAGE_TOOLCHAIN_DIGESTS - .get_or_init(|| { - global_package_build_input_digests_for(&repo_root(), GLOBAL_PACKAGE_TOOLCHAIN_INPUTS) - }) - .clone() -} + let mut packages = BTreeMap::new(); + let mut resolver_paths: Vec<(String, String, String)> = Vec::new(); + for (selected_name, manifest) in &selected_manifests { + let directory_name = manifest + .dir + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(|| { + format!( + "program registry package directory is not valid UTF-8: {}", + manifest.dir.display() + ) + })?; + let manifest_path = manifest.dir.join("package.toml"); + if manifest.name != directory_name || &manifest.name != selected_name { + return Err(format!( + "{}: package name {:?} does not match registry directory {:?}", + manifest_path.display(), + manifest.name, + directory_name + )); + } + let manifest_sha256 = package_manifest_sha256(&manifest_path)?; + let identity = identities.get(&manifest.name).ok_or_else(|| { + format!( + "{}: package {:?} is not selected by the configured ordered registry roots; generate this index with its owning root first", + manifest_path.display(), + manifest.name + ) + })?; + if identity.manifest_sha256 != manifest_sha256 { + return Err(format!( + "{}: selected package {:?} does not match its authoritative first-hit identity", + manifest_path.display(), + manifest.name, + )); + } + if !matches!(manifest.kind, ManifestKind::Program) { + continue; + } + // The kernel and userspace adapter are published as root boot + // artifacts (`binaries/kernel.wasm` and `binaries/userspace.wasm`), + // not as architecture-scoped guest programs. They therefore do not + // belong in the program-mirror projection. + if manifest.uses_root_binary_mirror() { + continue; + } -fn fork_instrument_tool_digests() -> Result, String> { - FORK_INSTRUMENT_TOOL_DIGESTS - .get_or_init(|| { - let root = repo_root(); - let mut digests = - global_package_build_input_digests_for(&root, FORK_INSTRUMENT_TOOL_INPUTS)?; - digests.push(BuildInputDigest { - label: "cargo-metadata:fork-instrument-build-deps".to_string(), - digest: fork_instrument_cargo_dependency_digest(&root)?, + let mut members = Vec::new(); + let mut source_artifacts = BTreeSet::new(); + let mut mirror_paths = BTreeSet::new(); + for output in &manifest.program_outputs { + let mirror_path = portable_projection_path( + &manifest, + &manifest.output_dest_rel_for(output), + "output mirror path", + )?; + insert_projection_identity( + &manifest, + &output.wasm, + &mirror_path, + &mut source_artifacts, + &mut mirror_paths, + )?; + members.push(ProgramPackageProjectionMember { + kind: "output", + source_artifact: output.wasm.clone(), + mirror_path, + output_name: Some(output.name.clone()), + fork_instrumentation: Some(output.fork_instrumentation.as_str().to_string()), + guest_path: None, + mode: None, }); - Ok(digests) - }) - .clone() -} + } + for runtime_file in &manifest.runtime_files { + let mirror_path = portable_projection_path( + &manifest, + &manifest.runtime_file_dest_rel_for(runtime_file), + "runtime-file mirror path", + )?; + insert_projection_identity( + &manifest, + &runtime_file.artifact, + &mirror_path, + &mut source_artifacts, + &mut mirror_paths, + )?; + members.push(ProgramPackageProjectionMember { + kind: "runtime-file", + source_artifact: runtime_file.artifact.clone(), + mirror_path, + output_name: None, + fork_instrumentation: None, + guest_path: Some(runtime_file.guest_path.clone()), + mode: Some(runtime_file.mode), + }); + } + if members.is_empty() { + return Err(format!( + "{}: program package has no projected members", + manifest.spec() + )); + } + if members.len() != manifest.program_closure_member_count() { + return Err(format!( + "{}: projected member count does not match the manifest closure", + manifest.spec() + )); + } -fn package_uses_fork_instrument_tool(target: &DepsManifest) -> bool { - matches!(target.kind, ManifestKind::Program) - && target - .program_outputs - .iter() - .any(|out| out.fork_instrumentation != ForkInstrumentationPolicy::Disabled) -} + let mut cache_keys = BTreeMap::new(); + let mut dependency_closures = BTreeMap::new(); + for arch in &manifest.target_arches { + cache_keys.insert( + arch.as_str().to_string(), + identity.cache_keys[arch.as_str()].clone(), + ); + dependency_closures.insert( + arch.as_str().to_string(), + program_dependency_closure(&manifest, registry, *arch)?, + ); + } + let projection = ProgramPackageProjection { + manifest_sha256, + arches: manifest + .target_arches + .iter() + .map(|arch| arch.as_str().to_string()) + .collect(), + cache_keys, + dependency_closures, + members, + }; + for arch in &projection.arches { + for member in &projection.members { + for (previous_arch, previous_path, previous_package) in &resolver_paths { + if previous_arch == arch + && file_paths_conflict(previous_path, &member.mirror_path) + { + return Err(format!( + "{}: resolver paths programs/{}/{} and programs/{}/{} conflict between packages {:?} and {:?}", + root.display(), + previous_arch, + previous_path, + arch, + member.mirror_path, + previous_package, + manifest.name + )); + } + } + resolver_paths.push(( + arch.clone(), + member.mirror_path.clone(), + manifest.name.clone(), + )); + } + } + if packages.insert(manifest.name.clone(), projection).is_some() { + return Err(format!( + "{}: duplicate program package name {:?}", + root.display(), + manifest.name + )); + } + } -#[derive(Debug, serde::Deserialize)] -struct CargoLock { - #[serde(default)] - package: Vec, + Ok(ProgramPackageIndex { + format: PROGRAM_PACKAGE_INDEX_FORMAT, + identities, + packages, + }) } -#[derive(Debug, serde::Deserialize)] -struct CargoLockPackage { - name: String, - version: String, - source: Option, - checksum: Option, +fn program_package_index_for_root( + root: &Path, + registry: &Registry, +) -> Result { + let mut after_first = || {}; + program_package_index_for_root_with(root, registry, &mut after_first) } -fn fork_instrument_cargo_dependency_digest(root: &Path) -> Result<[u8; 32], String> { - let host_target = host_target_triple()?; - let output = Command::new("cargo") - .arg("metadata") - .arg("--format-version=1") - .arg("--locked") - .arg("--filter-platform") - .arg(&host_target) - .current_dir(root) - .output() - .map_err(|e| format!("run cargo metadata for fork-instrument cache key: {e}"))?; - if !output.status.success() { +fn program_package_index_for_root_with( + root: &Path, + registry: &Registry, + after_first: &mut F, +) -> Result +where + F: FnMut(), +{ + let first = program_package_index_for_root_once(root, registry)?; + let first_snapshot = + serde_json::to_vec(&first).map_err(|e| format!("snapshot program package index: {e}"))?; + after_first(); + let second = program_package_index_for_root_once(root, registry)?; + let second_snapshot = serde_json::to_vec(&second) + .map_err(|e| format!("resnapshot program package index: {e}"))?; + if first_snapshot != second_snapshot { return Err(format!( - "cargo metadata for fork-instrument cache key failed: {}", - String::from_utf8_lossy(&output.stderr).trim() + "{}: package registry changed while generating program-packages.json; retry from one stable registry snapshot", + root.display(), )); } + Ok(second) +} - let metadata: serde_json::Value = serde_json::from_slice(&output.stdout) - .map_err(|e| format!("parse cargo metadata for fork-instrument cache key: {e}"))?; - let lock_text = std::fs::read_to_string(root.join("Cargo.lock")) - .map_err(|e| format!("read Cargo.lock for fork-instrument cache key: {e}"))?; - let lock: CargoLock = toml::from_str(&lock_text) - .map_err(|e| format!("parse Cargo.lock for fork-instrument cache key: {e}"))?; - fork_instrument_cargo_dependency_digest_from_metadata(root, &metadata, &lock) +fn portable_projection_path( + manifest: &DepsManifest, + path: &Path, + field: &str, +) -> Result { + let mut components = Vec::new(); + for component in path.components() { + match component { + Component::Normal(value) => { + let value = value.to_str().ok_or_else(|| { + format!( + "{}: {field} is not valid UTF-8: {}", + manifest.spec(), + path.display() + ) + })?; + components.push(value); + } + _ => { + return Err(format!( + "{}: {field} must be a normalized relative path: {}", + manifest.spec(), + path.display() + )); + } + } + } + if components.is_empty() { + return Err(format!("{}: {field} may not be empty", manifest.spec())); + } + Ok(components.join("/")) } -fn host_target_triple() -> Result { - let output = Command::new("rustc") - .arg("-vV") - .output() - .map_err(|e| format!("run rustc -vV: {e}"))?; - if !output.status.success() { +fn insert_projection_identity( + manifest: &DepsManifest, + source_artifact: &str, + mirror_path: &str, + source_artifacts: &mut BTreeSet, + mirror_paths: &mut BTreeSet, +) -> Result<(), String> { + if !source_artifacts.insert(source_artifact.to_string()) { return Err(format!( - "rustc -vV failed: {}", - String::from_utf8_lossy(&output.stderr).trim() + "{}: declared source artifact {:?} appears more than once in the program closure", + manifest.spec(), + source_artifact )); } - let stdout = String::from_utf8_lossy(&output.stdout); - stdout - .lines() - .find_map(|line| line.strip_prefix("host: ").map(str::to_owned)) - .filter(|host| !host.is_empty()) - .ok_or_else(|| "rustc -vV did not report host target".to_string()) + if !mirror_paths.insert(mirror_path.to_string()) { + return Err(format!( + "{}: resolver mirror path {:?} appears more than once in the program closure", + manifest.spec(), + mirror_path + )); + } + Ok(()) } -fn fork_instrument_cargo_dependency_digest_from_metadata( +fn serialize_program_package_index(root: &Path, registry: &Registry) -> Result { + let mut json = serde_json::to_string_pretty(&program_package_index_for_root(root, registry)?) + .map_err(|e| format!("serialize program package index: {e}"))?; + json.push('\n'); + Ok(json) +} + +fn cmd_program_package_index( root: &Path, - metadata: &serde_json::Value, - lock: &CargoLock, -) -> Result<[u8; 32], String> { - let packages = metadata_array(metadata, "packages")?; - let nodes = metadata - .get("resolve") - .and_then(|resolve| resolve.get("nodes")) - .and_then(|nodes| nodes.as_array()) - .ok_or_else(|| "cargo metadata missing resolve.nodes".to_string())?; + output: &Path, + registry: &Registry, +) -> Result<(), String> { + let json = serialize_program_package_index(root, registry)?; + let mut refresh_source = + || serialize_program_package_index(root, registry).map(String::into_bytes); + let mut replace = |from: &Path, to: &Path| std::fs::rename(from, to); + write_program_package_index_atomically_with_source( + output, + json.as_bytes(), + &mut refresh_source, + &mut replace, + ) +} - let mut packages_by_id: BTreeMap = BTreeMap::new(); - let mut root_package_id: Option = None; - for package in packages { - let id = metadata_str(package, "id")?.to_string(); - let name = metadata_str(package, "name")?; - let manifest_path = metadata_str(package, "manifest_path")?; - if name == "fork-instrument" - && manifest_path.ends_with("/crates/fork-instrument/Cargo.toml") - { - root_package_id = Some(id.clone()); - } - packages_by_id.insert(id, package); - } +#[cfg(test)] +fn write_program_package_index_atomically(output: &Path, bytes: &[u8]) -> Result<(), String> { + let expected = bytes.to_vec(); + let mut refresh_source = || Ok(expected.clone()); + let mut replace = |from: &Path, to: &Path| std::fs::rename(from, to); + write_program_package_index_atomically_with_source( + output, + bytes, + &mut refresh_source, + &mut replace, + ) +} - let root_package_id = root_package_id - .ok_or_else(|| "cargo metadata missing fork-instrument package".to_string())?; - let mut nodes_by_id: BTreeMap = BTreeMap::new(); - for node in nodes { - nodes_by_id.insert(metadata_str(node, "id")?.to_string(), node); - } +#[cfg(test)] +fn write_program_package_index_atomically_with( + output: &Path, + bytes: &[u8], + replace: &mut F, +) -> Result<(), String> +where + F: FnMut(&Path, &Path) -> std::io::Result<()>, +{ + let expected = bytes.to_vec(); + let mut refresh_source = || Ok(expected.clone()); + write_program_package_index_atomically_with_source(output, bytes, &mut refresh_source, replace) +} - let mut closure = BTreeSet::new(); - let mut stack = vec![root_package_id.clone()]; - while let Some(package_id) = stack.pop() { - if !closure.insert(package_id.clone()) { - continue; - } - let node = nodes_by_id - .get(&package_id) - .ok_or_else(|| format!("cargo metadata missing resolve node for {package_id}"))?; - for dep in selected_cargo_metadata_deps(node)? { - stack.push(dep); +fn write_program_package_index_atomically_with_source( + output: &Path, + bytes: &[u8], + refresh_source: &mut R, + replace: &mut F, +) -> Result<(), String> +where + F: FnMut(&Path, &Path) -> std::io::Result<()>, + R: FnMut() -> Result, String>, +{ + let parent = output + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + let parent = canonical_real_directory(parent, "program package index parent")?; + let file_name = output.file_name().ok_or_else(|| { + format!( + "program package index path has no file name: {}", + output.display() + ) + })?; + let output = parent.join(file_name); + let target_snapshot = inspect_program_package_index_target(&output)?; + let existing_permissions = target_snapshot.permissions.clone(); + let (transaction_root, stage, mut stage_file, stage_identity) = + reserve_program_package_index_transaction(&parent, file_name)?; + + let publish = (|| { + std::io::Write::write_all(&mut stage_file, bytes).map_err(|e| { + format!( + "write staged program package index {}: {e}", + stage.display() + ) + })?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let permissions = existing_permissions + .clone() + .unwrap_or_else(|| std::fs::Permissions::from_mode(0o644)); + std::fs::set_permissions(&stage, permissions).map_err(|e| { + format!( + "set staged program package index permissions {}: {e}", + stage.display() + ) + })?; } - } - - let lock_checksums = cargo_lock_checksums(lock); - let mut entries = Vec::with_capacity(closure.len()); - for package_id in closure { - let package = packages_by_id - .get(&package_id) - .ok_or_else(|| format!("cargo metadata missing package for {package_id}"))?; - let node = nodes_by_id - .get(&package_id) - .ok_or_else(|| format!("cargo metadata missing resolve node for {package_id}"))?; - let stable_id = stable_cargo_package_id(root, package)?; - let features = sorted_string_array(node, "features")?; - let deps = selected_cargo_metadata_deps(node)? - .into_iter() - .map(|dep_id| { - let dep_package = packages_by_id - .get(&dep_id) - .ok_or_else(|| format!("cargo metadata missing package for {dep_id}"))?; - stable_cargo_package_id(root, dep_package) - }) - .collect::, _>>()?; - let lock_key = cargo_lock_key(package)?; - let checksum = lock_checksums.get(&lock_key).cloned().unwrap_or_default(); - entries.push((stable_id, features, deps, checksum)); - } - entries.sort_by(|a, b| a.0.cmp(&b.0)); - - let mut h = Sha256::new(); - h.update(b"fork-instrument-cargo-build-deps-v1\n"); - for (stable_id, features, deps, checksum) in entries { - h.update(b"package\0"); - h.update(stable_id.as_bytes()); - h.update(b"\0checksum\0"); - h.update(checksum.as_bytes()); - h.update(b"\0features\0"); - for feature in features { - h.update(feature.as_bytes()); - h.update(b"\0"); + #[cfg(not(unix))] + if let Some(permissions) = existing_permissions.clone() { + std::fs::set_permissions(&stage, permissions).map_err(|e| { + format!( + "set staged program package index permissions {}: {e}", + stage.display() + ) + })?; } - h.update(b"deps\0"); - for dep in deps { - h.update(dep.as_bytes()); - h.update(b"\0"); + stage_file + .sync_all() + .map_err(|e| format!("sync staged program package index {}: {e}", stage.display()))?; + drop(stage_file); + + // The target snapshot check and overwriting rename are not a compare- + // and-swap by themselves: another generator could replace the target + // after validation and then be overwritten by this writer. All xtask + // index publishers coordinate through one durable lock inode. Keep the + // lock through source refresh, target validation, replacement, and the + // parent-directory sync so an older cooperating writer can never land + // after a newer one in that gap. + let _publication_lock = lock_program_package_index_publication(&parent, file_name)?; + + // Recompute the complete registry projection at the publication + // boundary. A writer that staged an older registry snapshot must not + // overwrite an index generated after the recipe graph changed. + let refreshed = refresh_source() + .map_err(|e| format!("refresh program package index before publication: {e}"))?; + if refreshed != bytes { + return Err( + "package registry changed after the program package index was staged; retry" + .to_string(), + ); } - h.update(b"\n"); - } - Ok(h.finalize().into()) -} -fn selected_cargo_metadata_deps(node: &serde_json::Value) -> Result, String> { - let deps = match node.get("deps").and_then(|deps| deps.as_array()) { - Some(deps) => deps, - None => return Ok(Vec::new()), - }; - let mut out = Vec::new(); - for dep in deps { - if !cargo_metadata_dep_is_build_input(dep)? { - continue; - } - out.push(metadata_str(dep, "pkg")?.to_string()); + // Refuse when another writer changed the old target after our initial + // snapshot. This is a cooperative compare-and-swap boundary: writers + // over unchanged source stage byte-identical content, while stale + // writers fail either this check or the source refresh above. + validate_program_package_index_target_snapshot(&output, &target_snapshot)?; + replace(&stage, &output).map_err(|e| { + format!( + "atomically publish program package index {} -> {}: {e}", + stage.display(), + output.display() + ) + })?; + + #[cfg(unix)] + std::fs::File::open(&parent) + .and_then(|directory| directory.sync_all()) + .map_err(|e| { + format!( + "sync program package index parent {}: {e}", + parent.display() + ) + })?; + std::fs::remove_dir(&transaction_root).map_err(|e| { + format!( + "remove empty program package index transaction {}: {e}", + transaction_root.display() + ) + }) + })(); + + if let Err(error) = publish { + let cleanup = + cleanup_program_package_index_transaction(&transaction_root, &stage, &stage_identity); + return match cleanup { + Ok(()) => Err(error), + Err(cleanup_error) => Err(format!( + "{error}; additionally failed to clean private index transaction: {cleanup_error}" + )), + }; } - out.sort(); - out.dedup(); - Ok(out) + Ok(()) } -fn cargo_metadata_dep_is_build_input(dep: &serde_json::Value) -> Result { - let dep_kinds = dep - .get("dep_kinds") - .and_then(|dep_kinds| dep_kinds.as_array()) - .ok_or_else(|| "cargo metadata dependency missing dep_kinds".to_string())?; - Ok(dep_kinds.iter().any(|kind| { - kind.get("kind") - .and_then(|kind| kind.as_str()) - .map(|kind| kind == "build") - .unwrap_or(true) - })) +fn program_package_index_lock_path(parent: &Path, file_name: &std::ffi::OsStr) -> PathBuf { + let mut lock_name = std::ffi::OsString::from("."); + lock_name.push(file_name); + lock_name.push(".kandelo-index.lock"); + parent.join(lock_name) } -fn stable_cargo_package_id(root: &Path, package: &serde_json::Value) -> Result { - let name = metadata_str(package, "name")?; - let version = metadata_str(package, "version")?; - let source = package.get("source").and_then(|source| source.as_str()); - if let Some(source) = source { - return Ok(format!("{source}#{name}@{version}")); +fn lock_program_package_index_publication( + parent: &Path, + file_name: &std::ffi::OsStr, +) -> Result { + let lock_path = program_package_index_lock_path(parent, file_name); + let mut options = std::fs::OpenOptions::new(); + options.read(true).write(true).create(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); } + let lock = options.open(&lock_path).map_err(|e| { + format!( + "open program package index publication lock {}: {e}", + lock_path.display() + ) + })?; + lock.lock().map_err(|e| { + format!( + "lock program package index publication {}: {e}", + lock_path.display() + ) + })?; - let manifest_path = PathBuf::from(metadata_str(package, "manifest_path")?); - let rel_manifest = manifest_path.strip_prefix(root).unwrap_or(&manifest_path); - Ok(format!( - "path:{}#{name}@{version}", - rel_manifest.to_string_lossy() - )) + let opened_metadata = lock.metadata().map_err(|e| { + format!( + "inspect opened program package index publication lock {}: {e}", + lock_path.display() + ) + })?; + let path_metadata = std::fs::symlink_metadata(&lock_path).map_err(|e| { + format!( + "inspect program package index publication lock path {}: {e}", + lock_path.display() + ) + })?; + if !opened_metadata.is_file() + || opened_metadata.file_type().is_symlink() + || opened_metadata.len() != 0 + || !path_metadata.is_file() + || path_metadata.file_type().is_symlink() + || path_metadata.len() != 0 + || package_mirror_identity(&opened_metadata)? != package_mirror_identity(&path_metadata)? + { + let _ = lock.unlock(); + return Err(format!( + "program package index publication lock must remain one empty regular non-symlink file: {}", + lock_path.display() + )); + } + Ok(lock) } -fn cargo_lock_key(package: &serde_json::Value) -> Result<(String, String, String), String> { - Ok(( - metadata_str(package, "name")?.to_string(), - metadata_str(package, "version")?.to_string(), - package - .get("source") - .and_then(|source| source.as_str()) - .unwrap_or("") - .to_string(), - )) +struct ProgramPackageIndexTargetSnapshot { + entry: Option, + permissions: Option, } -fn cargo_lock_checksums(lock: &CargoLock) -> BTreeMap<(String, String, String), String> { - lock.package - .iter() - .filter_map(|package| { - package.checksum.as_ref().map(|checksum| { - ( - ( - package.name.clone(), - package.version.clone(), - package.source.clone().unwrap_or_default(), - ), - checksum.clone(), - ) +fn inspect_program_package_index_target( + output: &Path, +) -> Result { + match std::fs::symlink_metadata(output) { + Ok(metadata) if metadata.file_type().is_symlink() || !metadata.is_file() => Err(format!( + "refusing to replace non-regular program package index: {}", + output.display() + )), + Ok(metadata) => { + let identity = package_mirror_identity(&metadata)?; + let entry = inspect_local_mirror_entry(output)?; + if entry.identity != identity + || !matches!(&entry.kind, LocalMirrorEntryKind::Regular { .. }) + { + return Err(format!( + "program package index changed while it was inspected: {}", + output.display() + )); + } + Ok(ProgramPackageIndexTargetSnapshot { + entry: Some(entry), + permissions: Some(metadata.permissions()), }) - }) - .collect() + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + Ok(ProgramPackageIndexTargetSnapshot { + entry: None, + permissions: None, + }) + } + Err(e) => Err(format!( + "inspect program package index {}: {e}", + output.display() + )), + } } -fn metadata_array<'a>( - value: &'a serde_json::Value, - field: &str, -) -> Result<&'a Vec, String> { - value - .get(field) - .and_then(|value| value.as_array()) - .ok_or_else(|| format!("cargo metadata missing {field} array")) +fn validate_program_package_index_target_snapshot( + output: &Path, + expected: &ProgramPackageIndexTargetSnapshot, +) -> Result<(), String> { + match &expected.entry { + Some(entry) => validate_local_mirror_entry(output, entry) + .map_err(|e| format!("program package index target changed before publication: {e}")), + None if path_entry_exists(output)? => Err(format!( + "program package index target appeared before publication: {}", + output.display() + )), + None => Ok(()), + } } -fn metadata_str<'a>(value: &'a serde_json::Value, field: &str) -> Result<&'a str, String> { - value - .get(field) - .and_then(|value| value.as_str()) - .ok_or_else(|| format!("cargo metadata missing {field} string")) +fn reserve_program_package_index_transaction( + parent: &Path, + file_name: &std::ffi::OsStr, +) -> Result<(PathBuf, PathBuf, std::fs::File, PackageMirrorIdentity), String> { + for _ in 0..1024 { + let sequence = MIRROR_TRANSACTION_COUNTER.fetch_add(1, Ordering::Relaxed); + let transaction_root = parent.join(format!( + ".{}.index-transaction-{}-{sequence}", + file_name.to_string_lossy(), + std::process::id() + )); + match create_private_transaction_directory(&transaction_root) { + Ok(()) => { + let stage = transaction_root.join("index"); + match std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&stage) + { + Ok(file) => { + let metadata = file.metadata().map_err(|e| { + format!( + "inspect staged program package index {}: {e}", + stage.display() + ) + })?; + let identity = package_mirror_identity(&metadata)?; + return Ok((transaction_root, stage, file, identity)); + } + Err(e) => { + let _ = std::fs::remove_dir(&transaction_root); + return Err(format!( + "create staged program package index {}: {e}", + stage.display() + )); + } + } + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(e) => { + return Err(format!( + "reserve program package index transaction {}: {e}", + transaction_root.display() + )); + } + } + } + Err(format!( + "could not allocate a unique program package index transaction below {}", + parent.display() + )) } -fn sorted_string_array(value: &serde_json::Value, field: &str) -> Result, String> { - let mut out = value - .get(field) - .and_then(|value| value.as_array()) - .ok_or_else(|| format!("cargo metadata missing {field} array"))? - .iter() - .map(|value| { - value - .as_str() - .map(str::to_string) - .ok_or_else(|| format!("cargo metadata {field} array contains a non-string")) - }) - .collect::, _>>()?; - out.sort(); - Ok(out) -} - -fn global_package_build_input_digests_for( - root: &Path, - inputs: &[&str], -) -> Result, String> { - let mut out = Vec::with_capacity(inputs.len()); - for input in inputs { - let path = root.join(input); - if !path.exists() { +fn cleanup_program_package_index_transaction( + transaction_root: &Path, + stage: &Path, + expected_identity: &PackageMirrorIdentity, +) -> Result<(), String> { + match std::fs::symlink_metadata(stage) { + Ok(metadata) + if metadata.is_file() + && !metadata.file_type().is_symlink() + && &package_mirror_identity(&metadata)? == expected_identity => + { + std::fs::remove_file(stage).map_err(|e| { + format!( + "remove staged program package index {}: {e}", + stage.display() + ) + })?; + } + Ok(_) => { return Err(format!( - "global package build input {:?} not found at {}", - input, - path.display() + "refusing to remove changed staged program package index {}", + stage.display() )); } - out.push(BuildInputDigest { - label: (*input).to_string(), - digest: hash_global_package_build_input(root, input, &path)?, - }); - } - Ok(out) -} - -fn hash_global_package_build_input( - root: &Path, - input: &str, - path: &Path, -) -> Result<[u8; 32], String> { - if input == "libc/musl" { - if let Some(digest) = hash_gitlink_input(root, input)? { - return Ok(digest); + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => { + return Err(format!( + "inspect staged program package index {}: {e}", + stage.display() + )); } } - hash_build_input(path) -} - -fn hash_gitlink_input(root: &Path, input: &str) -> Result, String> { - let output = match Command::new("git") - .arg("-C") - .arg(root) - .arg("ls-tree") - .arg("HEAD") - .arg("--") - .arg(input) - .output() - { - Ok(output) => output, - Err(_) => return Ok(None), - }; - if !output.status.success() { - return Ok(None); + match std::fs::remove_dir(transaction_root) { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(format!( + "remove program package index transaction {}: {e}", + transaction_root.display() + )), } - let stdout = String::from_utf8_lossy(&output.stdout); - let Some(line) = stdout.lines().next() else { - return Ok(None); - }; - let Some(rest) = line.strip_prefix("160000 commit ") else { - return Ok(None); - }; - let Some((object_id, _path)) = rest.split_once('\t') else { - return Err(format!("unexpected gitlink entry for {input:?}: {line:?}")); - }; - - let mut h = Sha256::new(); - h.update(b"gitlink\0"); - h.update(input.as_bytes()); - h.update(b"\0"); - h.update(object_id.as_bytes()); - h.update(b"\0"); - Ok(Some(h.finalize().into())) } -fn build_input_digests( - target: &DepsManifest, +fn cmd_check_program_package_index( + root: &Path, + index: &Path, registry: &Registry, -) -> Result, String> { - if !target.dir.join("build.toml").exists() { - return Ok(Vec::new()); - } - let build = BuildToml::load(&target.dir)?; - let mut out = Vec::with_capacity(build.inputs.len() + build.git_inputs.len()); - for input in &build.inputs { - let path = resolve_build_input_path(target, registry, input)?; - out.push(BuildInputDigest { - label: input.clone(), - digest: hash_build_input(&path)?, - }); - } - // External Git inputs are content-addressed before any network access. - // Preserve authored order and length-prefix every field so distinct - // tuples cannot collide through concatenation. Adding this section is - // intentionally additive: packages without git_inputs retain their - // existing cache keys. - for (index, input) in build.git_inputs.iter().enumerate() { - let mut h = Sha256::new(); - h.update(b"wasm-posix-build-git-input-v1\0"); - for field in [ - input.name.as_bytes(), - input.repository.as_bytes(), - input.commit.as_bytes(), - ] { - h.update((field.len() as u64).to_le_bytes()); - h.update(field); - } - out.push(BuildInputDigest { - label: format!("git-input:{index}:{}", input.name), - digest: h.finalize().into(), - }); +) -> Result<(), String> { + let expected = serialize_program_package_index(root, registry)?; + let actual = std::fs::read_to_string(index) + .map_err(|e| format!("read program package index {}: {e}", index.display()))?; + if actual != expected { + return Err(format!( + "{} is stale; regenerate it with `cargo run -p xtask -- build-deps program-index {} {}`", + index.display(), + root.display(), + index.display() + )); } - Ok(out) + Ok(()) } -fn resolve_build_input_path( - target: &DepsManifest, +/// Validate every program-package projection in the same ordered suffix +/// context that owns it. +/// +/// A source resolver needs a complete projection for each existing configured +/// registry root: when a higher-priority external root disappears, the next +/// root becomes authoritative. The broader `build-deps check` command retains +/// its historical behavior of validating indexes that are present, while the +/// runtime boundary uses `require_every_index = true` so an absent projection +/// cannot silently disable exact source-freshness validation. +fn check_program_package_indexes_in_context( registry: &Registry, - input: &str, -) -> Result { - let mut candidates = Vec::new(); - candidates.push(repo_root().join(input)); - candidates.extend(registry.roots.iter().map(|root| root.join(input))); - candidates.push(target.dir.join(input)); + require_every_index: bool, +) -> Result<(), String> { + for (root_index, root) in registry.roots.iter().enumerate() { + let root_metadata = match std::fs::metadata(root) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(error) => { + return Err(format!( + "inspect configured package registry root {}: {error}", + root.display() + )); + } + }; + if !root_metadata.is_dir() { + return Err(format!( + "configured package registry root is not a directory: {}", + root.display() + )); + } - for candidate in &candidates { - if candidate.exists() { - return Ok(candidate.clone()); + let index = root.join("program-packages.json"); + if !index.is_file() { + if require_every_index { + return Err(format!( + "configured package registry root {} is missing {}; generate it in its ordered registry context before resolving source packages", + root.display(), + index.display() + )); + } + continue; } - } - let tried = candidates - .iter() - .map(|p| p.display().to_string()) - .collect::>() - .join(", "); - Err(format!( - "{} build input {:?} not found (tried: {})", - target.spec(), - input, - tried - )) + // Each physical root owns an index for the ordered registry context + // beginning at that root. Higher-priority roots may add identities for + // the complete combined context, but must not make a lower root's + // committed suffix-context index appear stale. + let suffix_registry = Registry { + roots: registry.roots[root_index..].to_vec(), + }; + cmd_check_program_package_index(root, &index, &suffix_registry)?; + } + Ok(()) } -fn hash_build_input(path: &Path) -> Result<[u8; 32], String> { - let mut h = Sha256::new(); - hash_build_input_entry(&mut h, path, path)?; - Ok(h.finalize().into()) +/// Subset of [`Registry::walk_all`] containing only `kind = "program"` +/// manifests. Used by `bundle-program` and `archive-stage` to look +/// up source + license decoration for release artifacts. +pub fn programs_by_name(registry: &Registry) -> Result, String> { + Ok(registry + .walk_all()? + .into_iter() + .filter(|(_, m)| matches!(m.kind, ManifestKind::Program)) + .collect()) } -fn hash_build_input_entry(h: &mut Sha256, root: &Path, path: &Path) -> Result<(), String> { - let meta = - std::fs::symlink_metadata(path).map_err(|e| format!("stat {}: {e}", path.display()))?; - let rel = path.strip_prefix(root).unwrap_or(path); - let rel = rel.to_string_lossy(); - - if meta.file_type().is_symlink() { - let target = - std::fs::read_link(path).map_err(|e| format!("readlink {}: {e}", path.display()))?; - h.update(b"symlink\0"); - h.update(rel.as_bytes()); - h.update(b"\0"); - h.update(target.to_string_lossy().as_bytes()); - h.update(b"\0"); - return Ok(()); - } - - if meta.is_file() { - let bytes = std::fs::read(path).map_err(|e| format!("read {}: {e}", path.display()))?; - h.update(b"file\0"); - h.update(rel.as_bytes()); - h.update(b"\0"); - h.update((bytes.len() as u64).to_le_bytes()); - h.update(b"\0"); - h.update(bytes); - h.update(b"\0"); - return Ok(()); - } - - if meta.is_dir() { - h.update(b"dir\0"); - h.update(rel.as_bytes()); - h.update(b"\0"); - let mut entries = std::fs::read_dir(path) - .map_err(|e| format!("read_dir {}: {e}", path.display()))? - .collect::, _>>() - .map_err(|e| format!("read_dir {}: {e}", path.display()))?; - entries.sort_by_key(|entry| entry.path()); - for entry in entries { - hash_build_input_entry(h, root, &entry.path())?; +fn expand_tilde(s: &str) -> PathBuf { + if let Some(rest) = s.strip_prefix("~/") { + if let Some(home) = std::env::var_os("HOME") { + return PathBuf::from(home).join(rest); } - return Ok(()); } + PathBuf::from(s) +} - Err(format!( - "build input {} is not a file, directory, or symlink", - path.display() - )) +fn resolve_registry_root(repo: &Path, value: &str) -> PathBuf { + let expanded = expand_tilde(value); + if expanded.is_absolute() { + expanded + } else { + repo.join(expanded) + } } -/// Canonical cache directory for a resolved manifest. +/// Cache-key sha for a manifest. Recursively hashes transitive deps +/// so any change in the tree invalidates every downstream consumer. +/// The hash domain and inputs differ by manifest kind: /// -/// Layout: -/// libs/programs: `/libs/--rev--/` -/// sources: `/sources/--rev-/` +/// Library / program kind (arch- and ABI-specific artifacts): +/// domain `"wasm-posix-pkg\n"`, then +/// `name`, `version`, `revision`, `target_arch`, `abi_version`, +/// `source.url`, `source.sha256`, declared build input content +/// digests, global package build/toolchain content digests, optional +/// fork-instrument tool content digests for program outputs that use +/// that post-processor, then for each dep (sorted by name): +/// `dep.name`, `dep.version`, hex(dep_sha). /// -/// The directory suffix is the complete 64-character cache-key SHA-256. Archive -/// filenames may use an eight-character transport label, but canonical local -/// cache identity must not collapse distinct keys that share that prefix. Cache -/// entries created by older resolvers with a short suffix become unused and are -/// rebuilt under the full-key path; they are not migrated or trusted in place. +/// Source kind (raw upstream archive, arch- and ABI-agnostic): +/// domain `"wasm-posix-pkg-source\n"`, then +/// `name`, `version`, `revision`, `source.url`, `source.sha256`, +/// declared build input content digests, then the same per-dep +/// tail. `target_arch` and `abi_version` are intentionally omitted +/// — a source tarball does not change when the kernel ABI bumps or +/// when we cross-compile for a new arch. /// -/// For libs and programs, `arch` is part of the path so a single user -/// can host wasm32 and wasm64 builds of the same artifact side-by-side. -/// The cache-key sha already incorporates `arch` as of Task A.5, so the -/// full cache identity disambiguates — but a visible arch segment makes the -/// cache layout self-explanatory at a glance. +/// ABI-bump propagation: a kernel ABI bump shifts every library and +/// program leaf sha (because `abi_version` is in their input set), +/// and those shifts ripple up to their consumers via the per-dep +/// `hex(dep_sha)` tail. Source-kind leaf shas stay stable, but a +/// library or program that consumes a source-kind dep still +/// invalidates correctly because its own `abi_version` input changes. /// -/// For source-kind manifests, the layout omits the arch segment per -/// design decision 6: source artifacts are arch-agnostic, so a single -/// cache entry serves both wasm32 and wasm64 consumers. -pub fn canonical_path( - cache_root: &Path, - m: &DepsManifest, - arch: TargetArch, - sha: &[u8; 32], -) -> PathBuf { - let kind_subdir = match m.kind { - ManifestKind::Library => "libs", - ManifestKind::Program => "programs", - ManifestKind::Source => "sources", - }; - let basename = match m.kind { - ManifestKind::Source => format!("{}-{}-rev{}-{}", m.name, m.version, m.revision, hex(sha)), - ManifestKind::Library | ManifestKind::Program => format!( - "{}-{}-rev{}-{}-{}", - m.name, - m.version, - m.revision, - arch.as_str(), - hex(sha) - ), - }; - cache_root.join(kind_subdir).join(basename) -} - -use crate::util::hex; - -// --------------------------------------------------------------------- -// Build + cache-install -// --------------------------------------------------------------------- - -/// Options controlling where the resolver reads from and writes to. -/// Kept as a struct so tests can pass tempdirs without reaching into -/// `$HOME` / `$XDG_CACHE_HOME`. -pub struct ResolveOpts<'a> { - pub cache_root: &'a Path, - /// Optional `local-libs/` directory. When a `/build/` - /// subdirectory exists under this root, it wins over the cache - /// and the build script is not run. - pub local_libs: Option<&'a Path>, - /// Manifest names that must be source-built unconditionally, even - /// on a cache hit and even when a `[binary]` archive_url would - /// otherwise satisfy the request. Used by the manual `force-rebuild` - /// workflow to refresh archives whose cache key is suspected stale. - /// `None` means "no force rebuild" (the default for every consumer - /// other than the manual workflow). `local_libs` still wins over - /// force_source_build (a hand-patched override is always honored). - /// A force rebuild assumes no concurrent resolver invocation for - /// the same package -- see `build_into_cache`'s atomic-install comment. - pub force_source_build: Option<&'a BTreeSet>, - /// Refuse any source build or source fetch fallback. Used by CI - /// binary-materialization gates, where package bytes must come from - /// staging overlays, the durable index, or an existing valid cache entry. - pub fetch_only: bool, - /// Repo root used to resolve `[build].script_path` (which is - /// repo-relative as of Phase A-bis Task 2). `None` means "use - /// `crate::repo_root()`", which is the production default. - /// Tests use this to point the resolver at a tempdir. - pub repo_root: Option<&'a Path>, - /// When `Some`, the resolver places `binaries/programs//...` - /// symlinks for every program manifest in the dep graph (target + - /// transitive program deps). Required so a consumer's build - /// script can find sibling-package binaries via `tryResolveBinary` - /// after a `xtask build-deps resolve ` invocation. `None` - /// disables symlink placement (test fixtures, library-only - /// resolves, etc.). - pub binaries_dir: Option<&'a Path>, -} - -/// Resolve a library to a concrete on-disk path with the artifacts -/// declared in its `package.toml`. Ensures dependencies are resolved -/// first (depth-first), then runs the build script if neither a -/// `local-libs/` override nor a cache hit is available. +/// Note: the `abi_version` parameter here is the **consumer's** target +/// ABI. Archives separately advertise a `Vec` of compatible ABIs +/// via `[compatibility].abi_versions`; Task A.9 verifies the +/// consumer's value is in that set during remote-fetch. /// -/// Returns the path the consumer should point `CPPFLAGS=-I

/include -/// LDFLAGS=-L

/lib` at. -pub fn ensure_built( +/// Cycle detection via `chain`: a manifest may not transitively +/// depend on itself. +pub fn compute_sha( target: &DepsManifest, registry: &Registry, arch: TargetArch, abi_version: u32, - opts: &ResolveOpts<'_>, -) -> Result { - let mut memo: BTreeMap = BTreeMap::new(); - let mut building: Vec = Vec::new(); - let (path, _transitive) = ensure_built_inner( + memo: &mut BTreeMap, + chain: &mut Vec, +) -> Result<[u8; 32], String> { + compute_sha_with_global_toolchain_inputs( target, registry, arch, abi_version, - opts, - &mut memo, - &mut building, - )?; - Ok(path) + memo, + chain, + None, + ) } -/// One direct dependency's resolved cache path plus its manifest kind. -/// -/// Carried alongside `dep_dirs` so the build-script env-var emission -/// can switch the suffix per design 12: library/program deps export -/// under `WASM_POSIX_DEP__DIR` (a built-artifact root), source -/// deps under `WASM_POSIX_DEP__SRC_DIR` (an unbuilt source tree). -struct DirectDep { - path: PathBuf, - kind: ManifestKind, -} +fn compute_sha_with_global_toolchain_inputs( + target: &DepsManifest, + registry: &Registry, + arch: TargetArch, + abi_version: u32, + memo: &mut BTreeMap, + chain: &mut Vec, + global_toolchain_inputs_override: Option<&[BuildInputDigest]>, +) -> Result<[u8; 32], String> { + if chain.iter().any(|s| s == &target.name) { + return Err(format!( + "cycle in dep graph: {} -> {}", + chain.join(" -> "), + target.name + )); + } + // Memo key MUST include arch + abi: a single resolve chain can + // legitimately need the same package at multiple arches (e.g. a + // wasm64 program that transitively pulls a wasm32-only sibling + // via the wasm32-fallback path) and at multiple ABIs (rare today + // but the field is part of the sha input). Without these, a + // memo'd wasm64 sha bleeds into a later wasm32 lookup, producing + // a canonical cache path with wasm32 in the dir but the wasm64 + // sha in the suffix — which then can't possibly be satisfied by + // either archive. + let memo_key = format!("{}|{}|{}", target.spec(), arch.as_str(), abi_version); + if let Some(cached) = memo.get(&memo_key) { + return Ok(*cached); + } -/// Render a multi-tool probe-failure message for `ensure_built_inner`. -/// -/// Aggregates every `ProbeFailure` for `target` into one `Err(String)` -/// payload so a user fixes their toolchain in a single round-trip -/// rather than `cargo run`-ing once per missing tool. For each failure -/// we look up the matching `[[host_tools]]` declaration and append the -/// platform-keyed install hint chosen by `cfg!(target_os)`. If the -/// declaration ships hints but none for the current OS, we list which -/// platforms ARE covered so the user knows whether to translate one -/// or to file an issue. -/// Map Rust's `std::env::consts::OS` to the conventional platform key -/// used in `[[host_tools]].install_hints`. The deps-management package-system -/// schema uses unix-y names (`darwin` for macOS, matching bash and -/// `uname`); Rust's runtime constant is `"macos"`. Other names match -/// what users would expect (`linux`, `windows`, `freebsd`, etc.). -fn install_hints_key_for_current_os() -> &'static str { - match std::env::consts::OS { - "macos" => "darwin", - other => other, + chain.push(target.name.clone()); + + // Resolve deps first; sort by name so iteration order is stable. + let mut dep_shas: Vec<(DepRef, [u8; 32])> = Vec::with_capacity(target.depends_on.len()); + for dref in &target.depends_on { + let child = registry.load(&dref.name)?; + if child.version != dref.version { + return Err(format!( + "{} depends on {}@{}, but registry has {}", + target.spec(), + dref.name, + dref.version, + child.spec() + )); + } + let child_sha = compute_sha_with_global_toolchain_inputs( + &child, + registry, + arch, + abi_version, + memo, + chain, + global_toolchain_inputs_override, + )?; + dep_shas.push((dref.clone(), child_sha)); } -} + dep_shas.sort_by(|a, b| a.0.name.cmp(&b.0.name)); -fn render_probe_failures(target: &DepsManifest, failures: &[ProbeFailure]) -> String { - let mut out = String::new(); - out.push_str(&format!( - "{}: {} host-tool requirement{} unsatisfied:\n", - target.spec(), - failures.len(), - if failures.len() == 1 { "" } else { "s" } - )); - for f in failures { - out.push_str(&format!(" - {f}\n")); - let tool_name = match f { - ProbeFailure::Missing { tool, .. } - | ProbeFailure::BadOutput { tool, .. } - | ProbeFailure::BadVersion { tool, .. } - | ProbeFailure::TooOld { tool, .. } => tool, - }; - if let Some(decl) = target - .host_tools - .iter() - .find(|d: &&HostTool| &d.name == tool_name) - { - let os = install_hints_key_for_current_os(); - if let Some(hint) = decl.install_hints.get(os) { - out.push_str(&format!(" install hint ({os}): {hint}\n")); - } else if !decl.install_hints.is_empty() { - let keys: Vec<&str> = decl.install_hints.keys().map(String::as_str).collect(); - out.push_str(&format!( - " no {os} install hint; available platforms: [{}]\n", - keys.join(", ") - )); + chain.pop(); + + let build_inputs = build_input_digests(target, registry)?; + let global_toolchain_inputs = match target.kind { + ManifestKind::Library | ManifestKind::Program => { + match global_toolchain_inputs_override { + Some(inputs) => inputs.to_vec(), + None => global_package_toolchain_digests()?, } } - } - out -} + ManifestKind::Source => Vec::new(), + }; + let fork_instrument_tool_inputs = if package_uses_fork_instrument_tool(target) { + fork_instrument_tool_digests()? + } else { + Vec::new() + }; -/// Process-lifetime memo of `(name, arch, exact cache identity) → -/// ensure_built_uncached`'s result. Within a single `xtask` invocation (e.g. one -/// `archive-stage` run, or a `build-deps resolve` walk that pulls a -/// shared dep transitively), a manifest reached via multiple dependents -/// (mariadb is reached 6× during a force-rebuild-all: directly + via -/// lamp + via mariadb-test + via mariadb-vfs ×2) otherwise re-runs its -/// full source build N times — ~80 minutes of pointless work for -/// mariadb alone. The memo collapses that to one build per -/// `(name, arch)`. -/// -/// Caches BOTH `Ok` (so subsequent dependents reuse the resolved -/// path) and `Err` (so a failed manifest doesn't waste 10 more -/// minutes per dependent re-discovering the same failure). Cycle -/// errors are intentionally NOT cached — those depend on the call -/// stack at the moment of detection, and caching them could leak a -/// stale cycle result into a later acyclic traversal. -/// -/// Lifetime: process-only. A fresh xtask invocation starts with an -/// empty memo, which keeps CI semantics intact (every run from -/// scratch retries any failures). -/// -/// Key dimensions: -/// * `cache_root` — same process can host independent test cases -/// (cargo runs tests in parallel within one process; each test -/// uses a fresh tempdir). In production there's only ever one -/// cache_root per run, so this dimension is invisible to the -/// force-rebuild path. -/// * `name` — the manifest's identifier within its registry. -/// * `arch` — wasm32 vs wasm64. The same name builds independently -/// per-arch. -/// * `cache_identity` — the full recipe/dependency/toolchain digest computed -/// from the current registry. This prevents a result from surviving a -/// build.toml edit or being reused for a same-named package from another -/// registry inside one long-lived process. -/// * `was_force_rebuild` — `force_source_build` bypasses the -/// on-disk cache. Memoizing across the force-rebuild boundary -/// would mean a no-force result satisfies a later force request, -/// defeating the bypass intent. Keep them as separate slots so -/// a force-call after a no-force-call still rebuilds. In -/// a force-rebuild-all loop every call has the same flag, so the -/// memo collapses N calls per (name, arch) into 1 build — the -/// actual optimization we wanted. -/// * `fetch_only` — fetch-only failures must not poison later normal -/// resolves, which are allowed to build from source. -type BuildMemoKey = (PathBuf, String, TargetArch, [u8; 32], bool, bool); -type BuildMemoValue = Result<(PathBuf, BTreeSet), String>; - -fn build_memo() -> &'static Mutex> { - static MEMO: OnceLock>> = OnceLock::new(); - MEMO.get_or_init(|| Mutex::new(BTreeMap::new())) -} - -/// Cycle-error sentinel — these errors must NOT be memoized because -/// they describe the call stack at detection time, not a property of -/// the manifest. A later acyclic call for the same node should be -/// allowed to proceed. -fn is_cycle_error(e: &str) -> bool { - e.starts_with("cycle while building:") -} - -/// Fast path for archive-only resolver callers. -/// -/// Browser/dev-server preparation needs to materialize self-contained program -/// archives into `binaries/`. If one of those programs has a stale or corrupt -/// dependency archive, resolving dependencies first can incorrectly force a -/// source build even though the target archive itself is valid. Keep normal -/// source-build resolution unchanged, but allow program archive fetches in -/// binary-materialization mode to satisfy the request before walking deps. -/// -/// Fetch-only CI materialization is stricter: it accepts only a valid cache -/// entry or prebuilt archive for the target package and never falls through to -/// dependency resolution/source builds. -fn try_fetch_without_deps( - target: &DepsManifest, - registry: &Registry, - arch: TargetArch, - abi_version: u32, - opts: &ResolveOpts<'_>, - memo: &mut BTreeMap, -) -> Result, String> { - let binary_materialization_fast_path = opts.binaries_dir.is_some() - && matches!(target.kind, ManifestKind::Program) - && !target.program_outputs.is_empty(); - if (!opts.fetch_only && !binary_materialization_fast_path) - || !matches!(target.kind, ManifestKind::Library | ManifestKind::Program) - { - return Ok(None); + let mut h = Sha256::new(); + match target.kind { + ManifestKind::Source => { + h.update(b"wasm-posix-pkg-source\n"); + h.update(target.name.as_bytes()); + h.update(b"\n"); + h.update(target.version.as_bytes()); + h.update(b"\n"); + h.update(target.revision.to_le_bytes()); + h.update(b"\n"); + // No target_arch, no abi_version — sources are arch/ABI-agnostic. + h.update(target.source.url.as_bytes()); + h.update(b"\n"); + h.update(target.source.sha256.as_bytes()); + h.update(b"\n"); + } + ManifestKind::Library | ManifestKind::Program => { + h.update(b"wasm-posix-pkg\n"); + h.update(target.name.as_bytes()); + h.update(b"\n"); + h.update(target.version.as_bytes()); + h.update(b"\n"); + h.update(target.revision.to_le_bytes()); + h.update(b"\n"); + h.update(arch.as_str().as_bytes()); + h.update(b"\n"); + h.update(abi_version.to_le_bytes()); + h.update(b"\n"); + h.update(target.source.url.as_bytes()); + h.update(b"\n"); + h.update(target.source.sha256.as_bytes()); + h.update(b"\n"); + // Fold in declared outputs so changing what a build is + // expected to produce invalidates the cache. Without this, + // renaming a program's `wasm = "..."` (or any library + // libs/headers/pkgconfig/files path) leaves cache_key_sha + // unchanged — the resolver then serves a canonical + // directory that doesn't match the new declaration and + // archive-stage packs broken archives. Bug discovered in + // PR #384 (lamp.vfs → lamp.vfs.zst). + // + // Ordering: hashed in authored Vec order (no sort). That + // matches how consumers like `mirror_program_outputs` + // iterate, and re-ordering is a real semantic change + // worth invalidating on. `b"|"` separators keep + // adjacent strings unambiguous (e.g. lib `"a"` + `"bc"` ≠ + // lib `"ab"` + `"c"`). A section tag (`"libs:"`, etc.) + // before each list prevents cross-section collisions. + h.update(b"outputs.libs:\n"); + for s in &target.outputs.libs { + h.update(s.as_bytes()); + h.update(b"|"); + } + h.update(b"\n"); + h.update(b"outputs.headers:\n"); + for s in &target.outputs.headers { + h.update(s.as_bytes()); + h.update(b"|"); + } + h.update(b"\n"); + h.update(b"outputs.pkgconfig:\n"); + for s in &target.outputs.pkgconfig { + h.update(s.as_bytes()); + h.update(b"|"); + } + h.update(b"\n"); + // Preserve every existing package's cache key: the additive files + // field participates only when authored. A universally empty + // section would invalidate the entire package registry merely for + // learning a new output kind. + if !target.outputs.files.is_empty() { + h.update(b"outputs.files:v1\n"); + for s in &target.outputs.files { + h.update((s.len() as u64).to_le_bytes()); + h.update(s.as_bytes()); + } + } + h.update(b"program_outputs:\n"); + for out in &target.program_outputs { + h.update(out.name.as_bytes()); + h.update(b"|"); + h.update(out.wasm.as_bytes()); + if out.fork_instrumentation != ForkInstrumentationPolicy::Auto { + h.update(b"|fork_instrumentation="); + h.update(out.fork_instrumentation.as_str().as_bytes()); + } + h.update(b"\n"); + } + // Additive program runtime closure. Keep the section absent for + // existing manifests so learning this schema does not invalidate + // every historical package cache key. + if !target.runtime_files.is_empty() { + h.update(b"runtime_files:v1\n"); + for runtime_file in &target.runtime_files { + for field in [ + runtime_file.artifact.as_bytes(), + runtime_file.guest_path.as_bytes(), + ] { + h.update((field.len() as u64).to_le_bytes()); + h.update(field); + } + h.update(runtime_file.mode.to_le_bytes()); + } + } + } } - - let force_rebuild = opts - .force_source_build - .map(|s| s.contains(&target.name)) - .unwrap_or(false); - if force_rebuild { - if opts.fetch_only { - return Err(format!( - "{}: fetch-only resolve cannot honor force source-build for arch {}", - target.spec(), - arch.as_str(), - )); + if !build_inputs.is_empty() { + h.update(b"build-inputs:\n"); + for input in &build_inputs { + h.update(input.label.as_bytes()); + h.update(b"\n"); + h.update(input.digest); + h.update(b"\n"); } - return Ok(None); } - - if !opts.fetch_only { - if let Some(lr) = opts.local_libs { - let override_dir = lr.join(&target.name).join("build"); - if override_dir.is_dir() { - return Ok(Some(override_dir)); - } + if !global_toolchain_inputs.is_empty() { + h.update(b"global-toolchain-inputs:\n"); + for input in &global_toolchain_inputs { + h.update(input.label.as_bytes()); + h.update(b"\n"); + h.update(input.digest); + h.update(b"\n"); } } - - let mut chain: Vec = Vec::new(); - let sha = compute_sha(target, registry, arch, abi_version, memo, &mut chain)?; - let canonical = canonical_path(opts.cache_root, target, arch, &sha); - let cache_key_sha_hex = hex(&sha); - - if canonical.is_dir() { - match validate_cache_entry(target, &canonical, arch, abi_version, &cache_key_sha_hex) { - Ok(()) => return Ok(Some(canonical)), - Err(e) => { - eprintln!( - "warning: ignoring stale cached artifact for {} at {} ({})", - target.spec(), - canonical.display(), - e, - ); - remove_cache_entry(&canonical, &cache_key_sha_hex).map_err(|remove_err| { - format!( - "clear stale cache entry {} after validation failure: {remove_err}", - canonical.display() - ) - })?; - } + if !fork_instrument_tool_inputs.is_empty() { + h.update(b"fork-instrument-tool-inputs:\n"); + for input in &fork_instrument_tool_inputs { + h.update(input.label.as_bytes()); + h.update(b"\n"); + h.update(input.digest); + h.update(b"\n"); } } + for (dref, dsha) in &dep_shas { + h.update(dref.name.as_bytes()); + h.update(b"@"); + h.update(dref.version.as_bytes()); + h.update(b":"); + h.update(hex(dsha).as_bytes()); + h.update(b"\n"); + } - if let Some(binary) = target.binary.get(&arch) { - match remote_fetch::fetch_and_install( - binary, - &canonical, - target, - arch, - abi_version, - &cache_key_sha_hex, - ) { - Ok(()) => match validate_cache_entry( - target, - &canonical, - arch, - abi_version, - &cache_key_sha_hex, - ) { - Ok(()) => return Ok(Some(canonical)), - Err(e) => { - eprintln!( - "warning: direct binary fetch for {} from {} produced \ - a stale artifact ({}); {}", - target.spec(), - binary.archive_url, - e, - fetch_fallback_phrase(opts.fetch_only), - ); - let _ = remove_cache_entry(&canonical, &cache_key_sha_hex); - } - }, - Err(e) => { - eprintln!( - "warning: direct binary fetch for {} from {} failed ({}); \ - {}", - target.spec(), - binary.archive_url, - e, - fetch_fallback_phrase(opts.fetch_only), - ); - } - } - } + let out: [u8; 32] = h.finalize().into(); + memo.insert(memo_key, out); + Ok(out) +} - if let Some(()) = try_index_install( - target, - arch, - abi_version, - &canonical, - &cache_key_sha_hex, - opts.fetch_only, - ) { - return Ok(Some(canonical)); - } +#[derive(Clone, Debug)] +struct BuildInputDigest { + label: String, + digest: [u8; 32], +} - if opts.fetch_only { - return Err(format!( - "{}: fetch-only resolve could not install a valid archive for arch {}; \ - run package staging/prepare to publish this package instead of \ - source-building during binary materialization", - target.spec(), - arch.as_str(), - )); - } +const GLOBAL_PACKAGE_TOOLCHAIN_INPUTS: &[&str] = &[ + "flake.nix", + "flake.lock", + "rust-toolchain.toml", + "scripts/dev-shell.sh", + "scripts/build-musl.sh", + "scripts/install-overlay-headers.sh", + ".github/actions/package-archive-build", + ".github/actions/package-toolchain", + ".github/actions/fetch-submodules", + ".github/actions/download-run-artifacts", + "libc/glue", + "libc/musl-overlay", + "libc/musl", + "sdk/activate.sh", + "sdk/bin", + "sdk/config.site", + "sdk/package.json", + "sdk/package-lock.json", + "sdk/src", +]; - Ok(None) +const FORK_INSTRUMENT_TOOL_INPUTS: &[&str] = &[ + "Cargo.toml", + "crates/fork-instrument/Cargo.toml", + "crates/fork-instrument/src", + "scripts/build-fork-instrument-tool.sh", + "scripts/run-wasm-fork-instrument.sh", +]; + +static GLOBAL_PACKAGE_TOOLCHAIN_DIGESTS: OnceLock, String>> = + OnceLock::new(); +static FORK_INSTRUMENT_TOOL_DIGESTS: OnceLock, String>> = + OnceLock::new(); + +fn global_package_toolchain_digests() -> Result, String> { + GLOBAL_PACKAGE_TOOLCHAIN_DIGESTS + .get_or_init(|| { + global_package_build_input_digests_for(&repo_root(), GLOBAL_PACKAGE_TOOLCHAIN_INPUTS) + }) + .clone() } -/// Resolve `target`, returning its on-disk path *and* the set of -/// transitively-resolved lib paths underneath it (its direct deps, their -/// deps, and so on — but NOT `target`'s own path; the caller adds that). -/// -/// The transitive set lets the caller compose -/// `WASM_POSIX_DEP_PKG_CONFIG_PATH` for the build script: every node -/// gets every descendant's `lib/pkgconfig/` dir, which mirrors how -/// pkg-config follows `Requires.private` chains. -/// -/// Deduped via `BTreeSet` so a diamond dep (`libZ -> {libA, libB} -> -/// libCommon`) only contributes `libCommon`'s path once. -fn ensure_built_inner( - target: &DepsManifest, - registry: &Registry, - arch: TargetArch, - abi_version: u32, - opts: &ResolveOpts<'_>, - memo: &mut BTreeMap, - building: &mut Vec, -) -> Result<(PathBuf, BTreeSet), String> { - // Process-lifetime memo: the same (name, arch) often gets - // requested multiple times within one resolver run via different - // dep chains. Without this, mariadb wasm32 source-builds 4 times - // in a single force-rebuild-all (lamp, mariadb, mariadb-test, - // mariadb-vfs each independently demand it). See `build_memo`'s - // doc comment for full rationale. - // Compute the exact current identity before consulting the process memo. - // `ensure_built()` supplies a fresh hash memo for every top-level call, so - // an in-process build.toml edit cannot inherit the prior call's digest; - // recursive lookups in one unchanged graph remain cheap memo hits. - let mut identity_chain = Vec::new(); - let cache_identity = compute_sha( - target, - registry, - arch, - abi_version, - memo, - &mut identity_chain, - )?; - let was_force_rebuild = opts - .force_source_build - .map(|s| s.contains(&target.name)) - .unwrap_or(false); - let memo_key: BuildMemoKey = ( - opts.cache_root.to_path_buf(), - target.name.clone(), - arch, - cache_identity, - was_force_rebuild, - opts.fetch_only, - ); - { - let cache = build_memo().lock().unwrap(); - if let Some(cached) = cache.get(&memo_key) { - return cached.clone(); - } - } +fn fork_instrument_tool_digests() -> Result, String> { + FORK_INSTRUMENT_TOOL_DIGESTS + .get_or_init(|| { + let root = repo_root(); + let mut digests = + global_package_build_input_digests_for(&root, FORK_INSTRUMENT_TOOL_INPUTS)?; + digests.push(BuildInputDigest { + label: "cargo-metadata:fork-instrument-build-deps".to_string(), + digest: fork_instrument_cargo_dependency_digest(&root)?, + }); + Ok(digests) + }) + .clone() +} - let result = ensure_built_uncached(target, registry, arch, abi_version, opts, memo, building); +fn package_uses_fork_instrument_tool(target: &DepsManifest) -> bool { + matches!(target.kind, ManifestKind::Program) + && target + .program_outputs + .iter() + .any(|out| out.fork_instrumentation != ForkInstrumentationPolicy::Disabled) +} - // Don't poison the cache with cycle errors — those reflect the - // call stack at the moment of detection, not a stable property - // of the manifest. Everything else (Ok path + non-cycle Err) - // gets memoized. - let should_memo = match &result { - Ok(_) => true, - Err(e) => !is_cycle_error(e), - }; - if should_memo { - build_memo() - .lock() - .unwrap() - .insert(memo_key, result.clone()); - } - result +#[derive(Debug, serde::Deserialize)] +struct CargoLock { + #[serde(default)] + package: Vec, } -fn ensure_built_uncached( - target: &DepsManifest, - registry: &Registry, - arch: TargetArch, - abi_version: u32, - opts: &ResolveOpts<'_>, - memo: &mut BTreeMap, - building: &mut Vec, -) -> Result<(PathBuf, BTreeSet), String> { - if building.iter().any(|s| s == &target.name) { +#[derive(Debug, serde::Deserialize)] +struct CargoLockPackage { + name: String, + version: String, + source: Option, + checksum: Option, +} + +fn fork_instrument_cargo_dependency_digest(root: &Path) -> Result<[u8; 32], String> { + let host_target = host_target_triple()?; + let output = Command::new("cargo") + .arg("metadata") + .arg("--format-version=1") + .arg("--locked") + .arg("--filter-platform") + .arg(&host_target) + .current_dir(root) + .output() + .map_err(|e| format!("run cargo metadata for fork-instrument cache key: {e}"))?; + if !output.status.success() { return Err(format!( - "cycle while building: {} -> {}", - building.join(" -> "), - target.name + "cargo metadata for fork-instrument cache key failed: {}", + String::from_utf8_lossy(&output.stderr).trim() )); } - building.push(target.name.clone()); - if let Some(path) = try_fetch_without_deps(target, registry, arch, abi_version, opts, memo)? { - building.pop(); - return Ok((path, BTreeSet::new())); - } + let metadata: serde_json::Value = serde_json::from_slice(&output.stdout) + .map_err(|e| format!("parse cargo metadata for fork-instrument cache key: {e}"))?; + let lock_text = std::fs::read_to_string(root.join("Cargo.lock")) + .map_err(|e| format!("read Cargo.lock for fork-instrument cache key: {e}"))?; + let lock: CargoLock = toml::from_str(&lock_text) + .map_err(|e| format!("parse Cargo.lock for fork-instrument cache key: {e}"))?; + fork_instrument_cargo_dependency_digest_from_metadata(root, &metadata, &lock) +} - // Recursively resolve direct deps first; remember their paths so - // we can surface them to the build script via env vars. The - // transitive set accumulates every dep path in the subgraph, - // deduped — diamond deps must only contribute once. - // - // We track each direct dep's `kind` alongside its path so that - // `build_into_cache` can choose the env-var suffix per design 12: - // library/program → `WASM_POSIX_DEP__DIR` (built artifact - // root); source → `WASM_POSIX_DEP__SRC_DIR` (unbuilt source - // tree). Build scripts then self-document what shape they're - // consuming via the suffix. - let mut dep_dirs: BTreeMap = BTreeMap::new(); - let mut transitive: BTreeSet = BTreeSet::new(); - for dref in &target.depends_on { - let dep_m = registry.load(&dref.name)?; - if dep_m.version != dref.version { - return Err(format!( - "{} depends on {}@{}, but registry has {}", - target.spec(), - dref.name, - dref.version, - dep_m.spec() - )); - } - // Per the wasm64 build policy (memory/wasm64-build-policy.md): - // only MariaDB and PHP need wasm64 binaries; everything else - // is wasm32-only. So a wasm64 program (e.g. mariadb-vfs) - // depending on a wasm32-only dep (e.g. dinit) is the common - // case, not a misconfiguration. When the parent arch isn't in - // the dep's target_arches, fall back to wasm32 (the universal - // arch) for that dep. The resolver places the dep's binaries - // under binaries/programs/wasm32/, where build scripts' - // arch-agnostic tryResolveBinary("programs/.wasm") finds - // them. The kernel runs mixed-arch programs. - let dep_arch = if dep_m.target_arches.contains(&arch) { - arch - } else if dep_m.target_arches.contains(&TargetArch::Wasm32) { - TargetArch::Wasm32 - } else { - return Err(format!( - "{} depends on {}@{} (arch {}), but {} declares neither {} nor wasm32 in target_arches (declared: {:?})", - target.spec(), - dref.name, - dref.version, - arch.as_str(), - dep_m.spec(), - arch.as_str(), - dep_m - .target_arches - .iter() - .map(|a| a.as_str()) - .collect::>(), - )); - }; - let (dep_path, dep_transitive) = ensure_built_inner( - &dep_m, - registry, - dep_arch, - abi_version, - opts, - memo, - building, - )?; - // Place binaries/programs// symlinks for each - // program dep so consumer build scripts can find them via - // `tryResolveBinary("programs/.wasm")`. Only kicks in when - // the caller opts in with binaries_dir; other ensure_built - // consumers leave binaries_dir = None and no symlinks land. - // Library deps and source deps are linked at compile time via - // WASM_POSIX_DEP_* env vars and don't need a binaries/ entry. - if let Some(bdir) = opts.binaries_dir { - if matches!(dep_m.kind, ManifestKind::Program) && !dep_m.program_outputs.is_empty() { - place_binaries_symlinks(&dep_m, &dep_path, bdir, dep_arch)?; - } - } - dep_dirs.insert( - dep_m.name.clone(), - DirectDep { - path: dep_path.clone(), - kind: dep_m.kind, - }, - ); - transitive.insert(dep_path); - transitive.extend(dep_transitive); +fn host_target_triple() -> Result { + let output = Command::new("rustc") + .arg("-vV") + .output() + .map_err(|e| format!("run rustc -vV: {e}"))?; + if !output.status.success() { + return Err(format!( + "rustc -vV failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + )); } + let stdout = String::from_utf8_lossy(&output.stdout); + stdout + .lines() + .find_map(|line| line.strip_prefix("host: ").map(str::to_owned)) + .filter(|host| !host.is_empty()) + .ok_or_else(|| "rustc -vV did not report host target".to_string()) +} - building.pop(); +fn fork_instrument_cargo_dependency_digest_from_metadata( + root: &Path, + metadata: &serde_json::Value, + lock: &CargoLock, +) -> Result<[u8; 32], String> { + let packages = metadata_array(metadata, "packages")?; + let nodes = metadata + .get("resolve") + .and_then(|resolve| resolve.get("nodes")) + .and_then(|nodes| nodes.as_array()) + .ok_or_else(|| "cargo metadata missing resolve.nodes".to_string())?; - // Local-libs override: hand-patched source wins. The override dir - // still contributes to `transitive` for any consumer above us. - if let Some(lr) = opts.local_libs { - let override_dir = lr.join(&target.name).join("build"); - if override_dir.is_dir() { - return Ok((override_dir, transitive)); + let mut packages_by_id: BTreeMap = BTreeMap::new(); + let mut root_package_id: Option = None; + for package in packages { + let id = metadata_str(package, "id")?.to_string(); + let name = metadata_str(package, "name")?; + let manifest_path = metadata_str(package, "manifest_path")?; + if name == "fork-instrument" + && manifest_path.ends_with("/crates/fork-instrument/Cargo.toml") + { + root_package_id = Some(id.clone()); } + packages_by_id.insert(id, package); } - // Compute canonical cache path. - let mut chain: Vec = Vec::new(); - let sha = compute_sha(target, registry, arch, abi_version, memo, &mut chain)?; - let canonical = canonical_path(opts.cache_root, target, arch, &sha); - let cache_key_sha_hex = hex(&sha); - - let force_rebuild = opts - .force_source_build - .map(|s| s.contains(&target.name)) - .unwrap_or(false); + let root_package_id = root_package_id + .ok_or_else(|| "cargo metadata missing fork-instrument package".to_string())?; + let mut nodes_by_id: BTreeMap = BTreeMap::new(); + for node in nodes { + nodes_by_id.insert(metadata_str(node, "id")?.to_string(), node); + } - // Cache hit: validate before using it. The cache key includes the - // numeric kernel ABI, but fork-continuation mechanism changes have - // previously produced stale artifacts with a matching ABI number - // (legacy Asyncify exports instead of wpk_fork_*). Reject those so - // the resolver can fetch a current remote artifact or source-build. - if !force_rebuild && canonical.is_dir() { - match validate_cache_entry(target, &canonical, arch, abi_version, &cache_key_sha_hex) { - Ok(()) => return Ok((canonical, transitive)), - Err(e) => { - eprintln!( - "warning: ignoring stale cached artifact for {} at {} ({})", - target.spec(), - canonical.display(), - e, - ); - remove_cache_entry(&canonical, &cache_key_sha_hex).map_err(|remove_err| { - format!( - "clear stale cache entry {} after validation failure: {remove_err}", - canonical.display() - ) - })?; - } + let mut closure = BTreeSet::new(); + let mut stack = vec![root_package_id.clone()]; + while let Some(package_id) = stack.pop() { + if !closure.insert(package_id.clone()) { + continue; + } + let node = nodes_by_id + .get(&package_id) + .ok_or_else(|| format!("cargo metadata missing resolve node for {package_id}"))?; + for dep in selected_cargo_metadata_deps(node)? { + stack.push(dep); } } - if force_rebuild && canonical.is_dir() { - remove_cache_entry(&canonical, &cache_key_sha_hex) - .map_err(|e| format!("force-rebuild: clear {}: {e}", canonical.display()))?; + + let lock_checksums = cargo_lock_checksums(lock); + let mut entries = Vec::with_capacity(closure.len()); + for package_id in closure { + let package = packages_by_id + .get(&package_id) + .ok_or_else(|| format!("cargo metadata missing package for {package_id}"))?; + let node = nodes_by_id + .get(&package_id) + .ok_or_else(|| format!("cargo metadata missing resolve node for {package_id}"))?; + let stable_id = stable_cargo_package_id(root, package)?; + let features = sorted_string_array(node, "features")?; + let deps = selected_cargo_metadata_deps(node)? + .into_iter() + .map(|dep_id| { + let dep_package = packages_by_id + .get(&dep_id) + .ok_or_else(|| format!("cargo metadata missing package for {dep_id}"))?; + stable_cargo_package_id(root, dep_package) + }) + .collect::, _>>()?; + let lock_key = cargo_lock_key(package)?; + let checksum = lock_checksums.get(&lock_key).cloned().unwrap_or_default(); + entries.push((stable_id, features, deps, checksum)); } + entries.sort_by(|a, b| a.0.cmp(&b.0)); - // Run host-tool probes before any work that might invoke a build - // script (or fetch+extract a source-kind tarball). Cache hits skip - // this — probes are only needed when we might actually invoke - // `bash build-.sh` or similar work. Aggregate ALL probe - // failures so users fix everything in one round-trip. - if !target.host_tools.is_empty() { - let mut failures: Vec = Vec::new(); - for tool in &target.host_tools { - if let Err(e) = host_tool_probe::probe(tool) { - failures.push(e); - } + let mut h = Sha256::new(); + h.update(b"fork-instrument-cargo-build-deps-v1\n"); + for (stable_id, features, deps, checksum) in entries { + h.update(b"package\0"); + h.update(stable_id.as_bytes()); + h.update(b"\0checksum\0"); + h.update(checksum.as_bytes()); + h.update(b"\0features\0"); + for feature in features { + h.update(feature.as_bytes()); + h.update(b"\0"); } - if !failures.is_empty() { - return Err(render_probe_failures(target, &failures)); + h.update(b"deps\0"); + for dep in deps { + h.update(dep.as_bytes()); + h.update(b"\0"); } + h.update(b"\n"); } + Ok(h.finalize().into()) +} - // Cache-miss dispatch. Three flavors of recipe: - // - // (Source, None) — default fetch+extract from `[source]`. - // Source-kind manifests never carry - // `[binary]` (Task C.1 enforces), so this - // branch short-circuits before the binary - // block. - // (Source, Some(_)) — override path (Task C.5): the manifest - // ships its own build script (e.g. patch - // overlay, git clone, multi-tarball - // assembly). Run it through - // `build_into_cache` with the standard - // env-var contract; validation is - // non-emptiness of OUT_DIR rather than a - // declared outputs list. - // (Library | Program,_) — try `package.pr.toml` / source - // `[binary]` direct archives first, then - // the `build.toml` index path, then fall - // back to the build script. - match (target.kind, target.build.script_path.is_some()) { - (ManifestKind::Source, false) => { - if opts.fetch_only { - return Err(format!( - "{}: fetch-only resolve cannot fetch source package fallback for arch {}", - target.spec(), - arch.as_str(), - )); - } - let parent = canonical - .parent() - .ok_or_else(|| format!("canonical path has no parent: {}", canonical.display()))?; - std::fs::create_dir_all(parent) - .map_err(|e| format!("create cache parent {}: {e}", parent.display()))?; - let tmp = parent.join(format!( - "{}.tmp-{}", - canonical - .file_name() - .expect("canonical path has a filename") - .to_string_lossy(), - std::process::id() - )); - if tmp.exists() { - std::fs::remove_dir_all(&tmp) - .map_err(|e| format!("clean stale {}: {e}", tmp.display()))?; - } - if let Err(e) = - source_extract::fetch_and_extract(&target.source.url, &target.source.sha256, &tmp) - { - let _ = std::fs::remove_dir_all(&tmp); - return Err(format!( - "{}: source fetch+extract failed: {e}", - target.spec() - )); - } - if let Err(e) = - write_cache_provenance(target, &canonical, arch, abi_version, &cache_key_sha_hex) - { - let _ = std::fs::remove_dir_all(&tmp); - return Err(e); - } - // Race against a peer process that finished its own extract - // first: keep theirs, drop ours. Identical inputs produce - // identical outputs. - if canonical.exists() { - let _ = std::fs::remove_dir_all(&tmp); - validate_cache_entry(target, &canonical, arch, abi_version, &cache_key_sha_hex)?; - return Ok((canonical, transitive)); - } - std::fs::rename(&tmp, &canonical) - .map_err(|e| format!("rename {} -> {}: {e}", tmp.display(), canonical.display()))?; - Ok((canonical, transitive)) - } - (ManifestKind::Source, true) => { - if opts.fetch_only { - return Err(format!( - "{}: fetch-only resolve cannot run source package build script for arch {}", - target.spec(), - arch.as_str(), - )); - } - // Override path: run the script. No remote-binary fetch for - // sources (`[binary]` is rejected at parse time for source - // kind), so we go straight to `build_into_cache`. - let pkgconfig_path = compose_pkgconfig_path(&transitive); - let repo_root = opts - .repo_root - .map(Path::to_path_buf) - .unwrap_or_else(crate::repo_root); - build_into_cache( - target, - arch, - abi_version, - &cache_key_sha_hex, - &canonical, - &dep_dirs, - &pkgconfig_path, - &repo_root, - )?; - Ok((canonical, transitive)) - } - (ManifestKind::Library | ManifestKind::Program, _) => { - // Resolution priority 3a: direct archive fetch from the - // source manifest's `[binary]` map. In normal source - // package.toml files this map is empty post index-ledger - // migration, but CI writes sibling `package.pr.toml` - // overlays with direct file:// archives for same-run - // matrix outputs. Those must win over the durable - // `build.toml` index below. - // - // Resolution priority 3b: index-based remote fetch. The - // resolver loads the sibling `build.toml`, resolves its - // `[binary]` block to an index URL (or a direct archive - // URL), then looks up this package's entry. Status - // `success` fetches the current archive; status - // `failed`/`pending`/`building` falls back to the - // last-green `fallback_*` archive when one is preserved. - // - // Any failure along the way logs and falls through to the - // source build — a remote-fetch error should never cause - // the resolver to refuse to produce an artifact. - // - // `force_rebuild` short-circuits remote fetch entirely. - if !force_rebuild { - if let Some(binary) = target.binary.get(&arch) { - match remote_fetch::fetch_and_install( - binary, - &canonical, - target, - arch, - abi_version, - &cache_key_sha_hex, - ) { - Ok(()) => match validate_cache_entry( - target, - &canonical, - arch, - abi_version, - &cache_key_sha_hex, - ) { - Ok(()) => return Ok((canonical, transitive)), - Err(e) => { - eprintln!( - "warning: direct binary fetch for {} from {} produced \ - a stale artifact ({}); {}", - target.spec(), - binary.archive_url, - e, - fetch_fallback_phrase(opts.fetch_only), - ); - let _ = remove_cache_entry(&canonical, &cache_key_sha_hex); - } - }, - Err(e) => { - eprintln!( - "warning: direct binary fetch for {} from {} failed ({}); \ - {}", - target.spec(), - binary.archive_url, - e, - fetch_fallback_phrase(opts.fetch_only), - ); - } - } - } - if let Some(()) = try_index_install( - target, - arch, - abi_version, - &canonical, - &cache_key_sha_hex, - opts.fetch_only, - ) { - return Ok((canonical, transitive)); - } - } - - if opts.fetch_only { - return Err(format!( - "{}: fetch-only resolve could not install a valid archive for arch {}; \ - package staging or the durable release must provide one", - target.spec(), - arch.as_str(), - )); - } - - let pkgconfig_path = compose_pkgconfig_path(&transitive); - let repo_root = opts - .repo_root - .map(Path::to_path_buf) - .unwrap_or_else(crate::repo_root); - build_into_cache( - target, - arch, - abi_version, - &cache_key_sha_hex, - &canonical, - &dep_dirs, - &pkgconfig_path, - &repo_root, - )?; - Ok((canonical, transitive)) +fn selected_cargo_metadata_deps(node: &serde_json::Value) -> Result, String> { + let deps = match node.get("deps").and_then(|deps| deps.as_array()) { + Some(deps) => deps, + None => return Ok(Vec::new()), + }; + let mut out = Vec::new(); + for dep in deps { + if !cargo_metadata_dep_is_build_input(dep)? { + continue; } + out.push(metadata_str(dep, "pkg")?.to_string()); } + out.sort(); + out.dedup(); + Ok(out) } -/// Attempt to install a prebuilt archive from this package's -/// `build.toml`-declared binary source. Returns `Some(())` on success -/// (caller returns the canonical path); returns `None` for any -/// "fall through to source build" condition (no build.toml, no -/// archive in the index, network failure, sha mismatch, etc.). -/// -/// Logging is on stderr (matching the prior remote-fetch -/// implementation's UX): users see warnings about why the index -/// path was skipped. Normal resolves then build from source; fetch-only -/// resolves turn the miss into an error at the caller. -fn fetch_fallback_phrase(fetch_only: bool) -> &'static str { - if fetch_only { - "source builds disabled by fetch-only mode" - } else { - "falling back to source build" +fn cargo_metadata_dep_is_build_input(dep: &serde_json::Value) -> Result { + let dep_kinds = dep + .get("dep_kinds") + .and_then(|dep_kinds| dep_kinds.as_array()) + .ok_or_else(|| "cargo metadata dependency missing dep_kinds".to_string())?; + Ok(dep_kinds.iter().any(|kind| { + kind.get("kind") + .and_then(|kind| kind.as_str()) + .map(|kind| kind == "build") + .unwrap_or(true) + })) +} + +fn stable_cargo_package_id(root: &Path, package: &serde_json::Value) -> Result { + let name = metadata_str(package, "name")?; + let version = metadata_str(package, "version")?; + let source = package.get("source").and_then(|source| source.as_str()); + if let Some(source) = source { + return Ok(format!("{source}#{name}@{version}")); } + + let manifest_path = PathBuf::from(metadata_str(package, "manifest_path")?); + let rel_manifest = manifest_path.strip_prefix(root).unwrap_or(&manifest_path); + Ok(format!( + "path:{}#{name}@{version}", + rel_manifest.to_string_lossy() + )) } -fn try_index_install( - target: &DepsManifest, - arch: TargetArch, - abi_version: u32, - canonical: &Path, - cache_key_sha_hex: &str, - fetch_only: bool, -) -> Option<()> { - // 1. Load build.toml. Source manifests without one (e.g. an - // upstream package that hasn't been ported to the new schema - // yet) fall through silently — Phase 9's migration should - // leave every first-party package with a build.toml; the - // silent fall-through is for clean integration with - // third-party manifests that might not. - let build = BuildToml::load(&target.dir).ok()?; +fn cargo_lock_key(package: &serde_json::Value) -> Result<(String, String, String), String> { + Ok(( + metadata_str(package, "name")?.to_string(), + metadata_str(package, "version")?.to_string(), + package + .get("source") + .and_then(|source| source.as_str()) + .unwrap_or("") + .to_string(), + )) +} - // 2. Resolve the binary source to a concrete URL pair. Direct - // form: use the URL + sha verbatim. Indexed form: fetch - // index.toml + look up this package. CI can override indexed - // URLs with WASM_POSIX_BINARY_INDEX_URL so staging/prepare - // jobs consume the release they are publishing instead of the - // committed durable-release default. - let (archive_url, archive_sha256) = match &build.binary { - BinarySource::Direct { url, sha256 } => (url.clone(), sha256.clone()), - BinarySource::Indexed { .. } => { - let index_url = std::env::var("WASM_POSIX_BINARY_INDEX_URL") - .ok() - .filter(|s| !s.is_empty()) - .or_else(|| build.binary.resolve_index_url(abi_version))?; - let cache_dir = default_cache_root().join("indexes"); - let index = match index_toml::fetch_index(&index_url, &cache_dir) { - Ok(idx) => idx, - Err(e) => { - eprintln!( - "warning: index fetch for {} from {} failed ({}); \ - {}", - target.spec(), - index_url, - e, - fetch_fallback_phrase(fetch_only), - ); - return None; - } - }; - if index.abi_version != abi_version { - eprintln!( - "warning: index for {} from {} declares ABI {}, but resolver ABI is {}; \ - {}", - target.spec(), - index_url, - index.abi_version, - abi_version, - fetch_fallback_phrase(fetch_only), - ); - return None; - } - let entry = match index.lookup(&target.name, &target.version, arch) { - Some(e) => e, - None => { - eprintln!( - "warning: no index entry for {} in {}; \ - {}", - target.spec(), - index_url, - fetch_fallback_phrase(fetch_only), - ); - return None; - } - }; - // Pick the authoritative archive fields for the entry's - // current status. Success → current archive_*; other - // statuses → fallback_* if preserved; otherwise nothing - // usable and we fall through. - let (rel_url, sha) = match entry.status { - EntryStatus::Success - if entry.archive_url.is_some() && entry.archive_sha256.is_some() => - { - ( - entry.archive_url.as_ref().unwrap().clone(), - entry.archive_sha256.as_ref().unwrap().clone(), - ) - } - EntryStatus::Failed | EntryStatus::Pending | EntryStatus::Building - if entry.fallback_archive_url.is_some() - && entry.fallback_archive_sha256.is_some() => - { - eprintln!( - "note: {} index entry is status={:?}; \ - using last-green fallback archive", - target.spec(), - entry.status, - ); +fn cargo_lock_checksums(lock: &CargoLock) -> BTreeMap<(String, String, String), String> { + lock.package + .iter() + .filter_map(|package| { + package.checksum.as_ref().map(|checksum| { + ( ( - entry.fallback_archive_url.as_ref().unwrap().clone(), - entry.fallback_archive_sha256.as_ref().unwrap().clone(), - ) - } - _ => { - eprintln!( - "warning: {} index entry status={:?} has no usable archive; \ - {}", - target.spec(), - entry.status, - fetch_fallback_phrase(fetch_only), - ); - return None; - } - }; - (resolve_relative_url(&index_url, &rel_url), sha) - } - }; + package.name.clone(), + package.version.clone(), + package.source.clone().unwrap_or_default(), + ), + checksum.clone(), + ) + }) + }) + .collect() +} - // 3. Fetch + verify + install. Any failure (sha mismatch, arch - // mismatch, abi mismatch, cache_key mismatch, transport - // error) falls through. - match remote_fetch::fetch_and_install_direct( - &archive_url, - &archive_sha256, - canonical, - target, - arch, - abi_version, - cache_key_sha_hex, - ) { - Ok(()) => { - match validate_cache_entry(target, canonical, arch, abi_version, cache_key_sha_hex) { - Ok(()) => Some(()), - Err(e) => { - eprintln!( - "warning: index-based fetch for {} from {} produced \ - a stale artifact ({}); {}", - target.spec(), - archive_url, - e, - fetch_fallback_phrase(fetch_only), - ); - let _ = remove_cache_entry(canonical, cache_key_sha_hex); - None - } - } +fn metadata_array<'a>( + value: &'a serde_json::Value, + field: &str, +) -> Result<&'a Vec, String> { + value + .get(field) + .and_then(|value| value.as_array()) + .ok_or_else(|| format!("cargo metadata missing {field} array")) +} + +fn metadata_str<'a>(value: &'a serde_json::Value, field: &str) -> Result<&'a str, String> { + value + .get(field) + .and_then(|value| value.as_str()) + .ok_or_else(|| format!("cargo metadata missing {field} string")) +} + +fn sorted_string_array(value: &serde_json::Value, field: &str) -> Result, String> { + let mut out = value + .get(field) + .and_then(|value| value.as_array()) + .ok_or_else(|| format!("cargo metadata missing {field} array"))? + .iter() + .map(|value| { + value + .as_str() + .map(str::to_string) + .ok_or_else(|| format!("cargo metadata {field} array contains a non-string")) + }) + .collect::, _>>()?; + out.sort(); + Ok(out) +} + +fn global_package_build_input_digests_for( + root: &Path, + inputs: &[&str], +) -> Result, String> { + let mut out = Vec::with_capacity(inputs.len()); + for input in inputs { + let path = root.join(input); + if !path.exists() { + return Err(format!( + "global package build input {:?} not found at {}", + input, + path.display() + )); } - Err(e) => { - eprintln!( - "warning: index-based fetch for {} from {} failed ({}); \ - {}", - target.spec(), - archive_url, - e, - fetch_fallback_phrase(fetch_only), - ); - None + out.push(BuildInputDigest { + label: (*input).to_string(), + digest: hash_global_package_build_input(root, input, &path)?, + }); + } + Ok(out) +} + +fn hash_global_package_build_input( + root: &Path, + input: &str, + path: &Path, +) -> Result<[u8; 32], String> { + if input == "libc/musl" { + if let Some(digest) = hash_gitlink_input(root, input)? { + return Ok(digest); } } + hash_build_input(path) +} + +fn hash_gitlink_input(root: &Path, input: &str) -> Result, String> { + let output = match Command::new("git") + .arg("-C") + .arg(root) + .arg("ls-tree") + .arg("HEAD") + .arg("--") + .arg(input) + .output() + { + Ok(output) => output, + Err(_) => return Ok(None), + }; + if !output.status.success() { + return Ok(None); + } + let stdout = String::from_utf8_lossy(&output.stdout); + let Some(line) = stdout.lines().next() else { + return Ok(None); + }; + let Some(rest) = line.strip_prefix("160000 commit ") else { + return Ok(None); + }; + let Some((object_id, _path)) = rest.split_once('\t') else { + return Err(format!("unexpected gitlink entry for {input:?}: {line:?}")); + }; + + let mut h = Sha256::new(); + h.update(b"gitlink\0"); + h.update(input.as_bytes()); + h.update(b"\0"); + h.update(object_id.as_bytes()); + h.update(b"\0"); + Ok(Some(h.finalize().into())) +} + +fn build_input_digests( + target: &DepsManifest, + registry: &Registry, +) -> Result, String> { + if !target.dir.join("build.toml").exists() { + return Ok(Vec::new()); + } + let build = BuildToml::load(&target.dir)?; + let mut out = Vec::with_capacity(build.inputs.len() + build.git_inputs.len()); + for input in &build.inputs { + let path = resolve_build_input_path(target, registry, input)?; + out.push(BuildInputDigest { + label: input.clone(), + digest: hash_build_input(&path)?, + }); + } + // External Git inputs are content-addressed before any network access. + // Preserve authored order and length-prefix every field so distinct + // tuples cannot collide through concatenation. Adding this section is + // intentionally additive: packages without git_inputs retain their + // existing cache keys. + for (index, input) in build.git_inputs.iter().enumerate() { + let mut h = Sha256::new(); + h.update(b"wasm-posix-build-git-input-v1\0"); + for field in [ + input.name.as_bytes(), + input.repository.as_bytes(), + input.commit.as_bytes(), + ] { + h.update((field.len() as u64).to_le_bytes()); + h.update(field); + } + out.push(BuildInputDigest { + label: format!("git-input:{index}:{}", input.name), + digest: h.finalize().into(), + }); + } + Ok(out) +} + +fn resolve_build_input_path( + target: &DepsManifest, + registry: &Registry, + input: &str, +) -> Result { + resolve_build_input_path_from_repo(target, registry, input, &repo_root()) +} + +fn resolve_build_input_path_from_repo( + target: &DepsManifest, + registry: &Registry, + input: &str, + main_repo_root: &Path, +) -> Result { + // Canonical Kandelo build inputs are authored relative to the repository + // (`packages/registry//...`). Registry priority is package-level, + // not file-level: once a first-hit package.toml selects an external + // package, every declared input below that package must exist there. Never + // fill a missing file from a lower package generation. + if let Ok(registry_relative) = Path::new(input).strip_prefix("packages/registry") { + let (package_name, package_relative) = + split_registry_build_input(registry_relative, input)?; + if let Some(selected) = selected_registry_package_dir( + registry, + main_repo_root, + package_name, + ) { + return require_selected_registry_build_input( + target, + input, + package_name, + &selected, + package_relative, + ); + } + + // Some first-party helper trees under packages/registry (currently + // npm and node-compat) deliberately are not packages. They remain + // main-checkout inputs; an unclaimed directory in a higher registry + // root must not shadow them. + let main_candidate = main_repo_root.join(input); + if main_candidate.exists() { + return Ok(main_candidate); + } + return Err(format!( + "{} build input {:?} does not name a selected registry package and was not found at {}", + target.spec(), + input, + main_candidate.display(), + )); + } + + let main_candidate = main_repo_root.join(input); + if main_candidate.exists() { + return Ok(main_candidate); + } + + // Preserve the historical registry-relative input form + // (`/build.sh`) for third-party registries that use it, with the + // same package-level first-hit rule as canonical inputs. + let input_path = Path::new(input); + let mut legacy_components = input_path.components(); + if let Some(first_component) = legacy_components.next() { + if let std::path::Component::Normal(package_component) = first_component { + let package_name = package_component.to_str().ok_or_else(|| { + format!( + "{} build input {:?} has a non-UTF-8 registry package name", + target.spec(), + input, + ) + })?; + let package_relative = legacy_components.as_path(); + if let Some(selected) = + selected_registry_package_dir(registry, main_repo_root, package_name) + { + return require_selected_registry_build_input( + target, + input, + package_name, + &selected, + package_relative, + ); + } + } + } + + let main_registry_candidate = main_repo_root.join("packages/registry").join(input); + if main_registry_candidate.exists() { + return Ok(main_registry_candidate); + } + let package_relative_candidate = target.dir.join(input); + if package_relative_candidate.exists() { + return Ok(package_relative_candidate); + } + + Err(format!( + "{} build input {:?} not found (tried: {}, {}, {})", + target.spec(), + input, + main_candidate.display(), + main_registry_candidate.display(), + package_relative_candidate.display(), + )) +} + +fn split_registry_build_input<'a>( + registry_relative: &'a Path, + authored_input: &str, +) -> Result<(&'a str, &'a Path), String> { + let mut components = registry_relative.components(); + let package_component = match components.next() { + Some(std::path::Component::Normal(component)) => component, + _ => { + return Err(format!( + "canonical registry build input must name a package below packages/registry: {authored_input:?}", + )); + } + }; + let package_name = package_component.to_str().ok_or_else(|| { + format!( + "canonical registry build input has a non-UTF-8 package name: {authored_input:?}", + ) + })?; + Ok((package_name, components.as_path())) +} + +fn selected_registry_package_dir( + registry: &Registry, + main_repo_root: &Path, + package_name: &str, +) -> Option { + if let Some(manifest) = registry.find(package_name) { + return manifest.parent().map(Path::to_path_buf); + } + let main_package = main_repo_root.join("packages/registry").join(package_name); + main_package + .join("package.toml") + .is_file() + .then_some(main_package) +} + +fn require_selected_registry_build_input( + target: &DepsManifest, + authored_input: &str, + package_name: &str, + selected_package_dir: &Path, + package_relative: &Path, +) -> Result { + let candidate = selected_package_dir.join(package_relative); + if candidate.exists() { + return Ok(candidate); + } + Err(format!( + "{} build input {:?} is missing from first-hit registry package {:?} at {}; lower-priority package roots were not consulted", + target.spec(), + authored_input, + package_name, + selected_package_dir.display(), + )) +} + +fn hash_build_input(path: &Path) -> Result<[u8; 32], String> { + let mut h = Sha256::new(); + hash_build_input_entry(&mut h, path, path)?; + Ok(h.finalize().into()) +} + +fn hash_build_input_entry(h: &mut Sha256, root: &Path, path: &Path) -> Result<(), String> { + let meta = + std::fs::symlink_metadata(path).map_err(|e| format!("stat {}: {e}", path.display()))?; + let rel = path.strip_prefix(root).unwrap_or(path); + let rel = rel.to_string_lossy(); + + if meta.file_type().is_symlink() { + let target = + std::fs::read_link(path).map_err(|e| format!("readlink {}: {e}", path.display()))?; + h.update(b"symlink\0"); + h.update(rel.as_bytes()); + h.update(b"\0"); + h.update(target.to_string_lossy().as_bytes()); + h.update(b"\0"); + return Ok(()); + } + + if meta.is_file() { + let bytes = std::fs::read(path).map_err(|e| format!("read {}: {e}", path.display()))?; + h.update(b"file\0"); + h.update(rel.as_bytes()); + h.update(b"\0"); + h.update((bytes.len() as u64).to_le_bytes()); + h.update(b"\0"); + h.update(bytes); + h.update(b"\0"); + return Ok(()); + } + + if meta.is_dir() { + h.update(b"dir\0"); + h.update(rel.as_bytes()); + h.update(b"\0"); + let mut entries = std::fs::read_dir(path) + .map_err(|e| format!("read_dir {}: {e}", path.display()))? + .collect::, _>>() + .map_err(|e| format!("read_dir {}: {e}", path.display()))?; + entries.sort_by_key(|entry| entry.path()); + for entry in entries { + hash_build_input_entry(h, root, &entry.path())?; + } + return Ok(()); + } + + Err(format!( + "build input {} is not a file, directory, or symlink", + path.display() + )) +} + +/// Canonical cache directory for a resolved manifest. +/// +/// Layout: +/// libs/programs: `/libs/--rev--/` +/// sources: `/sources/--rev-/` +/// +/// The directory suffix is the complete 64-character cache-key SHA-256. Archive +/// filenames may use an eight-character transport label, but canonical local +/// cache identity must not collapse distinct keys that share that prefix. Cache +/// entries created by older resolvers with a short suffix become unused and are +/// rebuilt under the full-key path; they are not migrated or trusted in place. +/// +/// For libs and programs, `arch` is part of the path so a single user +/// can host wasm32 and wasm64 builds of the same artifact side-by-side. +/// The cache-key sha already incorporates `arch` as of Task A.5, so the +/// full cache identity disambiguates — but a visible arch segment makes the +/// cache layout self-explanatory at a glance. +/// +/// For source-kind manifests, the layout omits the arch segment per +/// design decision 6: source artifacts are arch-agnostic, so a single +/// cache entry serves both wasm32 and wasm64 consumers. +pub fn canonical_path( + cache_root: &Path, + m: &DepsManifest, + arch: TargetArch, + sha: &[u8; 32], +) -> PathBuf { + let kind_subdir = match m.kind { + ManifestKind::Library => "libs", + ManifestKind::Program => "programs", + ManifestKind::Source => "sources", + }; + let basename = match m.kind { + ManifestKind::Source => format!("{}-{}-rev{}-{}", m.name, m.version, m.revision, hex(sha)), + ManifestKind::Library | ManifestKind::Program => format!( + "{}-{}-rev{}-{}-{}", + m.name, + m.version, + m.revision, + arch.as_str(), + hex(sha) + ), + }; + cache_root.join(kind_subdir).join(basename) +} + +use crate::util::hex; + +// --------------------------------------------------------------------- +// Build + cache-install +// --------------------------------------------------------------------- + +/// Options controlling where the resolver reads from and writes to. +/// Kept as a struct so tests can pass tempdirs without reaching into +/// `$HOME` / `$XDG_CACHE_HOME`. +pub struct ResolveOpts<'a> { + pub cache_root: &'a Path, + /// Optional `local-libs/` directory. When a `/build/` + /// subdirectory exists under this root, it wins over the cache + /// and the build script is not run. + pub local_libs: Option<&'a Path>, + /// Manifest names that must be source-built unconditionally, even + /// on a cache hit and even when a `[binary]` archive_url would + /// otherwise satisfy the request. Used by the manual `force-rebuild` + /// workflow to refresh archives whose cache key is suspected stale. + /// `None` means "no force rebuild" (the default for every consumer + /// other than the manual workflow). `local_libs` still wins over + /// force_source_build (a hand-patched override is always honored). + /// A force rebuild assumes no concurrent resolver invocation for + /// the same package -- see `build_into_cache`'s atomic-install comment. + pub force_source_build: Option<&'a BTreeSet>, + /// Refuse any source build or source fetch fallback. Used by CI + /// binary-materialization gates, where package bytes must come from + /// staging overlays, the durable index, or an existing valid cache entry. + pub fetch_only: bool, + /// Repo root used to resolve `[build].script_path` (which is + /// repo-relative as of Phase A-bis Task 2). `None` means "use + /// `crate::repo_root()`", which is the production default. + /// Tests use this to point the resolver at a tempdir. + pub repo_root: Option<&'a Path>, + /// When `Some`, the resolver places `binaries/programs//...` + /// symlinks for every program manifest in the dep graph (target + + /// transitive program deps). Required so a consumer's build + /// script can find sibling-package binaries via `tryResolveBinary` + /// after a `xtask build-deps resolve ` invocation. `None` + /// disables symlink placement (test fixtures, library-only + /// resolves, etc.). + pub binaries_dir: Option<&'a Path>, +} + +/// Resolve a library to a concrete on-disk path with the artifacts +/// declared in its `package.toml`. Ensures dependencies are resolved +/// first (depth-first), then runs the build script if neither a +/// `local-libs/` override nor a cache hit is available. +/// +/// Returns the path the consumer should point `CPPFLAGS=-I

/include +/// LDFLAGS=-L

/lib` at. +pub fn ensure_built( + target: &DepsManifest, + registry: &Registry, + arch: TargetArch, + abi_version: u32, + opts: &ResolveOpts<'_>, +) -> Result { + let mut memo: BTreeMap = BTreeMap::new(); + let mut building: Vec = Vec::new(); + let (path, _transitive) = ensure_built_inner( + target, + registry, + arch, + abi_version, + opts, + &mut memo, + &mut building, + )?; + Ok(path) +} + +/// One direct dependency's resolved cache path plus its manifest kind. +/// +/// Carried alongside `dep_dirs` so the build-script env-var emission +/// can switch the suffix per design 12: library/program deps export +/// under `WASM_POSIX_DEP__DIR` (a built-artifact root), source +/// deps under `WASM_POSIX_DEP__SRC_DIR` (an unbuilt source tree). +struct DirectDep { + path: PathBuf, + kind: ManifestKind, +} + +/// Render a multi-tool probe-failure message for `ensure_built_inner`. +/// +/// Aggregates every `ProbeFailure` for `target` into one `Err(String)` +/// payload so a user fixes their toolchain in a single round-trip +/// rather than `cargo run`-ing once per missing tool. For each failure +/// we look up the matching `[[host_tools]]` declaration and append the +/// platform-keyed install hint chosen by `cfg!(target_os)`. If the +/// declaration ships hints but none for the current OS, we list which +/// platforms ARE covered so the user knows whether to translate one +/// or to file an issue. +/// Map Rust's `std::env::consts::OS` to the conventional platform key +/// used in `[[host_tools]].install_hints`. The deps-management package-system +/// schema uses unix-y names (`darwin` for macOS, matching bash and +/// `uname`); Rust's runtime constant is `"macos"`. Other names match +/// what users would expect (`linux`, `windows`, `freebsd`, etc.). +fn install_hints_key_for_current_os() -> &'static str { + match std::env::consts::OS { + "macos" => "darwin", + other => other, + } +} + +fn render_probe_failures(target: &DepsManifest, failures: &[ProbeFailure]) -> String { + let mut out = String::new(); + out.push_str(&format!( + "{}: {} host-tool requirement{} unsatisfied:\n", + target.spec(), + failures.len(), + if failures.len() == 1 { "" } else { "s" } + )); + for f in failures { + out.push_str(&format!(" - {f}\n")); + let tool_name = match f { + ProbeFailure::Missing { tool, .. } + | ProbeFailure::BadOutput { tool, .. } + | ProbeFailure::BadVersion { tool, .. } + | ProbeFailure::TooOld { tool, .. } => tool, + }; + if let Some(decl) = target + .host_tools + .iter() + .find(|d: &&HostTool| &d.name == tool_name) + { + let os = install_hints_key_for_current_os(); + if let Some(hint) = decl.install_hints.get(os) { + out.push_str(&format!(" install hint ({os}): {hint}\n")); + } else if !decl.install_hints.is_empty() { + let keys: Vec<&str> = decl.install_hints.keys().map(String::as_str).collect(); + out.push_str(&format!( + " no {os} install hint; available platforms: [{}]\n", + keys.join(", ") + )); + } + } + } + out +} + +/// Process-lifetime memo of `(name, arch, exact cache identity) → +/// ensure_built_uncached`'s result. Within a single `xtask` invocation (e.g. one +/// `archive-stage` run, or a `build-deps resolve` walk that pulls a +/// shared dep transitively), a manifest reached via multiple dependents +/// (mariadb is reached 6× during a force-rebuild-all: directly + via +/// lamp + via mariadb-test + via mariadb-vfs ×2) otherwise re-runs its +/// full source build N times — ~80 minutes of pointless work for +/// mariadb alone. The memo collapses that to one build per +/// `(name, arch)`. +/// +/// Caches BOTH `Ok` (so subsequent dependents reuse the resolved +/// path) and `Err` (so a failed manifest doesn't waste 10 more +/// minutes per dependent re-discovering the same failure). Cycle +/// errors are intentionally NOT cached — those depend on the call +/// stack at the moment of detection, and caching them could leak a +/// stale cycle result into a later acyclic traversal. +/// +/// Lifetime: process-only. A fresh xtask invocation starts with an +/// empty memo, which keeps CI semantics intact (every run from +/// scratch retries any failures). +/// +/// Key dimensions: +/// * `cache_root` — same process can host independent test cases +/// (cargo runs tests in parallel within one process; each test +/// uses a fresh tempdir). In production there's only ever one +/// cache_root per run, so this dimension is invisible to the +/// force-rebuild path. +/// * `name` — the manifest's identifier within its registry. +/// * `arch` — wasm32 vs wasm64. The same name builds independently +/// per-arch. +/// * `cache_identity` — the full recipe/dependency/toolchain digest computed +/// from the current registry. This prevents a result from surviving a +/// build.toml edit or being reused for a same-named package from another +/// registry inside one long-lived process. +/// * `was_force_rebuild` — `force_source_build` bypasses the +/// on-disk cache. Memoizing across the force-rebuild boundary +/// would mean a no-force result satisfies a later force request, +/// defeating the bypass intent. Keep them as separate slots so +/// a force-call after a no-force-call still rebuilds. In +/// a force-rebuild-all loop every call has the same flag, so the +/// memo collapses N calls per (name, arch) into 1 build — the +/// actual optimization we wanted. +/// * `fetch_only` — fetch-only failures must not poison later normal +/// resolves, which are allowed to build from source. +type BuildMemoKey = (PathBuf, String, TargetArch, [u8; 32], bool, bool); +type BuildMemoValue = Result<(PathBuf, BTreeSet), String>; + +fn build_memo() -> &'static Mutex> { + static MEMO: OnceLock>> = OnceLock::new(); + MEMO.get_or_init(|| Mutex::new(BTreeMap::new())) +} + +/// Cycle-error sentinel — these errors must NOT be memoized because +/// they describe the call stack at detection time, not a property of +/// the manifest. A later acyclic call for the same node should be +/// allowed to proceed. +fn is_cycle_error(e: &str) -> bool { + e.starts_with("cycle while building:") +} + +/// Fast path for archive-only resolver callers. +/// +/// Browser/dev-server preparation needs to materialize self-contained program +/// archives into `binaries/`. If one of those programs has a stale or corrupt +/// dependency archive, resolving dependencies first can incorrectly force a +/// source build even though the target archive itself is valid. Keep normal +/// source-build resolution unchanged, but allow program archive fetches in +/// binary-materialization mode to satisfy the request before walking deps. +/// +/// Fetch-only CI materialization is stricter: it accepts only a valid cache +/// entry or prebuilt archive for the target package and never falls through to +/// dependency resolution/source builds. +fn try_fetch_without_deps( + target: &DepsManifest, + registry: &Registry, + arch: TargetArch, + abi_version: u32, + opts: &ResolveOpts<'_>, + memo: &mut BTreeMap, +) -> Result, String> { + let binary_materialization_fast_path = opts.binaries_dir.is_some() + && matches!(target.kind, ManifestKind::Program) + && !target.program_outputs.is_empty(); + if (!opts.fetch_only && !binary_materialization_fast_path) + || !matches!(target.kind, ManifestKind::Library | ManifestKind::Program) + { + return Ok(None); + } + + let force_rebuild = opts + .force_source_build + .map(|s| s.contains(&target.name)) + .unwrap_or(false); + if force_rebuild { + if opts.fetch_only { + return Err(format!( + "{}: fetch-only resolve cannot honor force source-build for arch {}", + target.spec(), + arch.as_str(), + )); + } + return Ok(None); + } + + if !opts.fetch_only { + if let Some(lr) = opts.local_libs { + let override_dir = lr.join(&target.name).join("build"); + if override_dir.is_dir() { + return Ok(Some(override_dir)); + } + } + } + + let mut chain: Vec = Vec::new(); + let sha = compute_sha(target, registry, arch, abi_version, memo, &mut chain)?; + let canonical = canonical_path(opts.cache_root, target, arch, &sha); + let cache_key_sha_hex = hex(&sha); + + if canonical.is_dir() { + match validate_cache_entry(target, &canonical, arch, abi_version, &cache_key_sha_hex) { + Ok(()) => return Ok(Some(canonical)), + Err(e) => { + eprintln!( + "warning: ignoring stale cached artifact for {} at {} ({})", + target.spec(), + canonical.display(), + e, + ); + remove_cache_entry(&canonical, &cache_key_sha_hex).map_err(|remove_err| { + format!( + "clear stale cache entry {} after validation failure: {remove_err}", + canonical.display() + ) + })?; + } + } + } + + if let Some(binary) = target.binary.get(&arch) { + match remote_fetch::fetch_and_install( + binary, + &canonical, + target, + arch, + abi_version, + &cache_key_sha_hex, + ) { + Ok(()) => match validate_cache_entry( + target, + &canonical, + arch, + abi_version, + &cache_key_sha_hex, + ) { + Ok(()) => return Ok(Some(canonical)), + Err(e) => { + eprintln!( + "warning: direct binary fetch for {} from {} produced \ + a stale artifact ({}); {}", + target.spec(), + binary.archive_url, + e, + fetch_fallback_phrase(opts.fetch_only), + ); + let _ = remove_cache_entry(&canonical, &cache_key_sha_hex); + } + }, + Err(e) => { + eprintln!( + "warning: direct binary fetch for {} from {} failed ({}); \ + {}", + target.spec(), + binary.archive_url, + e, + fetch_fallback_phrase(opts.fetch_only), + ); + } + } + } + + if let Some(()) = try_index_install( + target, + arch, + abi_version, + &canonical, + &cache_key_sha_hex, + opts.fetch_only, + ) { + return Ok(Some(canonical)); + } + + if opts.fetch_only { + return Err(format!( + "{}: fetch-only resolve could not install a valid archive for arch {}; \ + run package staging/prepare to publish this package instead of \ + source-building during binary materialization", + target.spec(), + arch.as_str(), + )); + } + + Ok(None) +} + +/// Resolve `target`, returning its on-disk path *and* the set of +/// transitively-resolved lib paths underneath it (its direct deps, their +/// deps, and so on — but NOT `target`'s own path; the caller adds that). +/// +/// The transitive set lets the caller compose +/// `WASM_POSIX_DEP_PKG_CONFIG_PATH` for the build script: every node +/// gets every descendant's `lib/pkgconfig/` dir, which mirrors how +/// pkg-config follows `Requires.private` chains. +/// +/// Deduped via `BTreeSet` so a diamond dep (`libZ -> {libA, libB} -> +/// libCommon`) only contributes `libCommon`'s path once. +fn ensure_built_inner( + target: &DepsManifest, + registry: &Registry, + arch: TargetArch, + abi_version: u32, + opts: &ResolveOpts<'_>, + memo: &mut BTreeMap, + building: &mut Vec, +) -> Result<(PathBuf, BTreeSet), String> { + // Process-lifetime memo: the same (name, arch) often gets + // requested multiple times within one resolver run via different + // dep chains. Without this, mariadb wasm32 source-builds 4 times + // in a single force-rebuild-all (lamp, mariadb, mariadb-test, + // mariadb-vfs each independently demand it). See `build_memo`'s + // doc comment for full rationale. + // Compute the exact current identity before consulting the process memo. + // `ensure_built()` supplies a fresh hash memo for every top-level call, so + // an in-process build.toml edit cannot inherit the prior call's digest; + // recursive lookups in one unchanged graph remain cheap memo hits. + let mut identity_chain = Vec::new(); + let cache_identity = compute_sha( + target, + registry, + arch, + abi_version, + memo, + &mut identity_chain, + )?; + let was_force_rebuild = opts + .force_source_build + .map(|s| s.contains(&target.name)) + .unwrap_or(false); + let memo_key: BuildMemoKey = ( + opts.cache_root.to_path_buf(), + target.name.clone(), + arch, + cache_identity, + was_force_rebuild, + opts.fetch_only, + ); + { + let cache = build_memo().lock().unwrap(); + if let Some(cached) = cache.get(&memo_key) { + return cached.clone(); + } + } + + let result = ensure_built_uncached(target, registry, arch, abi_version, opts, memo, building); + + // Don't poison the cache with cycle errors — those reflect the + // call stack at the moment of detection, not a stable property + // of the manifest. Everything else (Ok path + non-cycle Err) + // gets memoized. + let should_memo = match &result { + Ok(_) => true, + Err(e) => !is_cycle_error(e), + }; + if should_memo { + build_memo() + .lock() + .unwrap() + .insert(memo_key, result.clone()); + } + result +} + +fn ensure_built_uncached( + target: &DepsManifest, + registry: &Registry, + arch: TargetArch, + abi_version: u32, + opts: &ResolveOpts<'_>, + memo: &mut BTreeMap, + building: &mut Vec, +) -> Result<(PathBuf, BTreeSet), String> { + if building.iter().any(|s| s == &target.name) { + return Err(format!( + "cycle while building: {} -> {}", + building.join(" -> "), + target.name + )); + } + building.push(target.name.clone()); + + if let Some(path) = try_fetch_without_deps(target, registry, arch, abi_version, opts, memo)? { + building.pop(); + return Ok((path, BTreeSet::new())); + } + + // Recursively resolve direct deps first; remember their paths so + // we can surface them to the build script via env vars. The + // transitive set accumulates every dep path in the subgraph, + // deduped — diamond deps must only contribute once. + // + // We track each direct dep's `kind` alongside its path so that + // `build_into_cache` can choose the env-var suffix per design 12: + // library/program → `WASM_POSIX_DEP__DIR` (built artifact + // root); source → `WASM_POSIX_DEP__SRC_DIR` (unbuilt source + // tree). Build scripts then self-document what shape they're + // consuming via the suffix. + let mut dep_dirs: BTreeMap = BTreeMap::new(); + let mut transitive: BTreeSet = BTreeSet::new(); + for dref in &target.depends_on { + let dep_m = registry.load(&dref.name)?; + if dep_m.version != dref.version { + return Err(format!( + "{} depends on {}@{}, but registry has {}", + target.spec(), + dref.name, + dref.version, + dep_m.spec() + )); + } + // Per the wasm64 build policy (memory/wasm64-build-policy.md): + // only MariaDB and PHP need wasm64 binaries; everything else + // is wasm32-only. So a wasm64 program (e.g. mariadb-vfs) + // depending on a wasm32-only dep (e.g. dinit) is the common + // case, not a misconfiguration. When the parent arch isn't in + // the dep's target_arches, fall back to wasm32 (the universal + // arch) for that dep. The resolver places the dep's binaries + // under binaries/programs/wasm32/, where build scripts' + // arch-agnostic tryResolveBinary("programs/.wasm") finds + // them. The kernel runs mixed-arch programs. + let dep_arch = if dep_m.target_arches.contains(&arch) { + arch + } else if dep_m.target_arches.contains(&TargetArch::Wasm32) { + TargetArch::Wasm32 + } else { + return Err(format!( + "{} depends on {}@{} (arch {}), but {} declares neither {} nor wasm32 in target_arches (declared: {:?})", + target.spec(), + dref.name, + dref.version, + arch.as_str(), + dep_m.spec(), + arch.as_str(), + dep_m + .target_arches + .iter() + .map(|a| a.as_str()) + .collect::>(), + )); + }; + let (dep_path, dep_transitive) = ensure_built_inner( + &dep_m, + registry, + dep_arch, + abi_version, + opts, + memo, + building, + )?; + // Place binaries/programs// symlinks for each + // program dep so consumer build scripts can find them via + // `tryResolveBinary("programs/.wasm")`. Only kicks in when + // the caller opts in with binaries_dir; other ensure_built + // consumers leave binaries_dir = None and no symlinks land. + // Library deps and source deps are linked at compile time via + // WASM_POSIX_DEP_* env vars and don't need a binaries/ entry. + if let Some(bdir) = opts.binaries_dir { + if matches!(dep_m.kind, ManifestKind::Program) && !dep_m.program_outputs.is_empty() { + place_binaries_symlinks(&dep_m, &dep_path, bdir, dep_arch)?; + } + } + dep_dirs.insert( + dep_m.name.clone(), + DirectDep { + path: dep_path.clone(), + kind: dep_m.kind, + }, + ); + transitive.insert(dep_path); + transitive.extend(dep_transitive); + } + + building.pop(); + + // Local-libs override: hand-patched source wins. The override dir + // still contributes to `transitive` for any consumer above us. + if let Some(lr) = opts.local_libs { + let override_dir = lr.join(&target.name).join("build"); + if override_dir.is_dir() { + return Ok((override_dir, transitive)); + } + } + + // Compute canonical cache path. + let mut chain: Vec = Vec::new(); + let sha = compute_sha(target, registry, arch, abi_version, memo, &mut chain)?; + let canonical = canonical_path(opts.cache_root, target, arch, &sha); + let cache_key_sha_hex = hex(&sha); + + let force_rebuild = opts + .force_source_build + .map(|s| s.contains(&target.name)) + .unwrap_or(false); + + // Cache hit: validate before using it. The cache key includes the + // numeric kernel ABI, but fork-continuation mechanism changes have + // previously produced stale artifacts with a matching ABI number + // (legacy Asyncify exports instead of wpk_fork_*). Reject those so + // the resolver can fetch a current remote artifact or source-build. + if !force_rebuild && canonical.is_dir() { + match validate_cache_entry(target, &canonical, arch, abi_version, &cache_key_sha_hex) { + Ok(()) => return Ok((canonical, transitive)), + Err(e) => { + eprintln!( + "warning: ignoring stale cached artifact for {} at {} ({})", + target.spec(), + canonical.display(), + e, + ); + remove_cache_entry(&canonical, &cache_key_sha_hex).map_err(|remove_err| { + format!( + "clear stale cache entry {} after validation failure: {remove_err}", + canonical.display() + ) + })?; + } + } + } + if force_rebuild && canonical.is_dir() { + remove_cache_entry(&canonical, &cache_key_sha_hex) + .map_err(|e| format!("force-rebuild: clear {}: {e}", canonical.display()))?; + } + + // Run host-tool probes before any work that might invoke a build + // script (or fetch+extract a source-kind tarball). Cache hits skip + // this — probes are only needed when we might actually invoke + // `bash build-.sh` or similar work. Aggregate ALL probe + // failures so users fix everything in one round-trip. + if !target.host_tools.is_empty() { + let mut failures: Vec = Vec::new(); + for tool in &target.host_tools { + if let Err(e) = host_tool_probe::probe(tool) { + failures.push(e); + } + } + if !failures.is_empty() { + return Err(render_probe_failures(target, &failures)); + } + } + + // Cache-miss dispatch. Three flavors of recipe: + // + // (Source, None) — default fetch+extract from `[source]`. + // Source-kind manifests never carry + // `[binary]` (Task C.1 enforces), so this + // branch short-circuits before the binary + // block. + // (Source, Some(_)) — override path (Task C.5): the manifest + // ships its own build script (e.g. patch + // overlay, git clone, multi-tarball + // assembly). Run it through + // `build_into_cache` with the standard + // env-var contract; validation is + // non-emptiness of OUT_DIR rather than a + // declared outputs list. + // (Library | Program,_) — try `package.pr.toml` / source + // `[binary]` direct archives first, then + // the `build.toml` index path, then fall + // back to the build script. + match (target.kind, target.build.script_path.is_some()) { + (ManifestKind::Source, false) => { + if opts.fetch_only { + return Err(format!( + "{}: fetch-only resolve cannot fetch source package fallback for arch {}", + target.spec(), + arch.as_str(), + )); + } + let parent = canonical + .parent() + .ok_or_else(|| format!("canonical path has no parent: {}", canonical.display()))?; + std::fs::create_dir_all(parent) + .map_err(|e| format!("create cache parent {}: {e}", parent.display()))?; + let tmp = parent.join(format!( + "{}.tmp-{}", + canonical + .file_name() + .expect("canonical path has a filename") + .to_string_lossy(), + std::process::id() + )); + if tmp.exists() { + std::fs::remove_dir_all(&tmp) + .map_err(|e| format!("clean stale {}: {e}", tmp.display()))?; + } + if let Err(e) = + source_extract::fetch_and_extract(&target.source.url, &target.source.sha256, &tmp) + { + let _ = std::fs::remove_dir_all(&tmp); + return Err(format!( + "{}: source fetch+extract failed: {e}", + target.spec() + )); + } + if let Err(e) = + write_cache_provenance(target, &canonical, arch, abi_version, &cache_key_sha_hex) + { + let _ = std::fs::remove_dir_all(&tmp); + return Err(e); + } + // Race against a peer process that finished its own extract + // first: keep theirs, drop ours. Identical inputs produce + // identical outputs. + if canonical.exists() { + let _ = std::fs::remove_dir_all(&tmp); + validate_cache_entry(target, &canonical, arch, abi_version, &cache_key_sha_hex)?; + return Ok((canonical, transitive)); + } + std::fs::rename(&tmp, &canonical) + .map_err(|e| format!("rename {} -> {}: {e}", tmp.display(), canonical.display()))?; + Ok((canonical, transitive)) + } + (ManifestKind::Source, true) => { + if opts.fetch_only { + return Err(format!( + "{}: fetch-only resolve cannot run source package build script for arch {}", + target.spec(), + arch.as_str(), + )); + } + // Override path: run the script. No remote-binary fetch for + // sources (`[binary]` is rejected at parse time for source + // kind), so we go straight to `build_into_cache`. + let pkgconfig_path = compose_pkgconfig_path(&transitive); + let repo_root = opts + .repo_root + .map(Path::to_path_buf) + .unwrap_or_else(crate::repo_root); + build_into_cache( + target, + arch, + abi_version, + &cache_key_sha_hex, + &canonical, + &dep_dirs, + &pkgconfig_path, + &repo_root, + )?; + Ok((canonical, transitive)) + } + (ManifestKind::Library | ManifestKind::Program, _) => { + // Resolution priority 3a: direct archive fetch from the + // source manifest's `[binary]` map. In normal source + // package.toml files this map is empty post index-ledger + // migration, but CI writes sibling `package.pr.toml` + // overlays with direct file:// archives for same-run + // matrix outputs. Those must win over the durable + // `build.toml` index below. + // + // Resolution priority 3b: index-based remote fetch. The + // resolver loads the sibling `build.toml`, resolves its + // `[binary]` block to an index URL (or a direct archive + // URL), then looks up this package's entry. Status + // `success` fetches the current archive; status + // `failed`/`pending`/`building` falls back to the + // last-green `fallback_*` archive when one is preserved. + // + // Any failure along the way logs and falls through to the + // source build — a remote-fetch error should never cause + // the resolver to refuse to produce an artifact. + // + // `force_rebuild` short-circuits remote fetch entirely. + if !force_rebuild { + if let Some(binary) = target.binary.get(&arch) { + match remote_fetch::fetch_and_install( + binary, + &canonical, + target, + arch, + abi_version, + &cache_key_sha_hex, + ) { + Ok(()) => match validate_cache_entry( + target, + &canonical, + arch, + abi_version, + &cache_key_sha_hex, + ) { + Ok(()) => return Ok((canonical, transitive)), + Err(e) => { + eprintln!( + "warning: direct binary fetch for {} from {} produced \ + a stale artifact ({}); {}", + target.spec(), + binary.archive_url, + e, + fetch_fallback_phrase(opts.fetch_only), + ); + let _ = remove_cache_entry(&canonical, &cache_key_sha_hex); + } + }, + Err(e) => { + eprintln!( + "warning: direct binary fetch for {} from {} failed ({}); \ + {}", + target.spec(), + binary.archive_url, + e, + fetch_fallback_phrase(opts.fetch_only), + ); + } + } + } + if let Some(()) = try_index_install( + target, + arch, + abi_version, + &canonical, + &cache_key_sha_hex, + opts.fetch_only, + ) { + return Ok((canonical, transitive)); + } + } + + if opts.fetch_only { + return Err(format!( + "{}: fetch-only resolve could not install a valid archive for arch {}; \ + package staging or the durable release must provide one", + target.spec(), + arch.as_str(), + )); + } + + let pkgconfig_path = compose_pkgconfig_path(&transitive); + let repo_root = opts + .repo_root + .map(Path::to_path_buf) + .unwrap_or_else(crate::repo_root); + build_into_cache( + target, + arch, + abi_version, + &cache_key_sha_hex, + &canonical, + &dep_dirs, + &pkgconfig_path, + &repo_root, + )?; + Ok((canonical, transitive)) + } + } +} + +/// Attempt to install a prebuilt archive from this package's +/// `build.toml`-declared binary source. Returns `Some(())` on success +/// (caller returns the canonical path); returns `None` for any +/// "fall through to source build" condition (no build.toml, no +/// archive in the index, network failure, sha mismatch, etc.). +/// +/// Logging is on stderr (matching the prior remote-fetch +/// implementation's UX): users see warnings about why the index +/// path was skipped. Normal resolves then build from source; fetch-only +/// resolves turn the miss into an error at the caller. +fn fetch_fallback_phrase(fetch_only: bool) -> &'static str { + if fetch_only { + "source builds disabled by fetch-only mode" + } else { + "falling back to source build" + } +} + +fn try_index_install( + target: &DepsManifest, + arch: TargetArch, + abi_version: u32, + canonical: &Path, + cache_key_sha_hex: &str, + fetch_only: bool, +) -> Option<()> { + // 1. Load build.toml. Source manifests without one (e.g. an + // upstream package that hasn't been ported to the new schema + // yet) fall through silently — Phase 9's migration should + // leave every first-party package with a build.toml; the + // silent fall-through is for clean integration with + // third-party manifests that might not. + let build = BuildToml::load(&target.dir).ok()?; + + // 2. Resolve the binary source to a concrete URL pair. Direct + // form: use the URL + sha verbatim. Indexed form: fetch + // index.toml + look up this package. CI can override indexed + // URLs with WASM_POSIX_BINARY_INDEX_URL so staging/prepare + // jobs consume the release they are publishing instead of the + // committed durable-release default. + let (archive_url, archive_sha256) = match &build.binary { + BinarySource::Direct { url, sha256 } => (url.clone(), sha256.clone()), + BinarySource::Indexed { .. } => { + let index_url = std::env::var("WASM_POSIX_BINARY_INDEX_URL") + .ok() + .filter(|s| !s.is_empty()) + .or_else(|| build.binary.resolve_index_url(abi_version))?; + let cache_dir = default_cache_root().join("indexes"); + let index = match index_toml::fetch_index(&index_url, &cache_dir) { + Ok(idx) => idx, + Err(e) => { + eprintln!( + "warning: index fetch for {} from {} failed ({}); \ + {}", + target.spec(), + index_url, + e, + fetch_fallback_phrase(fetch_only), + ); + return None; + } + }; + if index.abi_version != abi_version { + eprintln!( + "warning: index for {} from {} declares ABI {}, but resolver ABI is {}; \ + {}", + target.spec(), + index_url, + index.abi_version, + abi_version, + fetch_fallback_phrase(fetch_only), + ); + return None; + } + let entry = match index.lookup(&target.name, &target.version, arch) { + Some(e) => e, + None => { + eprintln!( + "warning: no index entry for {} in {}; \ + {}", + target.spec(), + index_url, + fetch_fallback_phrase(fetch_only), + ); + return None; + } + }; + // Pick the authoritative archive fields for the entry's + // current status. Success → current archive_*; other + // statuses → fallback_* if preserved; otherwise nothing + // usable and we fall through. + let (rel_url, sha) = match entry.status { + EntryStatus::Success + if entry.archive_url.is_some() && entry.archive_sha256.is_some() => + { + ( + entry.archive_url.as_ref().unwrap().clone(), + entry.archive_sha256.as_ref().unwrap().clone(), + ) + } + EntryStatus::Failed | EntryStatus::Pending | EntryStatus::Building + if entry.fallback_archive_url.is_some() + && entry.fallback_archive_sha256.is_some() => + { + eprintln!( + "note: {} index entry is status={:?}; \ + using last-green fallback archive", + target.spec(), + entry.status, + ); + ( + entry.fallback_archive_url.as_ref().unwrap().clone(), + entry.fallback_archive_sha256.as_ref().unwrap().clone(), + ) + } + _ => { + eprintln!( + "warning: {} index entry status={:?} has no usable archive; \ + {}", + target.spec(), + entry.status, + fetch_fallback_phrase(fetch_only), + ); + return None; + } + }; + (resolve_relative_url(&index_url, &rel_url), sha) + } + }; + + // 3. Fetch + verify + install. Any failure (sha mismatch, arch + // mismatch, abi mismatch, cache_key mismatch, transport + // error) falls through. + match remote_fetch::fetch_and_install_direct( + &archive_url, + &archive_sha256, + canonical, + target, + arch, + abi_version, + cache_key_sha_hex, + ) { + Ok(()) => { + match validate_cache_entry(target, canonical, arch, abi_version, cache_key_sha_hex) { + Ok(()) => Some(()), + Err(e) => { + eprintln!( + "warning: index-based fetch for {} from {} produced \ + a stale artifact ({}); {}", + target.spec(), + archive_url, + e, + fetch_fallback_phrase(fetch_only), + ); + let _ = remove_cache_entry(canonical, cache_key_sha_hex); + None + } + } + } + Err(e) => { + eprintln!( + "warning: index-based fetch for {} from {} failed ({}); \ + {}", + target.spec(), + archive_url, + e, + fetch_fallback_phrase(fetch_only), + ); + None + } + } +} + +/// Resolve `rel` against `base` for archive-URL lookup. If `rel` +/// already carries a scheme (`file://` / `http://` / `https://`) it +/// passes through unchanged; otherwise it's appended to `base`'s +/// parent directory (i.e. `https://host/dir/index.toml` + `foo.tar.zst` +/// → `https://host/dir/foo.tar.zst`). +pub(crate) fn resolve_relative_url(base: &str, rel: &str) -> String { + if rel.starts_with("file://") || rel.starts_with("http://") || rel.starts_with("https://") { + return rel.to_string(); + } + // Strip the last path segment of `base` and join with `rel`. + let last_slash = base.rfind('/').map(|i| i + 1).unwrap_or(0); + let mut out = String::with_capacity(last_slash + rel.len()); + out.push_str(&base[..last_slash]); + out.push_str(rel); + out +} + +/// Build the `WASM_POSIX_DEP_PKG_CONFIG_PATH` value for a build script. +/// +/// Joins every transitive lib path's `lib/pkgconfig/` subdirectory with +/// `:` — POSIX's standard search-path separator, and what pkg-config +/// itself uses for `PKG_CONFIG_PATH`. Paths whose `lib/pkgconfig/` +/// directory doesn't exist (e.g. ncurses, libs that ship no .pc file) +/// are skipped: handing pkg-config a list of nonexistent search paths +/// clutters diagnostics with no benefit. +/// +/// Returns an empty string when no transitive lib ships pkgconfig. The +/// caller still sets the env var to that empty string, keeping the +/// contract uniform: the var is *always* defined for build scripts. +fn compose_pkgconfig_path(paths: &BTreeSet) -> String { + paths + .iter() + .filter_map(|p| { + let pc = p.join("lib").join("pkgconfig"); + if pc.is_dir() { + Some(pc.to_string_lossy().into_owned()) + } else { + None + } + }) + .collect::>() + .join(":") +} + +#[derive(Debug)] +struct ProvisionedGitInput { + declaration: GitBuildInput, + checkout: PathBuf, + worktree_digest: [u8; 32], +} + +#[derive(Debug)] +struct GitCommandIsolation { + home: PathBuf, + xdg_config_home: PathBuf, + empty_templates: PathBuf, + empty_hooks: PathBuf, + askpass: PathBuf, +} + +/// Temporary, detached checkouts for a package's declared `git_inputs`. +/// Dropping the guard removes every checkout, including error paths. +#[derive(Debug, Default)] +struct ProvisionedGitInputs { + root: Option, + isolation: Option, + inputs: Vec, +} + +impl ProvisionedGitInputs { + fn provision(target: &DepsManifest, canonical: &Path) -> Result { + let build_path = target.dir.join("build.toml"); + if !build_path.exists() { + return Ok(Self::default()); + } + let declarations = BuildToml::load(&target.dir)?.git_inputs; + if declarations.is_empty() { + return Ok(Self::default()); + } + + Self::provision_declarations(&target.spec(), canonical, declarations) + } + + fn provision_declarations( + package_spec: &str, + canonical: &Path, + declarations: Vec, + ) -> Result { + Self::provision_declarations_inner(package_spec, canonical, declarations, &[]) + } + + #[cfg(test)] + fn provision_declarations_with_ambient_env( + package_spec: &str, + canonical: &Path, + declarations: Vec, + ambient_env: &[(std::ffi::OsString, std::ffi::OsString)], + ) -> Result { + Self::provision_declarations_inner(package_spec, canonical, declarations, ambient_env) + } + + fn provision_declarations_inner( + package_spec: &str, + canonical: &Path, + declarations: Vec, + ambient_env: &[(std::ffi::OsString, std::ffi::OsString)], + ) -> Result { + let parent = canonical.parent().ok_or_else(|| { + format!( + "{}: canonical cache path has no parent for git inputs: {}", + package_spec, + canonical.display() + ) + })?; + let basename = canonical + .file_name() + .ok_or_else(|| { + format!( + "canonical cache path has no filename: {}", + canonical.display() + ) + })? + .to_string_lossy(); + let root = create_git_input_root(parent, &basename)?; + + let mut provisioned = Self { + root: Some(root.clone()), + isolation: None, + inputs: Vec::with_capacity(declarations.len()), + }; + let isolation = create_git_command_isolation(&root)?; + let checkouts = root.join("checkouts"); + std::fs::create_dir(&checkouts).map_err(|e| { + format!( + "create immutable git-input checkout root {}: {e}", + checkouts.display() + ) + })?; + provisioned.isolation = Some(isolation); + for declaration in declarations { + let isolation = provisioned + .isolation + .as_ref() + .expect("git command isolation was initialized"); + let checkout = checkouts.join(&declaration.name); + std::fs::create_dir(&checkout) + .map_err(|e| format!("create git-input checkout {}: {e}", checkout.display()))?; + run_git( + &checkout, + &["init", "--quiet", "--object-format=sha1"], + &declaration.repository, + isolation, + ambient_env, + )?; + run_git( + &checkout, + &[ + "fetch", + "--quiet", + "--depth=1", + "--no-tags", + "--no-recurse-submodules", + &declaration.repository, + &declaration.commit, + ], + &declaration.repository, + isolation, + ambient_env, + )?; + run_git( + &checkout, + &["checkout", "--quiet", "--detach", "FETCH_HEAD"], + &declaration.repository, + isolation, + ambient_env, + )?; + verify_git_input(&declaration, &checkout, isolation, ambient_env)?; + validate_git_input_tree(&declaration, &checkout, isolation, ambient_env)?; + let worktree_digest = digest_git_input_worktree(&checkout)?; + set_git_input_tree_read_only(&checkout, true)?; + provisioned.inputs.push(ProvisionedGitInput { + declaration, + checkout, + worktree_digest, + }); + } + // Seal the containing directory as well as each checkout. Otherwise a + // build that cannot edit files could still rename or replace the path + // exported in WASM_POSIX_BUILD_GIT_*_DIR. + set_git_input_tree_read_only(&root, true)?; + Ok(provisioned) + } + + fn export_to(&self, command: &mut Command) { + for input in &self.inputs { + let key = input.declaration.name.to_ascii_uppercase(); + command.env(format!("WASM_POSIX_BUILD_GIT_{key}_DIR"), &input.checkout); + command.env( + format!("WASM_POSIX_BUILD_GIT_{key}_COMMIT"), + &input.declaration.commit, + ); + } + } + + fn verify_unchanged(&self) -> Result<(), String> { + if self.inputs.is_empty() { + return Ok(()); + } + let isolation = self + .isolation + .as_ref() + .ok_or_else(|| "immutable Git inputs lack command isolation".to_string())?; + for input in &self.inputs { + verify_git_input(&input.declaration, &input.checkout, isolation, &[])?; + validate_git_input_tree(&input.declaration, &input.checkout, isolation, &[])?; + let actual_digest = digest_git_input_worktree(&input.checkout)?; + if actual_digest != input.worktree_digest { + return Err(format!( + "git input {:?}: immutable working-tree digest changed during build", + input.declaration.name + )); + } + } + Ok(()) + } +} + +impl Drop for ProvisionedGitInputs { + fn drop(&mut self) { + if let Some(root) = self.root.take() { + let _ = set_git_input_tree_read_only(&root, false); + let _ = std::fs::remove_dir_all(root); + } + } +} + +static GIT_INPUT_TEMP_COUNTER: AtomicU64 = AtomicU64::new(0); + +fn create_git_input_root(parent: &Path, basename: &str) -> Result { + for _ in 0..10_000 { + let counter = GIT_INPUT_TEMP_COUNTER.fetch_add(1, Ordering::Relaxed); + let root = parent.join(format!( + ".{basename}.git-inputs-{}-{counter}", + std::process::id() + )); + match std::fs::create_dir(&root) { + Ok(()) => { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700)) + .map_err(|e| { + format!( + "set exclusive git-input root permissions {}: {e}", + root.display() + ) + })?; + } + return Ok(root); + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(e) => { + return Err(format!( + "create exclusive git-input root {}: {e}", + root.display() + )); + } + } + } + Err(format!( + "could not allocate an exclusive git-input root below {}", + parent.display() + )) +} + +fn create_git_command_isolation(root: &Path) -> Result { + let home = root.join("home"); + let xdg_config_home = root.join("xdg-config"); + let empty_templates = root.join("empty-templates"); + let empty_hooks = root.join("empty-hooks"); + for path in [&home, &xdg_config_home, &empty_templates, &empty_hooks] { + std::fs::create_dir(path) + .map_err(|e| format!("create isolated Git directory {}: {e}", path.display()))?; + } + + let askpass = root.join("askpass-deny.sh"); + std::fs::write(&askpass, "#!/bin/sh\nexit 1\n") + .map_err(|e| format!("write isolated Git askpass {}: {e}", askpass.display()))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&askpass, std::fs::Permissions::from_mode(0o500)) + .map_err(|e| format!("seal isolated Git askpass {}: {e}", askpass.display()))?; + } + + Ok(GitCommandIsolation { + home, + xdg_config_home, + empty_templates, + empty_hooks, + askpass, + }) +} + +/// Construct a Git subprocess that cannot inherit source credentials, token +/// headers, hooks, or repository selection from the caller's environment. +/// Build-time Git inputs are public source inputs; a private checkout would be +/// both unreproducible and an accidental credential dependency. +fn hardened_git_command( + repository: &str, + isolation: &GitCommandIsolation, + ambient_env: &[(std::ffi::OsString, std::ffi::OsString)], +) -> Command { + let mut command = Command::new("git"); + // Test callers can inject a hostile ambient environment without mutating + // process-global state. Production passes an empty slice; Command still + // begins with the real inherited environment in both cases. + command.envs(ambient_env.iter().cloned()); + command + .arg("-c") + .arg(format!( + "core.hooksPath={}", + isolation.empty_hooks.to_string_lossy() + )) + .arg("-c") + .arg(format!( + "init.templateDir={}", + isolation.empty_templates.to_string_lossy() + )) + .arg("-c") + .arg("credential.helper=") + .arg("-c") + .arg("credential.interactive=false") + .arg("-c") + .arg("http.extraHeader=") + .arg("-c") + .arg("http.cookieFile=") + .arg("-c") + .arg("http.saveCookies=false") + .arg("-c") + .arg("http.followRedirects=false") + .arg("-c") + .arg("submodule.recurse=false") + .arg("-c") + .arg("core.autocrlf=false") + .arg("-c") + .arg("core.eol=lf") + .env("HOME", &isolation.home) + .env("XDG_CONFIG_HOME", &isolation.xdg_config_home) + .env("GIT_ATTR_NOSYSTEM", "1") + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .env("GIT_CONFIG_SYSTEM", "/dev/null") + .env("GIT_CONFIG_COUNT", "0") + .env("GIT_TERMINAL_PROMPT", "0") + .env("GIT_ASKPASS", &isolation.askpass) + .env("GIT_ASKPASS_REQUIRE", "force") + .env("SSH_ASKPASS", &isolation.askpass) + .env("SSH_ASKPASS_REQUIRE", "force") + .env("GIT_OPTIONAL_LOCKS", "0") + .env("GIT_DEFAULT_HASH", "sha1") + .env("GIT_NO_REPLACE_OBJECTS", "1") + .env("GIT_PROTOCOL_FROM_USER", "0") + .env( + "GIT_ALLOW_PROTOCOL", + if repository.starts_with("file://") { + // Private test helpers construct local repositories directly; + // BuildToml validation makes this unreachable for real inputs. + "https:file" + } else { + "https" + }, + ); + + for key in [ + "GH_TOKEN", + "GITHUB_TOKEN", + "HOMEBREW_GITHUB_PACKAGES_TOKEN", + "HOMEBREW_GITHUB_API_TOKEN", + "HOMEBREW_DOCKER_REGISTRY_TOKEN", + "GIT_DIR", + "GIT_COMMON_DIR", + "GIT_WORK_TREE", + "GIT_INDEX_FILE", + "GIT_OBJECT_DIRECTORY", + "GIT_ALTERNATE_OBJECT_DIRECTORIES", + "GIT_CONFIG_PARAMETERS", + "GIT_CONFIG", + "GIT_SSH", + "GIT_SSH_COMMAND", + "GIT_TEMPLATE_DIR", + "GIT_EXEC_PATH", + "GIT_PROXY_COMMAND", + "GIT_NAMESPACE", + "GIT_REPLACE_REF_BASE", + "GIT_SHALLOW_FILE", + "GIT_GRAFT_FILE", + "GIT_CEILING_DIRECTORIES", + "GIT_DISCOVERY_ACROSS_FILESYSTEM", + "GIT_EXTERNAL_DIFF", + "GIT_DIFF_OPTS", + "GIT_EDITOR", + "GIT_PAGER", + "GIT_TRACE", + "GIT_TRACE2", + "GIT_TRACE_CURL", + "GIT_CURL_VERBOSE", + "NETRC", + "SSH_AUTH_SOCK", + ] { + command.env_remove(key); + } + for (key, _) in std::env::vars_os().chain(ambient_env.iter().cloned()) { + let key_text = key.to_string_lossy(); + if key_text.starts_with("GIT_CONFIG_KEY_") + || key_text.starts_with("GIT_CONFIG_VALUE_") + || key_text.starts_with("GIT_TRACE_") + { + command.env_remove(key); + } + } + command +} + +fn git_output( + checkout: &Path, + args: &[&str], + repository: &str, + isolation: &GitCommandIsolation, + ambient_env: &[(std::ffi::OsString, std::ffi::OsString)], +) -> Result { + let mut command = hardened_git_command(repository, isolation, ambient_env); + let output = command + .arg("-C") + .arg(checkout) + .args(args) + .output() + .map_err(|e| format!("spawn isolated git in {}: {e}", checkout.display()))?; + Ok(output) +} + +fn run_git( + checkout: &Path, + args: &[&str], + repository: &str, + isolation: &GitCommandIsolation, + ambient_env: &[(std::ffi::OsString, std::ffi::OsString)], +) -> Result<(), String> { + let output = git_output(checkout, args, repository, isolation, ambient_env)?; + if output.status.success() { + return Ok(()); + } + Err(format!( + "isolated git {:?} failed in {} with {}: {}", + args, + checkout.display(), + output.status, + String::from_utf8_lossy(&output.stderr).trim() + )) +} + +fn verify_git_input( + input: &GitBuildInput, + checkout: &Path, + isolation: &GitCommandIsolation, + ambient_env: &[(std::ffi::OsString, std::ffi::OsString)], +) -> Result<(), String> { + let head = git_output( + checkout, + &["rev-parse", "HEAD^{commit}"], + &input.repository, + isolation, + ambient_env, + )?; + if !head.status.success() { + return Err(format!( + "git input {:?}: cannot resolve detached HEAD in {}: {}", + input.name, + checkout.display(), + String::from_utf8_lossy(&head.stderr).trim() + )); + } + let actual = String::from_utf8_lossy(&head.stdout).trim().to_string(); + if actual != input.commit { + return Err(format!( + "git input {:?}: expected commit {}, checkout has {}", + input.name, input.commit, actual + )); + } + + let branch = git_output( + checkout, + &["rev-parse", "--abbrev-ref", "HEAD"], + &input.repository, + isolation, + ambient_env, + )?; + if !branch.status.success() || String::from_utf8_lossy(&branch.stdout).trim() != "HEAD" { + return Err(format!( + "git input {:?}: checkout must remain at a detached HEAD", + input.name + )); + } + + let status = git_output( + checkout, + &[ + "status", + "--porcelain=v1", + "--untracked-files=all", + "--ignored=matching", + ], + &input.repository, + isolation, + ambient_env, + )?; + if !status.status.success() { + return Err(format!( + "git input {:?}: cannot verify clean checkout: {}", + input.name, + String::from_utf8_lossy(&status.stderr).trim() + )); + } + if !status.stdout.is_empty() { + return Err(format!( + "git input {:?}: build mutated immutable checkout {}:\n{}", + input.name, + checkout.display(), + String::from_utf8_lossy(&status.stdout).trim() + )); + } + Ok(()) +} + +fn validate_git_input_tree( + input: &GitBuildInput, + checkout: &Path, + isolation: &GitCommandIsolation, + ambient_env: &[(std::ffi::OsString, std::ffi::OsString)], +) -> Result<(), String> { + let index = git_output( + checkout, + &["ls-files", "--stage", "-z"], + &input.repository, + isolation, + ambient_env, + )?; + if !index.status.success() { + return Err(format!( + "git input {:?}: cannot inspect index: {}", + input.name, + String::from_utf8_lossy(&index.stderr).trim() + )); + } + for record in index + .stdout + .split(|byte| *byte == 0) + .filter(|record| !record.is_empty()) + { + let metadata = record.split(|byte| *byte == b'\t').next().unwrap_or(record); + let mode = metadata + .split(|byte| *byte == b' ') + .next() + .unwrap_or(metadata); + if mode == b"160000" { + return Err(format!( + "git input {:?}: submodule gitlinks are not allowed in immutable build inputs", + input.name + )); + } + } + + let canonical = std::fs::canonicalize(checkout) + .map_err(|e| format!("resolve git input root {}: {e}", checkout.display()))?; + let git_metadata_path = checkout.join(".git"); + let git_metadata_lstat = std::fs::symlink_metadata(&git_metadata_path).map_err(|e| { + format!( + "inspect git input metadata directory {}: {e}", + git_metadata_path.display() + ) + })?; + if !git_metadata_lstat.is_dir() || git_metadata_lstat.file_type().is_symlink() { + return Err(format!( + "git input {:?}: .git must be a real non-symlink directory inside its checkout", + input.name + )); + } + let git_metadata = std::fs::canonicalize(&git_metadata_path).map_err(|e| { + format!( + "resolve git input metadata directory {}: {e}", + checkout.join(".git").display() + ) + })?; + if !git_metadata.starts_with(&canonical) { + return Err(format!( + "git input {:?}: .git resolves outside immutable checkout {}", + input.name, + checkout.display() + )); + } + validate_git_input_tree_entries(input, checkout, checkout, &canonical, &git_metadata) } -/// Resolve `rel` against `base` for archive-URL lookup. If `rel` -/// already carries a scheme (`file://` / `http://` / `https://`) it -/// passes through unchanged; otherwise it's appended to `base`'s -/// parent directory (i.e. `https://host/dir/index.toml` + `foo.tar.zst` -/// → `https://host/dir/foo.tar.zst`). -pub(crate) fn resolve_relative_url(base: &str, rel: &str) -> String { - if rel.starts_with("file://") || rel.starts_with("http://") || rel.starts_with("https://") { - return rel.to_string(); +fn validate_git_input_tree_entries( + input: &GitBuildInput, + checkout: &Path, + directory: &Path, + canonical: &Path, + git_metadata: &Path, +) -> Result<(), String> { + let mut entries = std::fs::read_dir(directory) + .map_err(|e| format!("read git input directory {}: {e}", directory.display()))? + .collect::, _>>() + .map_err(|e| format!("read git input entry below {}: {e}", directory.display()))?; + entries.sort_by_key(|entry| entry.path()); + for entry in entries { + let path = entry.path(); + if directory == checkout && entry.file_name() == ".git" { + continue; + } + let metadata = std::fs::symlink_metadata(&path) + .map_err(|e| format!("stat git input entry {}: {e}", path.display()))?; + if metadata.file_type().is_symlink() { + let resolved = std::fs::canonicalize(&path).map_err(|e| { + format!( + "git input {:?}: symlink {} cannot be resolved inside its checkout: {e}", + input.name, + path.display() + ) + })?; + if !resolved.starts_with(canonical) { + return Err(format!( + "git input {:?}: symlink {} escapes immutable checkout {}", + input.name, + path.display(), + checkout.display() + )); + } + if resolved == git_metadata || resolved.starts_with(git_metadata) { + return Err(format!( + "git input {:?}: symlink {} resolves into private Git metadata {}", + input.name, + path.display(), + git_metadata.display() + )); + } + } else if metadata.is_dir() { + validate_git_input_tree_entries(input, checkout, &path, canonical, git_metadata)?; + } else if !metadata.is_file() { + return Err(format!( + "git input {:?}: {} is not a regular file, directory, or contained symlink", + input.name, + path.display() + )); + } } - // Strip the last path segment of `base` and join with `rel`. - let last_slash = base.rfind('/').map(|i| i + 1).unwrap_or(0); - let mut out = String::with_capacity(last_slash + rel.len()); - out.push_str(&base[..last_slash]); - out.push_str(rel); - out + Ok(()) } -/// Build the `WASM_POSIX_DEP_PKG_CONFIG_PATH` value for a build script. -/// -/// Joins every transitive lib path's `lib/pkgconfig/` subdirectory with -/// `:` — POSIX's standard search-path separator, and what pkg-config -/// itself uses for `PKG_CONFIG_PATH`. Paths whose `lib/pkgconfig/` -/// directory doesn't exist (e.g. ncurses, libs that ship no .pc file) -/// are skipped: handing pkg-config a list of nonexistent search paths -/// clutters diagnostics with no benefit. -/// -/// Returns an empty string when no transitive lib ships pkgconfig. The -/// caller still sets the env var to that empty string, keeping the -/// contract uniform: the var is *always* defined for build scripts. -fn compose_pkgconfig_path(paths: &BTreeSet) -> String { - paths - .iter() - .filter_map(|p| { - let pc = p.join("lib").join("pkgconfig"); - if pc.is_dir() { - Some(pc.to_string_lossy().into_owned()) - } else { - None - } - }) - .collect::>() - .join(":") +/// Hash the exported working tree independently of Git's index/status view. +/// A build can toggle index flags such as `assume-unchanged`; it cannot make a +/// byte, symlink target, path, or executable-bit mutation disappear from this +/// resolver-owned digest. +fn digest_git_input_worktree(checkout: &Path) -> Result<[u8; 32], String> { + let mut hasher = Sha256::new(); + hasher.update(b"kandelo-immutable-git-working-tree-v1\0"); + digest_git_input_directory(checkout, checkout, &mut hasher)?; + Ok(hasher.finalize().into()) } -#[derive(Debug)] -struct ProvisionedGitInput { - declaration: GitBuildInput, - checkout: PathBuf, - worktree_digest: [u8; 32], +fn digest_git_input_directory( + checkout: &Path, + directory: &Path, + hasher: &mut Sha256, +) -> Result<(), String> { + let mut entries = std::fs::read_dir(directory) + .map_err(|e| format!("read immutable Git tree {}: {e}", directory.display()))? + .collect::, _>>() + .map_err(|e| format!("read immutable Git tree entry {}: {e}", directory.display()))?; + entries.sort_by_key(|entry| entry.file_name()); + for entry in entries { + if directory == checkout && entry.file_name() == ".git" { + continue; + } + let path = entry.path(); + let relative = path + .strip_prefix(checkout) + .map_err(|_| format!("immutable Git path escaped checkout: {}", path.display()))?; + let path_bytes = relative.as_os_str().as_encoded_bytes(); + let metadata = std::fs::symlink_metadata(&path) + .map_err(|e| format!("stat immutable Git path {}: {e}", path.display()))?; + if metadata.file_type().is_symlink() { + let target = std::fs::read_link(&path) + .map_err(|e| format!("read immutable Git symlink {}: {e}", path.display()))?; + hasher.update(b"symlink\0"); + hasher.update((path_bytes.len() as u64).to_le_bytes()); + hasher.update(path_bytes); + let target_bytes = target.as_os_str().as_encoded_bytes(); + hasher.update((target_bytes.len() as u64).to_le_bytes()); + hasher.update(target_bytes); + } else if metadata.is_dir() { + hasher.update(b"directory\0"); + hasher.update((path_bytes.len() as u64).to_le_bytes()); + hasher.update(path_bytes); + digest_git_input_directory(checkout, &path, hasher)?; + } else if metadata.is_file() { + hasher.update(b"file\0"); + hasher.update((path_bytes.len() as u64).to_le_bytes()); + hasher.update(path_bytes); + hasher.update([git_input_file_is_executable(&metadata) as u8]); + hasher.update(metadata.len().to_le_bytes()); + let mut file = std::fs::File::open(&path) + .map_err(|e| format!("open immutable Git file {}: {e}", path.display()))?; + std::io::copy(&mut file, hasher) + .map_err(|e| format!("hash immutable Git file {}: {e}", path.display()))?; + } else { + return Err(format!( + "immutable Git path is not a file, directory, or symlink: {}", + path.display() + )); + } + } + Ok(()) } -#[derive(Debug)] -struct GitCommandIsolation { - home: PathBuf, - xdg_config_home: PathBuf, - empty_templates: PathBuf, - empty_hooks: PathBuf, - askpass: PathBuf, +#[cfg(unix)] +fn git_input_file_is_executable(metadata: &std::fs::Metadata) -> bool { + use std::os::unix::fs::PermissionsExt; + metadata.permissions().mode() & 0o111 != 0 } -/// Temporary, detached checkouts for a package's declared `git_inputs`. -/// Dropping the guard removes every checkout, including error paths. -#[derive(Debug, Default)] -struct ProvisionedGitInputs { - root: Option, - isolation: Option, - inputs: Vec, +#[cfg(not(unix))] +fn git_input_file_is_executable(_metadata: &std::fs::Metadata) -> bool { + false } -impl ProvisionedGitInputs { - fn provision(target: &DepsManifest, canonical: &Path) -> Result { - let build_path = target.dir.join("build.toml"); - if !build_path.exists() { - return Ok(Self::default()); +#[cfg(unix)] +fn set_git_input_tree_read_only(path: &Path, read_only: bool) -> Result<(), String> { + use std::os::unix::fs::PermissionsExt; + + let metadata = std::fs::symlink_metadata(path) + .map_err(|e| format!("stat immutable git input {}: {e}", path.display()))?; + if metadata.file_type().is_symlink() { + return Ok(()); + } + if metadata.is_dir() && !read_only { + let mut permissions = metadata.permissions(); + permissions.set_mode(permissions.mode() | 0o700); + std::fs::set_permissions(path, permissions) + .map_err(|e| format!("unseal git input directory {}: {e}", path.display()))?; + } + if metadata.is_dir() { + let entries = std::fs::read_dir(path) + .map_err(|e| format!("read immutable git input {}: {e}", path.display()))?; + for entry in entries { + let entry = entry.map_err(|e| format!("read immutable git input entry: {e}"))?; + set_git_input_tree_read_only(&entry.path(), read_only)?; } - let declarations = BuildToml::load(&target.dir)?.git_inputs; - if declarations.is_empty() { - return Ok(Self::default()); + } + let metadata = std::fs::symlink_metadata(path) + .map_err(|e| format!("restat immutable git input {}: {e}", path.display()))?; + let mut permissions = metadata.permissions(); + if read_only { + permissions.set_mode(permissions.mode() & !0o222); + } else { + permissions.set_mode(permissions.mode() | 0o600); + } + std::fs::set_permissions(path, permissions).map_err(|e| { + format!( + "{} immutable git input {}: {e}", + if read_only { "seal" } else { "unseal" }, + path.display() + ) + }) +} + +#[cfg(not(unix))] +fn set_git_input_tree_read_only(path: &Path, read_only: bool) -> Result<(), String> { + let metadata = std::fs::symlink_metadata(path) + .map_err(|e| format!("stat immutable git input {}: {e}", path.display()))?; + if metadata.file_type().is_symlink() { + return Ok(()); + } + if metadata.is_dir() && !read_only { + let mut permissions = metadata.permissions(); + permissions.set_readonly(false); + std::fs::set_permissions(path, permissions) + .map_err(|e| format!("unseal git input directory {}: {e}", path.display()))?; + } + if metadata.is_dir() { + for entry in std::fs::read_dir(path) + .map_err(|e| format!("read immutable git input {}: {e}", path.display()))? + { + let entry = entry.map_err(|e| format!("read immutable git input entry: {e}"))?; + set_git_input_tree_read_only(&entry.path(), read_only)?; } - - Self::provision_declarations(&target.spec(), canonical, declarations) } + let mut permissions = std::fs::symlink_metadata(path) + .map_err(|e| format!("restat immutable git input {}: {e}", path.display()))? + .permissions(); + permissions.set_readonly(read_only); + std::fs::set_permissions(path, permissions).map_err(|e| { + format!( + "{} immutable git input {}: {e}", + if read_only { "seal" } else { "unseal" }, + path.display() + ) + }) +} - fn provision_declarations( - package_spec: &str, - canonical: &Path, - declarations: Vec, - ) -> Result { - Self::provision_declarations_inner(package_spec, canonical, declarations, &[]) +/// Run the build script with `WASM_POSIX_DEP_*` env vars set, validate +/// outputs under the temp directory, then `rename(2)` into place. +/// +/// `pkgconfig_path` is the pre-composed value for +/// `WASM_POSIX_DEP_PKG_CONFIG_PATH` — a colon-joined list of every +/// transitive lib's `lib/pkgconfig/` dir. Always set, even when empty, +/// so the contract for build scripts stays uniform. +fn build_into_cache( + target: &DepsManifest, + arch: TargetArch, + abi_version: u32, + cache_key_sha: &str, + canonical: &Path, + dep_dirs: &BTreeMap, + pkgconfig_path: &str, + repo_root: &Path, +) -> Result<(), String> { + let parent = canonical + .parent() + .ok_or_else(|| format!("canonical path has no parent: {}", canonical.display()))?; + std::fs::create_dir_all(parent) + .map_err(|e| format!("create cache parent {}: {e}", parent.display()))?; + + let tmp = parent.join(format!( + "{}.tmp-{}", + canonical + .file_name() + .expect("canonical path has a filename") + .to_string_lossy(), + std::process::id() + )); + // Fresh temp dir. If a leftover from a crashed build exists, wipe it. + if tmp.exists() { + std::fs::remove_dir_all(&tmp).map_err(|e| format!("clean stale {}: {e}", tmp.display()))?; } + std::fs::create_dir_all(&tmp).map_err(|e| format!("create temp {}: {e}", tmp.display()))?; - #[cfg(test)] - fn provision_declarations_with_ambient_env( - package_spec: &str, - canonical: &Path, - declarations: Vec, - ambient_env: &[(std::ffi::OsString, std::ffi::OsString)], - ) -> Result { - Self::provision_declarations_inner(package_spec, canonical, declarations, ambient_env) + let script = target.build_script_path(repo_root); + if !script.is_file() { + let _ = std::fs::remove_dir_all(&tmp); + return Err(format!( + "{}: build script {} not found", + target.spec(), + script.display() + )); } - fn provision_declarations_inner( - package_spec: &str, - canonical: &Path, - declarations: Vec, - ambient_env: &[(std::ffi::OsString, std::ffi::OsString)], - ) -> Result { - let parent = canonical.parent().ok_or_else(|| { - format!( - "{}: canonical cache path has no parent for git inputs: {}", - package_spec, - canonical.display() - ) - })?; - let basename = canonical - .file_name() - .ok_or_else(|| { - format!( - "canonical cache path has no filename: {}", - canonical.display() - ) - })? - .to_string_lossy(); - let root = create_git_input_root(parent, &basename)?; + let git_inputs = match ProvisionedGitInputs::provision(target, canonical) { + Ok(inputs) => inputs, + Err(e) => { + let _ = std::fs::remove_dir_all(&tmp); + return Err(format!( + "{}: provision immutable git inputs: {e}", + target.spec() + )); + } + }; - let mut provisioned = Self { - root: Some(root.clone()), - isolation: None, - inputs: Vec::with_capacity(declarations.len()), + let status = { + let mut cmd = Command::new("bash"); + cmd.arg(&script); + // Worktree-local SDK invocation. Prepend `/sdk/bin` to PATH + // so build scripts that call `wasm32posix-cc` (and friends) + // resolve to THIS worktree's SDK source — not whatever a global + // `npm link` last pointed at. Without this, a sibling worktree's + // SDK + sysroot can leak into the build, producing binaries with + // a foreign ABI. The shape of `/sdk/bin/` is committed + // symlinks pointing at `_wasm-posix-dispatch`; see + // `docs/package-management.md` "SDK toolchain invocation". + let sdk_bin = crate::repo_root().join("sdk").join("bin"); + let path_var = match std::env::var_os("PATH") { + Some(existing) => { + let mut p = std::ffi::OsString::from(&sdk_bin); + p.push(":"); + p.push(existing); + p + } + None => std::ffi::OsString::from(&sdk_bin), }; - let isolation = create_git_command_isolation(&root)?; - let checkouts = root.join("checkouts"); - std::fs::create_dir(&checkouts).map_err(|e| { - format!( - "create immutable git-input checkout root {}: {e}", - checkouts.display() - ) - })?; - provisioned.isolation = Some(isolation); - for declaration in declarations { - let isolation = provisioned - .isolation - .as_ref() - .expect("git command isolation was initialized"); - let checkout = checkouts.join(&declaration.name); - std::fs::create_dir(&checkout) - .map_err(|e| format!("create git-input checkout {}: {e}", checkout.display()))?; - run_git( - &checkout, - &["init", "--quiet", "--object-format=sha1"], - &declaration.repository, - isolation, - ambient_env, - )?; - run_git( - &checkout, - &[ - "fetch", - "--quiet", - "--depth=1", - "--no-tags", - "--no-recurse-submodules", - &declaration.repository, - &declaration.commit, - ], - &declaration.repository, - isolation, - ambient_env, - )?; - run_git( - &checkout, - &["checkout", "--quiet", "--detach", "FETCH_HEAD"], - &declaration.repository, - isolation, - ambient_env, - )?; - verify_git_input(&declaration, &checkout, isolation, ambient_env)?; - validate_git_input_tree(&declaration, &checkout, isolation, ambient_env)?; - let worktree_digest = digest_git_input_worktree(&checkout)?; - set_git_input_tree_read_only(&checkout, true)?; - provisioned.inputs.push(ProvisionedGitInput { - declaration, - checkout, - worktree_digest, - }); + cmd.env("PATH", path_var); + cmd.env("WASM_POSIX_DEP_OUT_DIR", &tmp); + cmd.env("WASM_POSIX_DEP_NAME", &target.name); + cmd.env("WASM_POSIX_DEP_VERSION", &target.version); + cmd.env("WASM_POSIX_DEP_REVISION", target.revision.to_string()); + cmd.env("WASM_POSIX_DEP_SOURCE_URL", &target.source.url); + cmd.env("WASM_POSIX_DEP_SOURCE_SHA256", &target.source.sha256); + cmd.env("WASM_POSIX_DEP_TARGET_ARCH", arch.as_str()); + cmd.env("WASM_POSIX_DEP_PKG_CONFIG_PATH", pkgconfig_path); + git_inputs.export_to(&mut cmd); + for (name, dep) in dep_dirs { + // Per design 12: library/program deps export under + // `*_DIR` (built-artifact root), source deps under + // `*_SRC_DIR` (unbuilt source tree). The suffix tells a + // build script unambiguously what shape it's consuming. + let suffix = match dep.kind { + ManifestKind::Library | ManifestKind::Program => "DIR", + ManifestKind::Source => "SRC_DIR", + }; + cmd.env( + format!("WASM_POSIX_DEP_{}_{}", env_key(name), suffix), + &dep.path, + ); } - // Seal the containing directory as well as each checkout. Otherwise a - // build that cannot edit files could still rename or replace the path - // exported in WASM_POSIX_BUILD_GIT_*_DIR. - set_git_input_tree_read_only(&root, true)?; - Ok(provisioned) + // INVARIANT: build-script stdout MUST NOT leak to xtask's stdout. + // + // `cmd_resolve` ends with a single `println!("{}", path.display())` + // and consumers shell-capture it with + // `PREFIX="$(cargo run -- build-deps resolve )"`. + // If the bash subprocess's stdout were inherited (the default), + // hundreds of lines of build output would land on xtask's stdout + // ahead of that final println, and `$(...)` would capture the + // entire build log as the "path" — breaking every consumer that + // uses the resolve_dep pattern on a cache miss. + // + // Fix: dup xtask's stderr FD and route the bash subprocess's + // stdout to it. The build progress remains visible to the user + // (it appears on the terminal's stderr stream just like before + // when stdout was a TTY); only the *captured* stdout pipe stays + // clean for the path output. stderr inheritance is unchanged. + let stderr_dup = std::io::stderr() + .as_fd() + .try_clone_to_owned() + .map_err(|e| format!("dup stderr fd for build-script stdout redirect: {e}"))?; + cmd.stdout(Stdio::from(stderr_dup)); + cmd.status() + .map_err(|e| format!("spawn bash {}: {e}", script.display()))? + }; + + if let Err(e) = git_inputs.verify_unchanged() { + let _ = std::fs::remove_dir_all(&tmp); + return Err(format!( + "{}: immutable git input verification failed after build: {e}", + target.spec() + )); + } + + if !status.success() { + let _ = std::fs::remove_dir_all(&tmp); + return Err(format!( + "{}: build script {} exited with {}", + target.spec(), + script.display(), + status + )); } - fn export_to(&self, command: &mut Command) { - for input in &self.inputs { - let key = input.declaration.name.to_ascii_uppercase(); - command.env(format!("WASM_POSIX_BUILD_GIT_{key}_DIR"), &input.checkout); - command.env( - format!("WASM_POSIX_BUILD_GIT_{key}_COMMIT"), - &input.declaration.commit, - ); - } + // Kind-aware validation. Library and program manifests carry a + // declared outputs list (libs/headers/pkgconfig/files or program wasms) + // that `validate_outputs` checks one-by-one. Source manifests have + // no declared outputs — design 11 calls for emptiness as the only + // signal — so we just verify the script populated OUT_DIR with at + // least one entry; an empty dir indicates a no-op script. + let validate_result = match target.kind { + ManifestKind::Library | ManifestKind::Program => validate_outputs(target, &tmp), + ManifestKind::Source => validate_source_dir_nonempty(&tmp), + }; + if let Err(e) = validate_result { + let _ = std::fs::remove_dir_all(&tmp); + return Err(e); } - fn verify_unchanged(&self) -> Result<(), String> { - if self.inputs.is_empty() { - return Ok(()); - } - let isolation = self - .isolation - .as_ref() - .ok_or_else(|| "immutable Git inputs lack command isolation".to_string())?; - for input in &self.inputs { - verify_git_input(&input.declaration, &input.checkout, isolation, &[])?; - validate_git_input_tree(&input.declaration, &input.checkout, isolation, &[])?; - let actual_digest = digest_git_input_worktree(&input.checkout)?; - if actual_digest != input.worktree_digest { - return Err(format!( - "git input {:?}: immutable working-tree digest changed during build", - input.declaration.name - )); - } + // autoconf / libtool bake `--prefix` (= $WASM_POSIX_DEP_OUT_DIR, + // i.e. the temp dir) into generated `.pc` and `.la` files at + // configure time. Rewrite those paths to the canonical location + // *before* the rename so parallel readers never observe a + // canonical cache entry with dead `prefix=` strings. + // + // Skip for source kind: source builds produce a tree (e.g. a + // patched upstream source dir) that won't have `lib/*.{pc,la}` + // and shouldn't — sources aren't installed anywhere. Calling + // `rewrite_install_prefix_paths` would be a harmless no-op + // (`rewrite_dir` returns Ok on missing `lib/`), but skipping + // documents intent and avoids one read_dir. + if !matches!(target.kind, ManifestKind::Source) { + if let Err(e) = rewrite_install_prefix_paths(&tmp, canonical) { + let _ = std::fs::remove_dir_all(&tmp); + return Err(e); } - Ok(()) } + + // Publish resolver metadata beside, never inside, the package tree. The + // marker lands first: a crash leaves harmless metadata without an artifact, + // while no reader can observe an artifact lacking its required provenance. + if let Err(e) = write_cache_provenance(target, canonical, arch, abi_version, cache_key_sha) { + let _ = std::fs::remove_dir_all(&tmp); + return Err(e); + } + + // Atomic install. If someone else finished first, keep theirs, + // discard ours — identical inputs produce identical outputs, and + // trying to overwrite a non-empty directory isn't portable. + if canonical.exists() { + let _ = std::fs::remove_dir_all(&tmp); + return validate_cache_entry(target, canonical, arch, abi_version, cache_key_sha).map_err( + |e| { + format!( + "concurrent cache winner {} failed exact validation: {e}", + canonical.display() + ) + }, + ); + } + std::fs::rename(&tmp, canonical) + .map_err(|e| format!("rename {} -> {}: {e}", tmp.display(), canonical.display()))?; + Ok(()) } -impl Drop for ProvisionedGitInputs { - fn drop(&mut self) { - if let Some(root) = self.root.take() { - let _ = set_git_input_tree_read_only(&root, false); - let _ = std::fs::remove_dir_all(root); +/// Replace every occurrence of `tmp` with `canonical` inside +/// installed `.pc` and `.la` files under `tmp/lib/…`. Runs while +/// the tree still lives at `tmp` so the observable canonical cache +/// entry never contains a stale temp path. +/// +/// Only regular files are rewritten: symlinks (e.g. libpng's +/// `libpng.pc → libpng16.pc`) point at the real file and resolve +/// correctly without needing their own rewrite; following them +/// would double-rewrite the target. +fn rewrite_install_prefix_paths(tmp: &Path, canonical: &Path) -> Result<(), String> { + let tmp_s = tmp.to_string_lossy(); + let canonical_s = canonical.to_string_lossy(); + if tmp_s == canonical_s { + return Ok(()); + } + let lib_dir = tmp.join("lib"); + rewrite_dir(&lib_dir, &tmp_s, &canonical_s)?; + let pc_dir = lib_dir.join("pkgconfig"); + rewrite_dir(&pc_dir, &tmp_s, &canonical_s)?; + Ok(()) +} + +fn rewrite_dir(dir: &Path, needle: &str, replacement: &str) -> Result<(), String> { + let rd = match std::fs::read_dir(dir) { + Ok(r) => r, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(e) => return Err(format!("read_dir {}: {e}", dir.display())), + }; + for entry in rd { + let entry = entry.map_err(|e| format!("read_dir {}: {e}", dir.display()))?; + let path = entry.path(); + let ext = match path.extension().and_then(|e| e.to_str()) { + Some(e) => e, + None => continue, + }; + if ext != "pc" && ext != "la" { + continue; + } + // `symlink_metadata` so we see the symlink itself, not its + // target. Skip symlinks — they resolve to the rewritten real + // file, and rewriting through them would double-rewrite the + // target (causing the replacement to match itself) or, worse, + // replace the symlink with a regular file via `write`. + let meta = std::fs::symlink_metadata(&path) + .map_err(|e| format!("symlink_metadata {}: {e}", path.display()))?; + if !meta.file_type().is_file() { + continue; + } + let content = + std::fs::read_to_string(&path).map_err(|e| format!("read {}: {e}", path.display()))?; + if !content.contains(needle) { + continue; } + let rewritten = content.replace(needle, replacement); + std::fs::write(&path, rewritten).map_err(|e| format!("write {}: {e}", path.display()))?; } + Ok(()) } -static GIT_INPUT_TEMP_COUNTER: AtomicU64 = AtomicU64::new(0); +const WASM_MAGIC: &[u8; 4] = b"\0asm"; +const WPK_FORK_EXPORTS: [&str; 5] = [ + "wpk_fork_unwind_begin", + "wpk_fork_unwind_end", + "wpk_fork_rewind_begin", + "wpk_fork_rewind_end", + "wpk_fork_state", +]; +const EXECUTABLE_PROGRAM_REQUIRED_EXPORTS: [&str; 2] = ["__abi_version", "_start"]; -fn create_git_input_root(parent: &Path, basename: &str) -> Result { - for _ in 0..10_000 { - let counter = GIT_INPUT_TEMP_COUNTER.fetch_add(1, Ordering::Relaxed); - let root = parent.join(format!( - ".{basename}.git-inputs-{}-{counter}", - std::process::id() - )); - match std::fs::create_dir(&root) { - Ok(()) => { - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700)) - .map_err(|e| { - format!( - "set exclusive git-input root permissions {}: {e}", - root.display() - ) - })?; +fn bytes_contain(haystack: &[u8], needle: &[u8]) -> bool { + !needle.is_empty() && haystack.windows(needle.len()).any(|w| w == needle) +} + +fn is_wasm_bytes(bytes: &[u8]) -> bool { + bytes.len() >= WASM_MAGIC.len() && &bytes[..WASM_MAGIC.len()] == WASM_MAGIC +} + +#[derive(Default)] +struct WasmArtifactFacts { + imports_kernel_fork: bool, + exports: BTreeSet, + is_relocatable_object: bool, +} + +fn wasm_artifact_facts(bytes: &[u8]) -> Result { + use wasmparser::{Imports, Parser, Payload}; + + let mut facts = WasmArtifactFacts::default(); + for payload in Parser::new(0).parse_all(bytes) { + match payload.map_err(|e| format!("parse wasm: {e}"))? { + Payload::ImportSection(r) => { + for group in r { + let group = group.map_err(|e| format!("import section: {e}"))?; + match group { + Imports::Single(_, imp) => { + if imp.module == "kernel" && imp.name == "kernel_fork" { + facts.imports_kernel_fork = true; + } + } + Imports::Compact1 { module, items } => { + for item in items { + let item = item.map_err(|e| format!("import section: {e}"))?; + if module == "kernel" && item.name == "kernel_fork" { + facts.imports_kernel_fork = true; + } + } + } + Imports::Compact2 { module, names, .. } => { + for name in names { + let name = name.map_err(|e| format!("import section: {e}"))?; + if module == "kernel" && name == "kernel_fork" { + facts.imports_kernel_fork = true; + } + } + } + } } - return Ok(root); } - Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, - Err(e) => { - return Err(format!( - "create exclusive git-input root {}: {e}", - root.display() - )); + Payload::ExportSection(r) => { + for export in r { + let export = export.map_err(|e| format!("export section: {e}"))?; + facts.exports.insert(export.name.to_string()); + } + } + Payload::CustomSection(c) => { + let name = c.name(); + if name == "linking" || name.starts_with("reloc.") { + facts.is_relocatable_object = true; + } } + _ => {} } } - Err(format!( - "could not allocate an exclusive git-input root below {}", - parent.display() - )) + Ok(facts) +} + +#[cfg(test)] +fn wasm_artifact_policy_failures( + bytes: &[u8], + fork_instrumentation: ForkInstrumentationPolicy, +) -> Vec { + wasm_artifact_policy_failures_for(bytes, fork_instrumentation, &[]) } -fn create_git_command_isolation(root: &Path) -> Result { - let home = root.join("home"); - let xdg_config_home = root.join("xdg-config"); - let empty_templates = root.join("empty-templates"); - let empty_hooks = root.join("empty-hooks"); - for path in [&home, &xdg_config_home, &empty_templates, &empty_hooks] { - std::fs::create_dir(path) - .map_err(|e| format!("create isolated Git directory {}: {e}", path.display()))?; +fn wasm_artifact_policy_failures_for( + bytes: &[u8], + fork_instrumentation: ForkInstrumentationPolicy, + required_exports: &[&str], +) -> Vec { + if !is_wasm_bytes(bytes) { + if required_exports.is_empty() { + return Vec::new(); + } + return vec!["is not a wasm binary".to_string()]; } - let askpass = root.join("askpass-deny.sh"); - std::fs::write(&askpass, "#!/bin/sh\nexit 1\n") - .map_err(|e| format!("write isolated Git askpass {}: {e}", askpass.display()))?; - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(&askpass, std::fs::Permissions::from_mode(0o500)) - .map_err(|e| format!("seal isolated Git askpass {}: {e}", askpass.display()))?; + let mut failures = Vec::new(); + if bytes_contain(bytes, b"asyncify_") { + failures.push("contains legacy asyncify_ instrumentation".to_string()); } - Ok(GitCommandIsolation { - home, - xdg_config_home, - empty_templates, - empty_hooks, - askpass, - }) -} + let facts = match wasm_artifact_facts(bytes) { + Ok(facts) => facts, + Err(e) => { + failures.push(e); + return failures; + } + }; -/// Construct a Git subprocess that cannot inherit source credentials, token -/// headers, hooks, or repository selection from the caller's environment. -/// Build-time Git inputs are public source inputs; a private checkout would be -/// both unreproducible and an accidental credential dependency. -fn hardened_git_command( - repository: &str, - isolation: &GitCommandIsolation, - ambient_env: &[(std::ffi::OsString, std::ffi::OsString)], -) -> Command { - let mut command = Command::new("git"); - // Test callers can inject a hostile ambient environment without mutating - // process-global state. Production passes an empty slice; Command still - // begins with the real inherited environment in both cases. - command.envs(ambient_env.iter().cloned()); - command - .arg("-c") - .arg(format!( - "core.hooksPath={}", - isolation.empty_hooks.to_string_lossy() - )) - .arg("-c") - .arg(format!( - "init.templateDir={}", - isolation.empty_templates.to_string_lossy() - )) - .arg("-c") - .arg("credential.helper=") - .arg("-c") - .arg("credential.interactive=false") - .arg("-c") - .arg("http.extraHeader=") - .arg("-c") - .arg("http.cookieFile=") - .arg("-c") - .arg("http.saveCookies=false") - .arg("-c") - .arg("http.followRedirects=false") - .arg("-c") - .arg("submodule.recurse=false") - .arg("-c") - .arg("core.autocrlf=false") - .arg("-c") - .arg("core.eol=lf") - .env("HOME", &isolation.home) - .env("XDG_CONFIG_HOME", &isolation.xdg_config_home) - .env("GIT_ATTR_NOSYSTEM", "1") - .env("GIT_CONFIG_NOSYSTEM", "1") - .env("GIT_CONFIG_GLOBAL", "/dev/null") - .env("GIT_CONFIG_SYSTEM", "/dev/null") - .env("GIT_CONFIG_COUNT", "0") - .env("GIT_TERMINAL_PROMPT", "0") - .env("GIT_ASKPASS", &isolation.askpass) - .env("GIT_ASKPASS_REQUIRE", "force") - .env("SSH_ASKPASS", &isolation.askpass) - .env("SSH_ASKPASS_REQUIRE", "force") - .env("GIT_OPTIONAL_LOCKS", "0") - .env("GIT_DEFAULT_HASH", "sha1") - .env("GIT_NO_REPLACE_OBJECTS", "1") - .env("GIT_PROTOCOL_FROM_USER", "0") - .env( - "GIT_ALLOW_PROTOCOL", - if repository.starts_with("file://") { - // Private test helpers construct local repositories directly; - // BuildToml validation makes this unreachable for real inputs. - "https:file" - } else { - "https" - }, - ); + if facts.is_relocatable_object { + return failures; + } - for key in [ - "GH_TOKEN", - "GITHUB_TOKEN", - "HOMEBREW_GITHUB_PACKAGES_TOKEN", - "HOMEBREW_GITHUB_API_TOKEN", - "HOMEBREW_DOCKER_REGISTRY_TOKEN", - "GIT_DIR", - "GIT_COMMON_DIR", - "GIT_WORK_TREE", - "GIT_INDEX_FILE", - "GIT_OBJECT_DIRECTORY", - "GIT_ALTERNATE_OBJECT_DIRECTORIES", - "GIT_CONFIG_PARAMETERS", - "GIT_CONFIG", - "GIT_SSH", - "GIT_SSH_COMMAND", - "GIT_TEMPLATE_DIR", - "GIT_EXEC_PATH", - "GIT_PROXY_COMMAND", - "GIT_NAMESPACE", - "GIT_REPLACE_REF_BASE", - "GIT_SHALLOW_FILE", - "GIT_GRAFT_FILE", - "GIT_CEILING_DIRECTORIES", - "GIT_DISCOVERY_ACROSS_FILESYSTEM", - "GIT_EXTERNAL_DIFF", - "GIT_DIFF_OPTS", - "GIT_EDITOR", - "GIT_PAGER", - "GIT_TRACE", - "GIT_TRACE2", - "GIT_TRACE_CURL", - "GIT_CURL_VERBOSE", - "NETRC", - "SSH_AUTH_SOCK", - ] { - command.env_remove(key); + let missing_required_exports = required_exports + .iter() + .copied() + .filter(|name| !facts.exports.contains(*name)) + .collect::>(); + if !missing_required_exports.is_empty() { + failures.push(format!( + "missing required exports: {}", + missing_required_exports.join(", ") + )); } - for (key, _) in std::env::vars_os().chain(ambient_env.iter().cloned()) { - let key_text = key.to_string_lossy(); - if key_text.starts_with("GIT_CONFIG_KEY_") - || key_text.starts_with("GIT_CONFIG_VALUE_") - || key_text.starts_with("GIT_TRACE_") - { - command.env_remove(key); + + let wpk_present: Vec<&str> = WPK_FORK_EXPORTS + .iter() + .copied() + .filter(|name| facts.exports.contains(*name)) + .collect(); + if fork_instrumentation == ForkInstrumentationPolicy::Disabled { + if !wpk_present.is_empty() { + failures.push( + "has wasm-fork-instrument exports but this output disables fork instrumentation" + .to_string(), + ); } + return failures; } - command + if !wpk_present.is_empty() && wpk_present.len() != WPK_FORK_EXPORTS.len() { + let missing = WPK_FORK_EXPORTS + .iter() + .copied() + .filter(|name| !wpk_present.contains(name)) + .collect::>() + .join(", "); + failures.push(format!( + "has incomplete wasm-fork-instrument exports; missing {missing}" + )); + } + if facts.imports_kernel_fork && wpk_present.len() != WPK_FORK_EXPORTS.len() { + failures.push( + "imports kernel.kernel_fork without complete wasm-fork-instrument exports".to_string(), + ); + } + failures } -fn git_output( - checkout: &Path, - args: &[&str], - repository: &str, - isolation: &GitCommandIsolation, - ambient_env: &[(std::ffi::OsString, std::ffi::OsString)], -) -> Result { - let mut command = hardened_git_command(repository, isolation, ambient_env); - let output = command - .arg("-C") - .arg(checkout) - .args(args) - .output() - .map_err(|e| format!("spawn isolated git in {}: {e}", checkout.display()))?; - Ok(output) +fn validate_wasm_artifact_policy( + path: &Path, + fork_instrumentation: ForkInstrumentationPolicy, + required_exports: &[&str], +) -> Result<(), String> { + let bytes = std::fs::read(path).map_err(|e| format!("read {}: {e}", path.display()))?; + let failures = + wasm_artifact_policy_failures_for(&bytes, fork_instrumentation, required_exports); + if failures.is_empty() { + Ok(()) + } else { + Err(format!("{}: {}", path.display(), failures.join("; "))) + } } -fn run_git( - checkout: &Path, - args: &[&str], - repository: &str, - isolation: &GitCommandIsolation, - ambient_env: &[(std::ffi::OsString, std::ffi::OsString)], -) -> Result<(), String> { - let output = git_output(checkout, args, repository, isolation, ambient_env)?; - if output.status.success() { - return Ok(()); +fn required_exports_for_program_output( + target: &DepsManifest, + out: &crate::pkg_manifest::ProgramOutput, +) -> &'static [&'static str] { + if target.name == "kernel" && out.name == "kernel" { + wasm_posix_shared::abi::HOST_ADAPTER_REQUIRED_KERNEL_EXPORTS + } else if out.wasm.ends_with(".wasm") && target.name != "userspace" { + &EXECUTABLE_PROGRAM_REQUIRED_EXPORTS + } else { + &[] } - Err(format!( - "isolated git {:?} failed in {} with {}: {}", - args, - checkout.display(), - output.status, - String::from_utf8_lossy(&output.stderr).trim() - )) } -fn verify_git_input( - input: &GitBuildInput, - checkout: &Path, - isolation: &GitCommandIsolation, - ambient_env: &[(std::ffi::OsString, std::ffi::OsString)], -) -> Result<(), String> { - let head = git_output( - checkout, - &["rev-parse", "HEAD^{commit}"], - &input.repository, - isolation, - ambient_env, - )?; - if !head.status.success() { +fn validate_declared_artifact( + target: &DepsManifest, + root: &Path, + rel: &str, + label: &str, + missing_suffix: &str, + require_regular_file: bool, +) -> Result { + let path = root.join(rel); + let metadata = std::fs::symlink_metadata(&path).map_err(|_| { + format!( + "{}: declared {} output {:?} {}", + target.spec(), + label, + rel, + missing_suffix + ) + })?; + if metadata.file_type().is_symlink() { return Err(format!( - "git input {:?}: cannot resolve detached HEAD in {}: {}", - input.name, - checkout.display(), - String::from_utf8_lossy(&head.stderr).trim() + "{}: declared {} output {:?} must not be a symlink", + target.spec(), + label, + rel )); } - let actual = String::from_utf8_lossy(&head.stdout).trim().to_string(); - if actual != input.commit { + let canonical_root = std::fs::canonicalize(root) + .map_err(|e| format!("{}: resolve package artifact root: {e}", target.spec()))?; + let resolved = std::fs::canonicalize(&path).map_err(|e| { + format!( + "{}: resolve declared {} output {:?}: {e}", + target.spec(), + label, + rel + ) + })?; + if !resolved.starts_with(&canonical_root) { return Err(format!( - "git input {:?}: expected commit {}, checkout has {}", - input.name, input.commit, actual + "{}: declared {} output {:?} resolves outside the package artifact root", + target.spec(), + label, + rel )); } - - let branch = git_output( - checkout, - &["rev-parse", "--abbrev-ref", "HEAD"], - &input.repository, - isolation, - ambient_env, - )?; - if !branch.status.success() || String::from_utf8_lossy(&branch.stdout).trim() != "HEAD" { + if require_regular_file && !metadata.is_file() { return Err(format!( - "git input {:?}: checkout must remain at a detached HEAD", - input.name + "{}: declared {} output {:?} must be a regular file", + target.spec(), + label, + rel )); } + if !require_regular_file { + if metadata.is_file() { + return Ok(path); + } + if !metadata.is_dir() { + return Err(format!( + "{}: declared {} output {:?} must be a regular file or directory", + target.spec(), + label, + rel + )); + } + let mut active_dirs = BTreeSet::new(); + let leaf_count = validate_artifact_tree(&canonical_root, &path, &mut active_dirs)?; + if leaf_count == 0 { + return Err(format!( + "{}: declared {} output {:?} is an empty directory and cannot round-trip through an artifact archive", + target.spec(), + label, + rel + )); + } + } + Ok(path) +} - let status = git_output( - checkout, - &[ - "status", - "--porcelain=v1", - "--untracked-files=all", - "--ignored=matching", - ], - &input.repository, - isolation, - ambient_env, - )?; - if !status.status.success() { +/// Validate every reachable leaf below a declared artifact directory. +/// Internal symlinks are allowed because several library packages publish +/// compatibility aliases; external/cyclic links and special files are not. +fn validate_artifact_tree( + canonical_root: &Path, + path: &Path, + active_dirs: &mut BTreeSet, +) -> Result { + let link_metadata = std::fs::symlink_metadata(path) + .map_err(|e| format!("stat package artifact {}: {e}", path.display()))?; + let resolved = std::fs::canonicalize(path) + .map_err(|e| format!("resolve package artifact {}: {e}", path.display()))?; + if !resolved.starts_with(canonical_root) { return Err(format!( - "git input {:?}: cannot verify clean checkout: {}", - input.name, - String::from_utf8_lossy(&status.stderr).trim() + "package artifact {} resolves outside {}", + path.display(), + canonical_root.display() )); } - if !status.stdout.is_empty() { + let metadata = if link_metadata.file_type().is_symlink() { + std::fs::metadata(path) + .map_err(|e| format!("follow package artifact symlink {}: {e}", path.display()))? + } else { + link_metadata + }; + if metadata.is_file() { + return Ok(1); + } + if !metadata.is_dir() { return Err(format!( - "git input {:?}: build mutated immutable checkout {}:\n{}", - input.name, - checkout.display(), - String::from_utf8_lossy(&status.stdout).trim() + "package artifact {} is not a regular file, directory, or contained symlink", + path.display() )); } - Ok(()) -} - -fn validate_git_input_tree( - input: &GitBuildInput, - checkout: &Path, - isolation: &GitCommandIsolation, - ambient_env: &[(std::ffi::OsString, std::ffi::OsString)], -) -> Result<(), String> { - let index = git_output( - checkout, - &["ls-files", "--stage", "-z"], - &input.repository, - isolation, - ambient_env, - )?; - if !index.status.success() { + if !active_dirs.insert(resolved.clone()) { return Err(format!( - "git input {:?}: cannot inspect index: {}", - input.name, - String::from_utf8_lossy(&index.stderr).trim() + "package artifact directory symlink cycle reaches {}", + path.display() )); } - for record in index - .stdout - .split(|byte| *byte == 0) - .filter(|record| !record.is_empty()) - { - let metadata = record.split(|byte| *byte == b'\t').next().unwrap_or(record); - let mode = metadata - .split(|byte| *byte == b' ') - .next() - .unwrap_or(metadata); - if mode == b"160000" { + let mut entries = std::fs::read_dir(path) + .map_err(|e| format!("read package artifact directory {}: {e}", path.display()))? + .collect::, _>>() + .map_err(|e| format!("read package artifact directory {}: {e}", path.display()))?; + entries.sort_by_key(|entry| entry.path()); + let mut leaves = 0usize; + for entry in entries { + leaves += validate_artifact_tree(canonical_root, &entry.path(), active_dirs)?; + } + active_dirs.remove(&resolved); + Ok(leaves) +} + +fn validate_cache_entry( + target: &DepsManifest, + dir: &Path, + arch: TargetArch, + abi_version: u32, + cache_key_sha: &str, +) -> Result<(), String> { + validate_cache_artifacts(target, dir)?; + validate_cache_provenance(target, dir, arch, abi_version, cache_key_sha) +} + +fn remove_cache_entry(canonical: &Path, cache_key_sha: &str) -> Result<(), String> { + match std::fs::symlink_metadata(canonical) { + Ok(metadata) if metadata.file_type().is_symlink() || metadata.is_file() => { + std::fs::remove_file(canonical) + .map_err(|e| format!("remove stale cache path {}: {e}", canonical.display()))?; + } + Ok(metadata) if metadata.is_dir() => { + std::fs::remove_dir_all(canonical) + .map_err(|e| format!("remove stale cache entry {}: {e}", canonical.display()))?; + } + Ok(_) => { return Err(format!( - "git input {:?}: submodule gitlinks are not allowed in immutable build inputs", - input.name + "refusing to remove special cache path {}", + canonical.display() )); } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(format!("inspect cache path {}: {e}", canonical.display())), } - - let canonical = std::fs::canonicalize(checkout) - .map_err(|e| format!("resolve git input root {}: {e}", checkout.display()))?; - let git_metadata_path = checkout.join(".git"); - let git_metadata_lstat = std::fs::symlink_metadata(&git_metadata_path).map_err(|e| { - format!( - "inspect git input metadata directory {}: {e}", - git_metadata_path.display() - ) - })?; - if !git_metadata_lstat.is_dir() || git_metadata_lstat.file_type().is_symlink() { - return Err(format!( - "git input {:?}: .git must be a real non-symlink directory inside its checkout", - input.name - )); - } - let git_metadata = std::fs::canonicalize(&git_metadata_path).map_err(|e| { - format!( - "resolve git input metadata directory {}: {e}", - checkout.join(".git").display() - ) - })?; - if !git_metadata.starts_with(&canonical) { - return Err(format!( - "git input {:?}: .git resolves outside immutable checkout {}", - input.name, - checkout.display() - )); - } - validate_git_input_tree_entries(input, checkout, checkout, &canonical, &git_metadata) + remove_cache_provenance(canonical, cache_key_sha) } -fn validate_git_input_tree_entries( - input: &GitBuildInput, - checkout: &Path, - directory: &Path, - canonical: &Path, - git_metadata: &Path, -) -> Result<(), String> { - let mut entries = std::fs::read_dir(directory) - .map_err(|e| format!("read git input directory {}: {e}", directory.display()))? - .collect::, _>>() - .map_err(|e| format!("read git input entry below {}: {e}", directory.display()))?; - entries.sort_by_key(|entry| entry.path()); - for entry in entries { - let path = entry.path(); - if directory == checkout && entry.file_name() == ".git" { - continue; +pub(crate) fn validate_cache_artifacts(target: &DepsManifest, dir: &Path) -> Result<(), String> { + match target.kind { + ManifestKind::Library => { + for rel in &target.outputs.libs { + validate_declared_artifact( + target, + dir, + rel, + "libs", + "missing from cache entry", + true, + )?; + } + for rel in &target.outputs.headers { + validate_declared_artifact( + target, + dir, + rel, + "headers", + "missing from cache entry", + false, + )?; + } + for rel in &target.outputs.pkgconfig { + validate_declared_artifact( + target, + dir, + rel, + "pkgconfig", + "missing from cache entry", + true, + )?; + } + for rel in &target.outputs.files { + validate_declared_artifact( + target, + dir, + rel, + "files", + "missing from cache entry", + true, + )?; + } } - let metadata = std::fs::symlink_metadata(&path) - .map_err(|e| format!("stat git input entry {}: {e}", path.display()))?; - if metadata.file_type().is_symlink() { - let resolved = std::fs::canonicalize(&path).map_err(|e| { - format!( - "git input {:?}: symlink {} cannot be resolved inside its checkout: {e}", - input.name, - path.display() - ) - })?; - if !resolved.starts_with(canonical) { - return Err(format!( - "git input {:?}: symlink {} escapes immutable checkout {}", - input.name, - path.display(), - checkout.display() - )); + ManifestKind::Program => { + for out in &target.program_outputs { + let path = validate_declared_artifact( + target, + dir, + &out.wasm, + "wasm", + "missing from cache entry", + true, + )?; + validate_wasm_artifact_policy( + &path, + out.fork_instrumentation, + required_exports_for_program_output(target, out), + )?; } - if resolved == git_metadata || resolved.starts_with(git_metadata) { - return Err(format!( - "git input {:?}: symlink {} resolves into private Git metadata {}", - input.name, - path.display(), - git_metadata.display() - )); + for runtime_file in &target.runtime_files { + validate_declared_artifact( + target, + dir, + &runtime_file.artifact, + "runtime file", + "missing from cache entry", + true, + )?; } - } else if metadata.is_dir() { - validate_git_input_tree_entries(input, checkout, &path, canonical, git_metadata)?; - } else if !metadata.is_file() { - return Err(format!( - "git input {:?}: {} is not a regular file, directory, or contained symlink", - input.name, - path.display() - )); } + ManifestKind::Source => {} } Ok(()) } -/// Hash the exported working tree independently of Git's index/status view. -/// A build can toggle index flags such as `assume-unchanged`; it cannot make a -/// byte, symlink target, path, or executable-bit mutation disappear from this -/// resolver-owned digest. -fn digest_git_input_worktree(checkout: &Path) -> Result<[u8; 32], String> { - let mut hasher = Sha256::new(); - hasher.update(b"kandelo-immutable-git-working-tree-v1\0"); - digest_git_input_directory(checkout, checkout, &mut hasher)?; - Ok(hasher.finalize().into()) -} - -fn digest_git_input_directory( - checkout: &Path, - directory: &Path, - hasher: &mut Sha256, -) -> Result<(), String> { - let mut entries = std::fs::read_dir(directory) - .map_err(|e| format!("read immutable Git tree {}: {e}", directory.display()))? - .collect::, _>>() - .map_err(|e| format!("read immutable Git tree entry {}: {e}", directory.display()))?; - entries.sort_by_key(|entry| entry.file_name()); - for entry in entries { - if directory == checkout && entry.file_name() == ".git" { - continue; +fn validate_outputs(target: &DepsManifest, out_dir: &Path) -> Result<(), String> { + match target.kind { + ManifestKind::Library => { + for rel in &target.outputs.libs { + validate_declared_artifact( + target, + out_dir, + rel, + "libs", + "not produced by build script", + true, + )?; + } + for rel in &target.outputs.headers { + validate_declared_artifact( + target, + out_dir, + rel, + "headers", + "not produced by build script", + false, + )?; + } + for rel in &target.outputs.pkgconfig { + validate_declared_artifact( + target, + out_dir, + rel, + "pkgconfig", + "not produced by build script", + true, + )?; + } + for rel in &target.outputs.files { + validate_declared_artifact( + target, + out_dir, + rel, + "files", + "not produced by build script", + true, + )?; + } } - let path = entry.path(); - let relative = path - .strip_prefix(checkout) - .map_err(|_| format!("immutable Git path escaped checkout: {}", path.display()))?; - let path_bytes = relative.as_os_str().as_encoded_bytes(); - let metadata = std::fs::symlink_metadata(&path) - .map_err(|e| format!("stat immutable Git path {}: {e}", path.display()))?; - if metadata.file_type().is_symlink() { - let target = std::fs::read_link(&path) - .map_err(|e| format!("read immutable Git symlink {}: {e}", path.display()))?; - hasher.update(b"symlink\0"); - hasher.update((path_bytes.len() as u64).to_le_bytes()); - hasher.update(path_bytes); - let target_bytes = target.as_os_str().as_encoded_bytes(); - hasher.update((target_bytes.len() as u64).to_le_bytes()); - hasher.update(target_bytes); - } else if metadata.is_dir() { - hasher.update(b"directory\0"); - hasher.update((path_bytes.len() as u64).to_le_bytes()); - hasher.update(path_bytes); - digest_git_input_directory(checkout, &path, hasher)?; - } else if metadata.is_file() { - hasher.update(b"file\0"); - hasher.update((path_bytes.len() as u64).to_le_bytes()); - hasher.update(path_bytes); - hasher.update([git_input_file_is_executable(&metadata) as u8]); - hasher.update(metadata.len().to_le_bytes()); - let mut file = std::fs::File::open(&path) - .map_err(|e| format!("open immutable Git file {}: {e}", path.display()))?; - std::io::copy(&mut file, hasher) - .map_err(|e| format!("hash immutable Git file {}: {e}", path.display()))?; - } else { - return Err(format!( - "immutable Git path is not a file, directory, or symlink: {}", - path.display() - )); + ManifestKind::Program => { + for out in &target.program_outputs { + let p = validate_declared_artifact( + target, + out_dir, + &out.wasm, + "wasm", + "not produced by build script", + true, + )?; + validate_wasm_artifact_policy( + &p, + out.fork_instrumentation, + required_exports_for_program_output(target, out), + )?; + } + for runtime_file in &target.runtime_files { + validate_declared_artifact( + target, + out_dir, + &runtime_file.artifact, + "runtime file", + "not produced by build script", + true, + )?; + } } + // No outputs to validate for source-kind (Chunk C). + ManifestKind::Source => return Ok(()), + } + Ok(()) +} + +/// Source-kind validation: the override script must have populated +/// `OUT_DIR` with *something*. Source manifests have no declared +/// outputs list (Task C.1 rejects `[outputs]` for source kind), so +/// non-emptiness is the only signal we have that the script did +/// useful work — an empty dir after a successful `bash` exit almost +/// always means the script forgot to write to `$WASM_POSIX_DEP_OUT_DIR` +/// (e.g. wrote to its own working dir, or hard-coded a path). +fn validate_source_dir_nonempty(out_dir: &Path) -> Result<(), String> { + let mut iter = + std::fs::read_dir(out_dir).map_err(|e| format!("read_dir {}: {e}", out_dir.display()))?; + if iter.next().is_none() { + return Err(format!( + "source build script left OUT_DIR empty at {}; \ + scripts MUST populate $WASM_POSIX_DEP_OUT_DIR with at \ + least one file before exiting", + out_dir.display() + )); } Ok(()) } -#[cfg(unix)] -fn git_input_file_is_executable(metadata: &std::fs::Metadata) -> bool { - use std::os::unix::fs::PermissionsExt; - metadata.permissions().mode() & 0o111 != 0 +/// `libcurl` → `LIBCURL`, `zlib-ng` → `ZLIB_NG`. +fn env_key(name: &str) -> String { + name.chars() + .map(|c| match c { + '-' => '_', + c => c.to_ascii_uppercase(), + }) + .collect() } -#[cfg(not(unix))] -fn git_input_file_is_executable(_metadata: &std::fs::Metadata) -> bool { - false -} +// --------------------------------------------------------------------- +// Subcommand dispatch +// --------------------------------------------------------------------- -#[cfg(unix)] -fn set_git_input_tree_read_only(path: &Path, read_only: bool) -> Result<(), String> { - use std::os::unix::fs::PermissionsExt; +/// Fallback default target architecture when neither `--arch` nor +/// `WASM_POSIX_DEFAULT_ARCH` is set. Wasm32 is the dominant target +/// today; wasm64 is opt-in via flag/env. +/// +/// Kept as a constant (rather than inlined) so tests and callers have +/// a single source of truth, and so future changes — e.g. flipping the +/// default once wasm64 is the dominant target — only have to touch +/// one site. +const DEFAULT_ARCH: TargetArch = TargetArch::Wasm32; - let metadata = std::fs::symlink_metadata(path) - .map_err(|e| format!("stat immutable git input {}: {e}", path.display()))?; - if metadata.file_type().is_symlink() { - return Ok(()); - } - if metadata.is_dir() && !read_only { - let mut permissions = metadata.permissions(); - permissions.set_mode(permissions.mode() | 0o700); - std::fs::set_permissions(path, permissions) - .map_err(|e| format!("unseal git input directory {}: {e}", path.display()))?; - } - if metadata.is_dir() { - let entries = std::fs::read_dir(path) - .map_err(|e| format!("read immutable git input {}: {e}", path.display()))?; - for entry in entries { - let entry = entry.map_err(|e| format!("read immutable git input entry: {e}"))?; - set_git_input_tree_read_only(&entry.path(), read_only)?; - } - } - let metadata = std::fs::symlink_metadata(path) - .map_err(|e| format!("restat immutable git input {}: {e}", path.display()))?; - let mut permissions = metadata.permissions(); - if read_only { - permissions.set_mode(permissions.mode() & !0o222); - } else { - permissions.set_mode(permissions.mode() | 0o600); - } - std::fs::set_permissions(path, permissions).map_err(|e| { - format!( - "{} immutable git input {}: {e}", - if read_only { "seal" } else { "unseal" }, - path.display() - ) - }) +/// Read the current kernel ABI version from `crates/shared`. Resolver +/// uses this as a hash input; ABI bumps therefore auto-invalidate every +/// dependent cache entry without any explicit cache-busting work. +fn current_abi_version() -> u32 { + wasm_posix_shared::ABI_VERSION } -#[cfg(not(unix))] -fn set_git_input_tree_read_only(path: &Path, read_only: bool) -> Result<(), String> { - let metadata = std::fs::symlink_metadata(path) - .map_err(|e| format!("stat immutable git input {}: {e}", path.display()))?; - if metadata.file_type().is_symlink() { - return Ok(()); - } - if metadata.is_dir() && !read_only { - let mut permissions = metadata.permissions(); - permissions.set_readonly(false); - std::fs::set_permissions(path, permissions) - .map_err(|e| format!("unseal git input directory {}: {e}", path.display()))?; - } - if metadata.is_dir() { - for entry in std::fs::read_dir(path) - .map_err(|e| format!("read immutable git input {}: {e}", path.display()))? - { - let entry = entry.map_err(|e| format!("read immutable git input entry: {e}"))?; - set_git_input_tree_read_only(&entry.path(), read_only)?; - } +/// Parse a CLI/env value into `TargetArch`. Accepts `wasm32` and +/// `wasm64`; everything else is rejected with an error message that +/// names the unknown value and lists the valid options. +pub(crate) fn parse_target_arch(s: &str) -> Result { + match s { + "wasm32" => Ok(TargetArch::Wasm32), + "wasm64" => Ok(TargetArch::Wasm64), + other => Err(format!( + "unknown --arch value {other:?}; expected wasm32 or wasm64" + )), } - let mut permissions = std::fs::symlink_metadata(path) - .map_err(|e| format!("restat immutable git input {}: {e}", path.display()))? - .permissions(); - permissions.set_readonly(read_only); - std::fs::set_permissions(path, permissions).map_err(|e| { - format!( - "{} immutable git input {}: {e}", - if read_only { "seal" } else { "unseal" }, - path.display() - ) - }) } -/// Run the build script with `WASM_POSIX_DEP_*` env vars set, validate -/// outputs under the temp directory, then `rename(2)` into place. +/// Default target arch for the CLI when no `--arch` is given: +/// 1. `WASM_POSIX_DEFAULT_ARCH` env var, if set and parseable. +/// 2. Fallback to [`DEFAULT_ARCH`]. /// -/// `pkgconfig_path` is the pre-composed value for -/// `WASM_POSIX_DEP_PKG_CONFIG_PATH` — a colon-joined list of every -/// transitive lib's `lib/pkgconfig/` dir. Always set, even when empty, -/// so the contract for build scripts stays uniform. -fn build_into_cache( - target: &DepsManifest, - arch: TargetArch, - abi_version: u32, - cache_key_sha: &str, - canonical: &Path, - dep_dirs: &BTreeMap, - pkgconfig_path: &str, - repo_root: &Path, -) -> Result<(), String> { - let parent = canonical - .parent() - .ok_or_else(|| format!("canonical path has no parent: {}", canonical.display()))?; - std::fs::create_dir_all(parent) - .map_err(|e| format!("create cache parent {}: {e}", parent.display()))?; - - let tmp = parent.join(format!( - "{}.tmp-{}", - canonical - .file_name() - .expect("canonical path has a filename") - .to_string_lossy(), - std::process::id() - )); - // Fresh temp dir. If a leftover from a crashed build exists, wipe it. - if tmp.exists() { - std::fs::remove_dir_all(&tmp).map_err(|e| format!("clean stale {}: {e}", tmp.display()))?; - } - std::fs::create_dir_all(&tmp).map_err(|e| format!("create temp {}: {e}", tmp.display()))?; - - let script = target.build_script_path(repo_root); - if !script.is_file() { - let _ = std::fs::remove_dir_all(&tmp); - return Err(format!( - "{}: build script {} not found", - target.spec(), - script.display() - )); +/// Unparseable env-var values are rejected loudly so a typo doesn't +/// silently fall through to wasm32 (which would be a confusing way to +/// debug "why did my wasm64 build land in the wrong cache slot?"). +fn default_target_arch() -> Result { + match std::env::var("WASM_POSIX_DEFAULT_ARCH") { + Ok(s) => parse_target_arch(&s).map_err(|e| format!("WASM_POSIX_DEFAULT_ARCH: {e}")), + Err(_) => Ok(DEFAULT_ARCH), } +} - let git_inputs = match ProvisionedGitInputs::provision(target, canonical) { - Ok(inputs) => inputs, - Err(e) => { - let _ = std::fs::remove_dir_all(&tmp); - return Err(format!( - "{}: provision immutable git inputs: {e}", - target.spec() - )); +/// Extract `--arch ` / `--arch=` from `args`, leaving +/// non-flag arguments in place. Returns the parsed arch (if any) and +/// the remaining arguments. +/// +/// Hand-rolled rather than pulling in clap; the CLI surface is small +/// and stable. Both forms are accepted and may appear anywhere after +/// the subcommand, so `build-deps path zlib --arch=wasm64`, +/// `build-deps path --arch wasm64 zlib`, and +/// `build-deps --arch=wasm64 path zlib` all work identically. +fn extract_arch_flag(args: Vec) -> Result<(Option, Vec), String> { + let mut arch: Option = None; + let mut rest: Vec = Vec::with_capacity(args.len()); + let mut it = args.into_iter(); + while let Some(a) = it.next() { + if let Some(value) = a.strip_prefix("--arch=") { + if arch.is_some() { + return Err("--arch given more than once".to_string()); + } + arch = Some(parse_target_arch(value)?); + } else if a == "--arch" { + if arch.is_some() { + return Err("--arch given more than once".to_string()); + } + let value = it + .next() + .ok_or_else(|| "--arch requires a value (wasm32 or wasm64)".to_string())?; + arch = Some(parse_target_arch(&value)?); + } else { + rest.push(a); } - }; + } + Ok((arch, rest)) +} - let status = { - let mut cmd = Command::new("bash"); - cmd.arg(&script); - // Worktree-local SDK invocation. Prepend `/sdk/bin` to PATH - // so build scripts that call `wasm32posix-cc` (and friends) - // resolve to THIS worktree's SDK source — not whatever a global - // `npm link` last pointed at. Without this, a sibling worktree's - // SDK + sysroot can leak into the build, producing binaries with - // a foreign ABI. The shape of `/sdk/bin/` is committed - // symlinks pointing at `_wasm-posix-dispatch`; see - // `docs/package-management.md` "SDK toolchain invocation". - let sdk_bin = crate::repo_root().join("sdk").join("bin"); - let path_var = match std::env::var_os("PATH") { - Some(existing) => { - let mut p = std::ffi::OsString::from(&sdk_bin); - p.push(":"); - p.push(existing); - p +/// Extract `--binaries-dir ` / `--binaries-dir=` from +/// `args`, leaving non-flag arguments in place. Mirrors +/// [`extract_arch_flag`]'s shape so `resolve --binaries-dir

` and +/// `--binaries-dir=

resolve` are equivalent. Only meaningful for the +/// `resolve` subcommand: when supplied, the resolver places +/// `/programs///.wasm` symlinks at +/// each declared `[[outputs]]` (see `place_binaries_symlinks`). +/// `install-local-artifact` uses the same root for its higher-priority +/// developer mirror. Other subcommands ignore the value. +fn extract_binaries_dir_flag(args: Vec) -> Result<(Option, Vec), String> { + let mut binaries_dir: Option = None; + let mut rest: Vec = Vec::with_capacity(args.len()); + let mut it = args.into_iter(); + while let Some(a) = it.next() { + if let Some(value) = a.strip_prefix("--binaries-dir=") { + if binaries_dir.is_some() { + return Err("--binaries-dir given more than once".to_string()); + } + binaries_dir = Some(PathBuf::from(value)); + } else if a == "--binaries-dir" { + if binaries_dir.is_some() { + return Err("--binaries-dir given more than once".to_string()); } - None => std::ffi::OsString::from(&sdk_bin), - }; - cmd.env("PATH", path_var); - cmd.env("WASM_POSIX_DEP_OUT_DIR", &tmp); - cmd.env("WASM_POSIX_DEP_NAME", &target.name); - cmd.env("WASM_POSIX_DEP_VERSION", &target.version); - cmd.env("WASM_POSIX_DEP_REVISION", target.revision.to_string()); - cmd.env("WASM_POSIX_DEP_SOURCE_URL", &target.source.url); - cmd.env("WASM_POSIX_DEP_SOURCE_SHA256", &target.source.sha256); - cmd.env("WASM_POSIX_DEP_TARGET_ARCH", arch.as_str()); - cmd.env("WASM_POSIX_DEP_PKG_CONFIG_PATH", pkgconfig_path); - git_inputs.export_to(&mut cmd); - for (name, dep) in dep_dirs { - // Per design 12: library/program deps export under - // `*_DIR` (built-artifact root), source deps under - // `*_SRC_DIR` (unbuilt source tree). The suffix tells a - // build script unambiguously what shape it's consuming. - let suffix = match dep.kind { - ManifestKind::Library | ManifestKind::Program => "DIR", - ManifestKind::Source => "SRC_DIR", - }; - cmd.env( - format!("WASM_POSIX_DEP_{}_{}", env_key(name), suffix), - &dep.path, - ); + let value = it + .next() + .ok_or_else(|| "--binaries-dir requires a directory path".to_string())?; + binaries_dir = Some(PathBuf::from(value)); + } else { + rest.push(a); } - // INVARIANT: build-script stdout MUST NOT leak to xtask's stdout. - // - // `cmd_resolve` ends with a single `println!("{}", path.display())` - // and consumers shell-capture it with - // `PREFIX="$(cargo run -- build-deps resolve )"`. - // If the bash subprocess's stdout were inherited (the default), - // hundreds of lines of build output would land on xtask's stdout - // ahead of that final println, and `$(...)` would capture the - // entire build log as the "path" — breaking every consumer that - // uses the resolve_dep pattern on a cache miss. - // - // Fix: dup xtask's stderr FD and route the bash subprocess's - // stdout to it. The build progress remains visible to the user - // (it appears on the terminal's stderr stream just like before - // when stdout was a TTY); only the *captured* stdout pipe stays - // clean for the path output. stderr inheritance is unchanged. - let stderr_dup = std::io::stderr() - .as_fd() - .try_clone_to_owned() - .map_err(|e| format!("dup stderr fd for build-script stdout redirect: {e}"))?; - cmd.stdout(Stdio::from(stderr_dup)); - cmd.status() - .map_err(|e| format!("spawn bash {}: {e}", script.display()))? + } + Ok((binaries_dir, rest)) +} + +/// Extract `--fetch-only` from `args`, leaving non-flag arguments in place. +/// Only meaningful for `resolve`: it turns archive/source mismatches into +/// errors instead of running package build scripts. +fn extract_fetch_only_flag(args: Vec) -> (bool, Vec) { + let mut fetch_only = false; + let mut rest: Vec = Vec::with_capacity(args.len()); + for a in args { + if a == "--fetch-only" { + fetch_only = true; + } else { + rest.push(a); + } + } + (fetch_only, rest) +} + +pub fn run(args: Vec) -> Result<(), String> { + let (arch_flag, rest) = extract_arch_flag(args)?; + let arch = match arch_flag { + Some(a) => a, + None => default_target_arch()?, }; + // Pull this out before subcommand dispatch so the flag remains + // location-independent, matching `--arch`'s shape. + let (binaries_dir, rest) = extract_binaries_dir_flag(rest)?; + let (fetch_only, rest) = extract_fetch_only_flag(rest); - if let Err(e) = git_inputs.verify_unchanged() { - let _ = std::fs::remove_dir_all(&tmp); + let mut it = rest.into_iter(); + let sub = it.next().ok_or( + "usage: xtask build-deps [--arch=wasm32|wasm64] [--binaries-dir ] [--fetch-only] \ + \ + [ []]", + )?; + let target = it.next(); + // Artifact metadata and local-install subcommands take a second positional + // artifact name; every other subcommand stops at one arg. Pull the extra + // slot up-front so the unexpected-arg check below still catches stray + // inputs for the simple subcommands. + let extra = it.next(); + if it.next().is_some() { + return Err(format!("build-deps {sub}: unexpected extra args")); + } + + let repo = repo_root(); + let registry = Registry::from_env(&repo); + + // Surface a clear error rather than silently ignoring this path on a + // metadata subcommand. + if binaries_dir.is_some() && sub != "resolve" && sub != "install-local-artifact" { return Err(format!( - "{}: immutable git input verification failed after build: {e}", - target.spec() + "build-deps {sub}: --binaries-dir is only valid for `resolve` or `install-local-artifact`" )); } - - if !status.success() { - let _ = std::fs::remove_dir_all(&tmp); + if fetch_only && sub != "resolve" { return Err(format!( - "{}: build script {} exited with {}", - target.spec(), - script.display(), - status + "build-deps {sub}: --fetch-only is only valid for `resolve`" )); } - // Kind-aware validation. Library and program manifests carry a - // declared outputs list (libs/headers/pkgconfig/files or program wasms) - // that `validate_outputs` checks one-by-one. Source manifests have - // no declared outputs — design 11 calls for emptiness as the only - // signal — so we just verify the script populated OUT_DIR with at - // least one entry; an empty dir indicates a no-op script. - let validate_result = match target.kind { - ManifestKind::Library | ManifestKind::Program => validate_outputs(target, &tmp), - ManifestKind::Source => validate_source_dir_nonempty(&tmp), - }; - if let Err(e) = validate_result { - let _ = std::fs::remove_dir_all(&tmp); - return Err(e); + match sub.as_str() { + "check" => { + if target.is_some() { + return Err("build-deps check: takes no arguments".into()); + } + cmd_check(®istry) + } + "program-index-context-check" => { + if target.is_some() || extra.is_some() { + return Err("build-deps program-index-context-check: takes no arguments".into()); + } + check_program_package_indexes_in_context(®istry, true) + } + "output-fork-instrumentation-for-rel" => { + let rel = target.ok_or_else(|| { + "build-deps output-fork-instrumentation-for-rel: missing " + .to_string() + })?; + if extra.is_some() { + return Err( + "build-deps output-fork-instrumentation-for-rel: unexpected extra arg".into(), + ); + } + cmd_output_fork_instrumentation_for_rel(®istry, &rel) + } + "program-index" | "program-index-check" => { + let root = + target.ok_or_else(|| format!("build-deps {sub}: missing "))?; + let output = extra.ok_or_else(|| format!("build-deps {sub}: missing "))?; + if sub == "program-index" { + cmd_program_package_index(Path::new(&root), Path::new(&output), ®istry) + } else { + cmd_check_program_package_index(Path::new(&root), Path::new(&output), ®istry) + } + } + _ => { + let target = target.ok_or_else(|| format!("build-deps {sub}: missing "))?; + // `target` is either a path to a package.toml (contains '/' + // or ends with .toml) or a bare name to look up in the + // registry. + let manifest = load_target(&target, ®istry)?; + match sub.as_str() { + "parse" => { + if extra.is_some() { + return Err("build-deps parse: unexpected extra arg".into()); + } + cmd_parse(&manifest) + } + "sha" => { + if extra.is_some() { + return Err("build-deps sha: unexpected extra arg".into()); + } + cmd_sha(&manifest, ®istry, arch) + } + "path" => { + if extra.is_some() { + return Err("build-deps path: unexpected extra arg".into()); + } + cmd_path(&manifest, ®istry, arch) + } + "resolve" => { + if extra.is_some() { + return Err("build-deps resolve: unexpected extra arg".into()); + } + cmd_resolve( + &manifest, + ®istry, + &repo, + arch, + binaries_dir.as_deref(), + fetch_only, + ) + } + "install-local-artifact" => { + let artifact = extra.ok_or_else(|| { + "build-deps install-local-artifact: missing \ + (usage: build-deps --binaries-dir install-local-artifact )" + .to_string() + })?; + let binaries_dir = binaries_dir.as_deref().ok_or_else(|| { + "build-deps install-local-artifact: --binaries-dir is required".to_string() + })?; + let source = std::env::var_os("WASM_POSIX_LOCAL_INSTALL_SOURCE") + .map(PathBuf::from) + .ok_or_else(|| { + "build-deps install-local-artifact: WASM_POSIX_LOCAL_INSTALL_SOURCE is required" + .to_string() + })?; + let session = std::env::var("WASM_POSIX_LOCAL_INSTALL_SESSION").map_err(|_| { + "build-deps install-local-artifact: WASM_POSIX_LOCAL_INSTALL_SESSION is required" + .to_string() + })?; + cmd_install_local_artifact( + &manifest, + ®istry, + &artifact, + &source, + &session, + binaries_dir, + arch, + ) + } + "output-path" => { + let basename = extra.ok_or_else(|| { + "build-deps output-path: missing \ + (usage: build-deps output-path )" + .to_string() + })?; + cmd_output_path(&manifest, &basename) + } + "output-metadata" => { + let artifact = extra.ok_or_else(|| { + "build-deps output-metadata: missing \ + (usage: build-deps output-metadata )" + .to_string() + })?; + cmd_output_metadata(&manifest, &artifact) + } + "runtime-file-path" => { + let artifact = extra.ok_or_else(|| { + "build-deps runtime-file-path: missing \ + (usage: build-deps runtime-file-path )" + .to_string() + })?; + cmd_runtime_file_path(&manifest, &artifact) + } + "runtime-file-metadata" => { + let artifact = extra.ok_or_else(|| { + "build-deps runtime-file-metadata: missing \ + (usage: build-deps runtime-file-metadata )" + .to_string() + })?; + cmd_runtime_file_metadata(&manifest, &artifact) + } + "output-fork-instrumentation" => { + let basename = extra.ok_or_else(|| { + "build-deps output-fork-instrumentation: missing \ + (usage: build-deps output-fork-instrumentation )" + .to_string() + })?; + cmd_output_fork_instrumentation(&manifest, &basename) + } + other => Err(format!("build-deps: unknown subcommand {other:?}")), + } + } } +} - // autoconf / libtool bake `--prefix` (= $WASM_POSIX_DEP_OUT_DIR, - // i.e. the temp dir) into generated `.pc` and `.la` files at - // configure time. Rewrite those paths to the canonical location - // *before* the rename so parallel readers never observe a - // canonical cache entry with dead `prefix=` strings. - // - // Skip for source kind: source builds produce a tree (e.g. a - // patched upstream source dir) that won't have `lib/*.{pc,la}` - // and shouldn't — sources aren't installed anywhere. Calling - // `rewrite_install_prefix_paths` would be a harmless no-op - // (`rewrite_dir` returns Ok on missing `lib/`), but skipping - // documents intent and avoids one read_dir. - if !matches!(target.kind, ManifestKind::Source) { - if let Err(e) = rewrite_install_prefix_paths(&tmp, canonical) { - let _ = std::fs::remove_dir_all(&tmp); - return Err(e); +fn load_target(target: &str, registry: &Registry) -> Result { + let looks_like_path = + target.ends_with(".toml") || target.contains('/') || target.starts_with('.'); + if looks_like_path { + // Path form: derive the package dir from the .toml path so the + // overlay (sibling `package.pr.toml`) gets honored just like + // for registry-name lookups. Falls through to the plain `load` + // when the path doesn't sit inside a parent dir (rare; a + // top-level filename has no parent). Matches `Registry::load`. + let path = Path::new(target); + match path.parent() { + Some(dir) if !dir.as_os_str().is_empty() => DepsManifest::load_with_overlay(dir), + _ => DepsManifest::load(path), } + } else { + registry.load(target) } +} - // Publish resolver metadata beside, never inside, the package tree. The - // marker lands first: a crash leaves harmless metadata without an artifact, - // while no reader can observe an artifact lacking its required provenance. - if let Err(e) = write_cache_provenance(target, canonical, arch, abi_version, cache_key_sha) { - let _ = std::fs::remove_dir_all(&tmp); - return Err(e); +fn cmd_parse(m: &DepsManifest) -> Result<(), String> { + println!("name = {}", m.name); + println!("version = {}", m.version); + println!("revision = {}", m.revision); + println!("source = {}", m.source.url); + println!("sha256 = {}", m.source.sha256); + println!( + "license = {}{}", + m.license.spdx, + m.license + .url + .as_deref() + .map(|u| format!(" ({u})")) + .unwrap_or_default() + ); + println!( + "depends_on= [{}]", + m.depends_on + .iter() + .map(|d| d.to_string()) + .collect::>() + .join(", ") + ); + println!( + "build = {}", + m.build_script_path(&crate::repo_root()).display() + ); + println!("outputs.libs = {:?}", m.outputs.libs); + println!("outputs.headers = {:?}", m.outputs.headers); + if !m.outputs.pkgconfig.is_empty() { + println!("outputs.pkgconfig= {:?}", m.outputs.pkgconfig); } - - // Atomic install. If someone else finished first, keep theirs, - // discard ours — identical inputs produce identical outputs, and - // trying to overwrite a non-empty directory isn't portable. - if canonical.exists() { - let _ = std::fs::remove_dir_all(&tmp); - return validate_cache_entry(target, canonical, arch, abi_version, cache_key_sha).map_err( - |e| { - format!( - "concurrent cache winner {} failed exact validation: {e}", - canonical.display() - ) - }, - ); + if !m.outputs.files.is_empty() { + println!("outputs.files = {:?}", m.outputs.files); + } + if !m.runtime_files.is_empty() { + println!("runtime_files = {:?}", m.runtime_files); } - std::fs::rename(&tmp, canonical) - .map_err(|e| format!("rename {} -> {}: {e}", tmp.display(), canonical.display()))?; Ok(()) } -/// Replace every occurrence of `tmp` with `canonical` inside -/// installed `.pc` and `.la` files under `tmp/lib/…`. Runs while -/// the tree still lives at `tmp` so the observable canonical cache -/// entry never contains a stale temp path. +fn cmd_sha(m: &DepsManifest, registry: &Registry, arch: TargetArch) -> Result<(), String> { + let mut memo = BTreeMap::new(); + let mut chain = Vec::new(); + let sha = compute_sha( + m, + registry, + arch, + current_abi_version(), + &mut memo, + &mut chain, + )?; + println!("{}", hex(&sha)); + Ok(()) +} + +fn cmd_path(m: &DepsManifest, registry: &Registry, arch: TargetArch) -> Result<(), String> { + let mut memo = BTreeMap::new(); + let mut chain = Vec::new(); + let sha = compute_sha( + m, + registry, + arch, + current_abi_version(), + &mut memo, + &mut chain, + )?; + let path = canonical_path(&default_cache_root(), m, arch, &sha); + println!("{}", path.display()); + Ok(()) +} + +/// `output-path `: print the relative path +/// (under `programs//`) where the resolver places this program's +/// `wasm_basename` output via `place_binaries_symlinks`. /// -/// Only regular files are rewritten: symlinks (e.g. libpng's -/// `libpng.pc → libpng16.pc`) point at the real file and resolve -/// correctly without needing their own rewrite; following them -/// would double-rewrite the target. -fn rewrite_install_prefix_paths(tmp: &Path, canonical: &Path) -> Result<(), String> { - let tmp_s = tmp.to_string_lossy(); - let canonical_s = canonical.to_string_lossy(); - if tmp_s == canonical_s { - return Ok(()); +/// Consumed by `scripts/install-local-binary.sh` so build scripts drop +/// their freshly-built bytes at the same path the resolver writes to. +/// Without this, the build-script-side install-local-binary path could +/// diverge from the resolver path (the case that surfaced for texlive: +/// program "texlive" with output "pdftex" — the resolver writes +/// pdftex.wasm, but install_local_binary historically wrote +/// texlive.wasm or texlive/pdftex.wasm). +fn cmd_output_path(m: &DepsManifest, wasm_basename: &str) -> Result<(), String> { + let rel = m.output_dest_rel(wasm_basename)?; + println!("{}", rel.display()); + Ok(()) +} + +/// One fail-closed lookup used by local build scripts before they mutate or +/// instrument the source artifact. Returning destination and policy together +/// prevents separate manifest reads from observing different registry state. +fn cmd_output_metadata(m: &DepsManifest, wasm_artifact: &str) -> Result<(), String> { + let output = m.output_for_wasm_artifact(wasm_artifact)?; + let value = serde_json::json!({ + "source_artifact": output.wasm, + "mirror_path": m.output_dest_rel_for(output), + "fork_instrumentation": output.fork_instrumentation.as_str(), + }); + println!( + "{}", + serde_json::to_string(&value).map_err(|e| format!("serialize output metadata: {e}"))? + ); + Ok(()) +} + +/// `runtime-file-path `: print the mirror path +/// below `programs//` used by local and resolver materialization. +fn cmd_runtime_file_path(m: &DepsManifest, artifact: &str) -> Result<(), String> { + let rel = m.runtime_file_dest_rel(artifact)?; + println!("{}", rel.display()); + Ok(()) +} + +/// Structured installation contract for VFS/image builders. JSON avoids +/// consumers scraping Debug output and keeps guest path/mode authoritative. +fn cmd_runtime_file_metadata(m: &DepsManifest, artifact: &str) -> Result<(), String> { + let value = runtime_file_metadata_value(m, artifact)?; + println!( + "{}", + serde_json::to_string(&value).map_err(|e| format!("serialize runtime metadata: {e}"))? + ); + Ok(()) +} + +fn runtime_file_metadata_value( + m: &DepsManifest, + artifact: &str, +) -> Result { + let runtime_file = m + .runtime_files + .iter() + .find(|runtime_file| runtime_file.artifact == artifact) + .ok_or_else(|| { + format!( + "program {:?} has no [[runtime_files]] artifact {:?}", + m.name, artifact + ) + })?; + // A runtime file is meaningful only alongside the exact executable and + // side-module outputs produced by the same program package archive. Give + // repo-side consumers the complete resolver mirror closure so they can + // select one materialization tier atomically instead of resolving each + // member independently and accidentally mixing builds. + let closure_mirror_paths: Vec = m + .program_outputs + .iter() + .map(|output| m.output_dest_rel_for(output)) + .chain( + m.runtime_files + .iter() + .map(|runtime_file| m.runtime_file_dest_rel_for(runtime_file)), + ) + .collect(); + Ok(serde_json::json!({ + "artifact": runtime_file.artifact, + "guest_path": runtime_file.guest_path, + "mode": runtime_file.mode, + "mirror_path": m.runtime_file_dest_rel_for(runtime_file), + "closure_mirror_paths": closure_mirror_paths, + })) +} + +fn cmd_output_fork_instrumentation(m: &DepsManifest, wasm_basename: &str) -> Result<(), String> { + let policy = m.output_fork_instrumentation(wasm_basename)?; + println!("{}", policy.as_str()); + Ok(()) +} + +fn cmd_output_fork_instrumentation_for_rel( + registry: &Registry, + resolver_rel: &str, +) -> Result<(), String> { + let policy = output_fork_instrumentation_for_rel(registry, resolver_rel)?; + println!("{}", policy.as_str()); + Ok(()) +} + +fn output_fork_instrumentation_for_rel( + registry: &Registry, + resolver_rel: &str, +) -> Result { + let rel = resolver_rel + .strip_prefix("programs/wasm32/") + .or_else(|| resolver_rel.strip_prefix("programs/wasm64/")) + .or_else(|| resolver_rel.strip_prefix("programs/")) + .unwrap_or(resolver_rel); + for (_, manifest) in programs_by_name(registry)? { + for out in &manifest.program_outputs { + if manifest.output_dest_rel_for(out).to_string_lossy().as_ref() == rel { + return Ok(out.fork_instrumentation); + } + } + } + Ok(ForkInstrumentationPolicy::Auto) +} + +fn cmd_resolve( + m: &DepsManifest, + registry: &Registry, + repo: &Path, + arch: TargetArch, + binaries_dir: Option<&Path>, + fetch_only: bool, +) -> Result<(), String> { + let cache_root = default_cache_root(); + let local_libs = repo.join("local-libs"); + let opts = ResolveOpts { + cache_root: &cache_root, + local_libs: Some(&local_libs), + force_source_build: None, + fetch_only, + repo_root: Some(repo), + // Plumb binaries_dir into ensure_built so place_binaries_symlinks + // runs for every transitive program dep, not just the target. + // The previous direct call here (post-ensure_built) only placed + // symlinks for `m`; consumer build scripts that read sibling + // package binaries via `tryResolveBinary` need the dep + // symlinks too. + binaries_dir, + }; + let path = ensure_built(m, registry, arch, current_abi_version(), &opts)?; + + // Top-level target: ensure_built places symlinks for transitive + // deps via opts.binaries_dir, but the *target's* own symlinks land + // here so we don't recurse into "place self" inside ensure_built + // (which would also fire from archive-stage's ensure_built call, + // where placing target symlinks isn't desired). + if let Some(bdir) = binaries_dir { + if matches!(m.kind, ManifestKind::Program) && !m.program_outputs.is_empty() { + place_binaries_symlinks(m, &path, bdir, arch)?; + } } - let lib_dir = tmp.join("lib"); - rewrite_dir(&lib_dir, &tmp_s, &canonical_s)?; - let pc_dir = lib_dir.join("pkgconfig"); - rewrite_dir(&pc_dir, &tmp_s, &canonical_s)?; + + println!("{}", path.display()); Ok(()) } -fn rewrite_dir(dir: &Path, needle: &str, replacement: &str) -> Result<(), String> { - let rd = match std::fs::read_dir(dir) { - Ok(r) => r, - Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()), - Err(e) => return Err(format!("read_dir {}: {e}", dir.display())), - }; - for entry in rd { - let entry = entry.map_err(|e| format!("read_dir {}: {e}", dir.display()))?; - let path = entry.path(); - let ext = match path.extension().and_then(|e| e.to_str()) { - Some(e) => e, - None => continue, - }; - if ext != "pc" && ext != "la" { - continue; +const LOCAL_GENERATIONS_DIR: &str = ".kandelo-local-generations"; + +#[derive(Clone, Debug)] +struct DeclaredLocalArtifact { + source_suffix: PathBuf, + mirror_relative: PathBuf, + output_index: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +enum LocalArtifactInstall { + Staged { + generation: PathBuf, + remaining: usize, + }, + Published { + mirror: PathBuf, + generation: PathBuf, + }, + Replaced { + mirror: PathBuf, + }, +} + +/// Install one directly built package artifact into the higher-priority +/// `local-binaries` mirror without ever copying through a live mirror symlink. +/// +/// One-member packages retain their historical flat regular-file mirror, but +/// replacement is staged beside the destination and linked into place without +/// following the previous entry. A package with multiple output/runtime +/// members collects exact declared suffixes in one hidden, append-only session +/// generation. Its live package directory changes only after that generation +/// is complete and passes the same cache-artifact validation as a fetched +/// release. +fn cmd_install_local_artifact( + manifest: &DepsManifest, + registry: &Registry, + artifact: &str, + source: &Path, + session: &str, + binaries_dir: &Path, + arch: TargetArch, +) -> Result<(), String> { + let mut memo = BTreeMap::new(); + let mut chain = Vec::new(); + let cache_key_sha = hex(&compute_sha( + manifest, + registry, + arch, + current_abi_version(), + &mut memo, + &mut chain, + )?); + let outcome = install_local_artifact( + manifest, + &cache_key_sha, + artifact, + source, + session, + binaries_dir, + arch, + )?; + match outcome { + LocalArtifactInstall::Staged { + generation, + remaining, + } => { + println!( + "staged {} (waiting for {remaining} declared package artifact{})", + generation.display(), + if remaining == 1 { "" } else { "s" } + ); } - // `symlink_metadata` so we see the symlink itself, not its - // target. Skip symlinks — they resolve to the rewritten real - // file, and rewriting through them would double-rewrite the - // target (causing the replacement to match itself) or, worse, - // replace the symlink with a regular file via `write`. - let meta = std::fs::symlink_metadata(&path) - .map_err(|e| format!("symlink_metadata {}: {e}", path.display()))?; - if !meta.file_type().is_file() { - continue; + LocalArtifactInstall::Published { mirror, generation } => { + println!( + "installed {} from complete local generation {}", + mirror.display(), + generation.display() + ); } - let content = - std::fs::read_to_string(&path).map_err(|e| format!("read {}: {e}", path.display()))?; - if !content.contains(needle) { - continue; + LocalArtifactInstall::Replaced { mirror } => { + println!("installed {}", mirror.display()); } - let rewritten = content.replace(needle, replacement); - std::fs::write(&path, rewritten).map_err(|e| format!("write {}: {e}", path.display()))?; } Ok(()) } -const WASM_MAGIC: &[u8; 4] = b"\0asm"; -const WPK_FORK_EXPORTS: [&str; 5] = [ - "wpk_fork_unwind_begin", - "wpk_fork_unwind_end", - "wpk_fork_rewind_begin", - "wpk_fork_rewind_end", - "wpk_fork_state", -]; -const EXECUTABLE_PROGRAM_REQUIRED_EXPORTS: [&str; 2] = ["__abi_version", "_start"]; +fn install_local_artifact( + manifest: &DepsManifest, + cache_key_sha: &str, + artifact: &str, + source: &Path, + session: &str, + binaries_dir: &Path, + arch: TargetArch, +) -> Result { + if !matches!(manifest.kind, ManifestKind::Program) { + return Err(format!( + "{}: direct local artifact installation is program-only", + manifest.spec() + )); + } + if manifest.program_outputs.is_empty() { + return Err(format!("program {:?} has no [[outputs]]", manifest.name)); + } + if cache_key_sha.len() != 64 + || !cache_key_sha + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(format!( + "{}: local generation cache identity must be 64 lowercase hexadecimal characters", + manifest.spec(), + )); + } -fn bytes_contain(haystack: &[u8], needle: &[u8]) -> bool { - !needle.is_empty() && haystack.windows(needle.len()).any(|w| w == needle) -} + let declared = declared_local_artifact(manifest, artifact)?; + let source_metadata = std::fs::symlink_metadata(source).map_err(|e| { + format!( + "{}: inspect direct local artifact source {}: {e}", + manifest.spec(), + source.display() + ) + })?; + if !source_metadata.is_file() || source_metadata.file_type().is_symlink() { + return Err(format!( + "{}: direct local artifact source must be a regular non-symlink file: {}", + manifest.spec(), + source.display() + )); + } -fn is_wasm_bytes(bytes: &[u8]) -> bool { - bytes.len() >= WASM_MAGIC.len() && &bytes[..WASM_MAGIC.len()] == WASM_MAGIC -} + let binaries_dir = canonical_real_directory(binaries_dir, "local binaries root")?; + let programs_root = binaries_dir.join("programs"); + ensure_real_child_directory(&binaries_dir, &programs_root, "program mirror root")?; + let arch_root = programs_root.join(arch.as_str()); + ensure_real_child_directory(&programs_root, &arch_root, "architecture mirror root")?; + + validate_local_install_session(session)?; + // Keep immutable backing bytes outside `programs//`, which is the + // public resolver namespace. Otherwise a caller could request a hidden + // generation member as an undeclared scalar path and bypass closure + // enforcement. This root is still below `binaries_dir`, so backing bytes + // and the live mirror remain on one filesystem. + let generations_root = binaries_dir.join(LOCAL_GENERATIONS_DIR); + ensure_real_child_directory(&binaries_dir, &generations_root, "local generations root")?; + let arch_generations = generations_root.join(arch.as_str()); + ensure_real_child_directory( + &generations_root, + &arch_generations, + "architecture generations root", + )?; + let package_generations = arch_generations.join(&manifest.name); + ensure_real_child_directory( + &arch_generations, + &package_generations, + "package generations root", + )?; + let identity_generations = package_generations.join(cache_key_sha); + ensure_real_child_directory( + &package_generations, + &identity_generations, + "package cache-identity generations root", + )?; + let generation = identity_generations.join(session); + + // A publication claim is deliberately one-shot and is created before the + // live transaction. If the process is killed at that boundary, a retry + // must use a new session instead of possibly replaying this generation + // over a newer local build. + let publication_claim = identity_generations.join(format!(".{session}.publication-claimed")); + let claimed_before_member = publication_claim_exists(&publication_claim)?; + if claimed_before_member { + // Consumers may already hold canonical paths below this session. + // Never recreate a claimed pathname after its root disappears. + ensure_existing_real_directory(&generation, "claimed local package generation")?; + } else { + ensure_real_child_directory( + &identity_generations, + &generation, + "local package generation", + )?; + } -#[derive(Default)] -struct WasmArtifactFacts { - imports_kernel_fork: bool, - exports: BTreeSet, - is_relocatable_object: bool, -} + let expected = declared_generation_members(manifest)?; + if claimed_before_member { + let present = validate_local_generation_tree(manifest, &generation, &expected)?; + if present != expected.len() { + return Err(format!( + "{}: publication-claimed local generation {} is incomplete; refusing to modify or recreate pinned bytes", + manifest.spec(), + generation.display() + )); + } + } + let generation_member = generation.join(&declared.source_suffix); + install_immutable_generation_member( + manifest, + source, + &generation_member, + &generation, + &identity_generations, + session, + )?; -fn wasm_artifact_facts(bytes: &[u8]) -> Result { - use wasmparser::{Imports, Parser, Payload}; + let present = validate_local_generation_tree(manifest, &generation, &expected)?; + if present < expected.len() { + if claimed_before_member { + return Err(format!( + "{}: publication-claimed local generation {} is incomplete; refusing to change the live mirror", + manifest.spec(), + generation.display() + )); + } + return Ok(LocalArtifactInstall::Staged { + generation, + remaining: expected.len() - present, + }); + } - let mut facts = WasmArtifactFacts::default(); - for payload in Parser::new(0).parse_all(bytes) { - match payload.map_err(|e| format!("parse wasm: {e}"))? { - Payload::ImportSection(r) => { - for group in r { - let group = group.map_err(|e| format!("import section: {e}"))?; - match group { - Imports::Single(_, imp) => { - if imp.module == "kernel" && imp.name == "kernel_fork" { - facts.imports_kernel_fork = true; - } - } - Imports::Compact1 { module, items } => { - for item in items { - let item = item.map_err(|e| format!("import section: {e}"))?; - if module == "kernel" && item.name == "kernel_fork" { - facts.imports_kernel_fork = true; - } - } - } - Imports::Compact2 { module, names, .. } => { - for name in names { - let name = name.map_err(|e| format!("import section: {e}"))?; - if module == "kernel" && name == "kernel_fork" { - facts.imports_kernel_fork = true; - } - } - } + validate_cache_artifacts(manifest, &generation)?; + if !manifest.uses_package_mirror_directory() { + let _output = declared + .output_index + .and_then(|index| manifest.program_outputs.get(index)) + .ok_or_else(|| { + format!( + "{}: a one-member program package must install its declared executable output", + manifest.spec(), + ) + })?; + let canonical_member = std::fs::canonicalize(&generation_member).map_err(|e| { + format!( + "{}: canonicalize immutable local generation member {}: {e}", + manifest.spec(), + generation_member.display(), + ) + })?; + let destination = arch_root.join(&declared.mirror_relative); + let already_claimed = publication_claim_exists(&publication_claim)?; + let live_matches = scalar_mirror_matches_target(&destination, &canonical_member)?; + if already_claimed { + if !live_matches { + return Err(format!( + "{}: local install session {:?} already consumed its one publication attempt but does not own {}; start a new session instead of risking stale-byte replay", + manifest.spec(), + session, + destination.display(), + )); + } + } else { + match claim_local_generation_publication(&publication_claim)? { + PublicationClaim::Created => { + if !live_matches { + replace_mirror_symlink_no_follow( + manifest, + &canonical_member, + &destination, + )?; + } + } + PublicationClaim::Existing => { + if !scalar_mirror_matches_target(&destination, &canonical_member)? { + return Err(format!( + "{}: another writer claimed publication for local install session {:?}; retry after it finishes or start a new session", + manifest.spec(), + session, + )); } } } - Payload::ExportSection(r) => { - for export in r { - let export = export.map_err(|e| format!("export section: {e}"))?; - facts.exports.insert(export.name.to_string()); + } + + return Ok(LocalArtifactInstall::Replaced { + mirror: destination, + }); + } + + let plan = PackageClosureMirrorPlan::validate(manifest, &generation, &arch_root)?; + // Re-read after collection. A concurrent completion may have claimed and + // published this session while this process was copying its member. + let already_claimed = publication_claim_exists(&publication_claim)?; + let live_matches = package_mirror_matches_plan(&plan)?; + if already_claimed { + if !live_matches { + return Err(format!( + "{}: local install session {:?} already consumed its one publication attempt but does not own {}; start a new session instead of risking stale-byte replay", + manifest.spec(), + session, + plan.package_dir.display() + )); + } + } else { + match claim_local_generation_publication(&publication_claim)? { + PublicationClaim::Created => { + if !live_matches { + install_package_closure_mirror(plan.clone())?; } } - Payload::CustomSection(c) => { - let name = c.name(); - if name == "linking" || name.starts_with("reloc.") { - facts.is_relocatable_object = true; + PublicationClaim::Existing => { + if !package_mirror_matches_plan(&plan)? { + return Err(format!( + "{}: another writer claimed publication for local install session {:?}; retry after it finishes or start a new session", + manifest.spec(), + session + )); } } - _ => {} } } - Ok(facts) + + Ok(LocalArtifactInstall::Published { + mirror: plan.package_dir, + generation, + }) } -#[cfg(test)] -fn wasm_artifact_policy_failures( - bytes: &[u8], - fork_instrumentation: ForkInstrumentationPolicy, -) -> Vec { - wasm_artifact_policy_failures_for(bytes, fork_instrumentation, &[]) +fn declared_local_artifact( + manifest: &DepsManifest, + artifact: &str, +) -> Result { + // Exact declaration paths are authoritative and keep otherwise-valid + // packages such as `a/foo.wasm` + `b/foo.wasm` installable. Basename + // matching below is compatibility for existing build scripts only. + let mut exact_matches = Vec::new(); + for (index, output) in manifest.program_outputs.iter().enumerate() { + if output.wasm == artifact { + exact_matches.push(DeclaredLocalArtifact { + source_suffix: PathBuf::from(&output.wasm), + mirror_relative: manifest.output_dest_rel_for(output), + output_index: Some(index), + }); + } + } + for runtime_file in &manifest.runtime_files { + if runtime_file.artifact == artifact { + exact_matches.push(DeclaredLocalArtifact { + source_suffix: PathBuf::from(&runtime_file.artifact), + mirror_relative: manifest.runtime_file_dest_rel_for(runtime_file), + output_index: None, + }); + } + } + match exact_matches.as_slice() { + [declared] => return Ok(declared.clone()), + [] => {} + _ => { + return Err(format!( + "{}: exact artifact path {:?} ambiguously names more than one declared package member", + manifest.spec(), + artifact + )); + } + } + + let mut matches = Vec::new(); + for (index, output) in manifest.program_outputs.iter().enumerate() { + let basename = Path::new(&output.wasm) + .file_name() + .and_then(|value| value.to_str()); + if basename == Some(artifact) { + matches.push(DeclaredLocalArtifact { + source_suffix: PathBuf::from(&output.wasm), + mirror_relative: manifest.output_dest_rel_for(output), + output_index: Some(index), + }); + } + } + match matches.as_slice() { + [declared] => Ok(declared.clone()), + [] => Err(format!( + "{}: {:?} is not a declared [[outputs]].wasm path, unique output basename, or [[runtime_files]].artifact", + manifest.spec(), + artifact + )), + _ => Err(format!( + "{}: {:?} ambiguously names more than one declared package artifact", + manifest.spec(), + artifact + )), + } } -fn wasm_artifact_policy_failures_for( - bytes: &[u8], - fork_instrumentation: ForkInstrumentationPolicy, - required_exports: &[&str], -) -> Vec { - if !is_wasm_bytes(bytes) { - if required_exports.is_empty() { - return Vec::new(); +fn validate_local_install_session(session: &str) -> Result<(), String> { + if session.is_empty() || session.len() > 128 { + return Err( + "WASM_POSIX_LOCAL_INSTALL_SESSION must contain 1..=128 portable characters".to_string(), + ); + } + let mut chars = session.chars(); + let first = chars.next().unwrap(); + if !first.is_ascii_alphanumeric() + || !chars.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '.')) + { + return Err(format!( + "WASM_POSIX_LOCAL_INSTALL_SESSION must begin with an ASCII letter or digit and contain only ASCII letters, digits, '.', '-', or '_': {session:?}" + )); + } + Ok(()) +} + +fn ensure_real_directory(path: &Path, label: &str) -> Result<(), String> { + match std::fs::symlink_metadata(path) { + Ok(metadata) if metadata.is_dir() && !metadata.file_type().is_symlink() => Ok(()), + Ok(_) => Err(format!( + "{label} must be a real directory, not a file or symlink: {}", + path.display() + )), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + std::fs::create_dir_all(path) + .map_err(|e| format!("create {label} {}: {e}", path.display()))?; + let metadata = std::fs::symlink_metadata(path) + .map_err(|e| format!("inspect created {label} {}: {e}", path.display()))?; + if metadata.is_dir() && !metadata.file_type().is_symlink() { + Ok(()) + } else { + Err(format!( + "created {label} is not a real directory: {}", + path.display() + )) + } } - return vec!["is not a wasm binary".to_string()]; + Err(e) => Err(format!("inspect {label} {}: {e}", path.display())), } +} - let mut failures = Vec::new(); - if bytes_contain(bytes, b"asyncify_") { - failures.push("contains legacy asyncify_ instrumentation".to_string()); +/// Authorize an externally supplied publication root once, then operate only +/// below its canonical identity. The root itself may not be a symlink; symlink +/// aliases in earlier host path components are resolved here rather than +/// repeatedly followed while package-owned children are created. +fn canonical_real_directory(path: &Path, label: &str) -> Result { + ensure_real_directory(path, label)?; + let canonical = std::fs::canonicalize(path) + .map_err(|e| format!("canonicalize {label} {}: {e}", path.display()))?; + let metadata = std::fs::symlink_metadata(&canonical) + .map_err(|e| format!("inspect canonical {label} {}: {e}", canonical.display()))?; + if !metadata.is_dir() || metadata.file_type().is_symlink() { + return Err(format!( + "canonical {label} must be a real directory: {}", + canonical.display() + )); + } + Ok(canonical) +} + +fn ensure_existing_real_directory(path: &Path, label: &str) -> Result<(), String> { + let metadata = std::fs::symlink_metadata(path) + .map_err(|e| format!("inspect {label} {}: {e}", path.display()))?; + if metadata.is_dir() && !metadata.file_type().is_symlink() { + Ok(()) + } else { + Err(format!( + "{label} must remain a real directory: {}", + path.display() + )) + } +} + +fn ensure_real_child_directory(parent: &Path, child: &Path, label: &str) -> Result<(), String> { + if child.parent() != Some(parent) { + return Err(format!( + "{label} {} is not an immediate child of {}", + child.display(), + parent.display() + )); + } + match std::fs::symlink_metadata(child) { + Ok(metadata) if metadata.is_dir() && !metadata.file_type().is_symlink() => Ok(()), + Ok(_) => Err(format!( + "{label} must be a real directory, not a file or symlink: {}", + child.display() + )), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => std::fs::create_dir(child) + .map_err(|e| format!("create {label} {}: {e}", child.display())), + Err(e) => Err(format!("inspect {label} {}: {e}", child.display())), + } +} + +fn ensure_generation_member_parent(generation: &Path, member: &Path) -> Result<(), String> { + let relative = member.strip_prefix(generation).map_err(|_| { + format!( + "local generation member {} escapes {}", + member.display(), + generation.display() + ) + })?; + let parent = relative.parent().unwrap_or_else(|| Path::new("")); + let mut current = generation.to_path_buf(); + for component in parent.components() { + let Component::Normal(component) = component else { + return Err(format!( + "local generation member has a non-portable parent path: {}", + relative.display() + )); + }; + let next = current.join(component); + ensure_real_child_directory(¤t, &next, "local generation member directory")?; + current = next; } + Ok(()) +} - let facts = match wasm_artifact_facts(bytes) { - Ok(facts) => facts, +fn install_immutable_generation_member( + manifest: &DepsManifest, + source: &Path, + destination: &Path, + generation: &Path, + package_generations: &Path, + session: &str, +) -> Result<(), String> { + ensure_generation_member_parent(generation, destination)?; + match std::fs::symlink_metadata(destination) { + Ok(metadata) if metadata.is_file() && !metadata.file_type().is_symlink() => { + if files_equal(source, destination)? { + return Ok(()); + } + return Err(format!( + "{}: immutable local generation member already has different bytes: {}; start a new install session", + manifest.spec(), + destination.display() + )); + } + Ok(_) => { + return Err(format!( + "{}: immutable local generation member is not a regular file: {}", + manifest.spec(), + destination.display() + )); + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} Err(e) => { - failures.push(e); - return failures; + return Err(format!( + "{}: inspect local generation member {}: {e}", + manifest.spec(), + destination.display() + )); } - }; + } - if facts.is_relocatable_object { - return failures; + let (stage, mut stage_file) = + reserve_local_member_stage(package_generations, &manifest.name, session)?; + let copied = (|| { + let mut source_file = std::fs::File::open(source) + .map_err(|e| format!("open local artifact source {}: {e}", source.display()))?; + std::io::copy(&mut source_file, &mut stage_file).map_err(|e| { + format!( + "copy local artifact {} into private generation stage {}: {e}", + source.display(), + stage.display() + ) + })?; + stage_file + .sync_all() + .map_err(|e| format!("sync local generation stage {}: {e}", stage.display()))?; + let mut generation_permissions = std::fs::symlink_metadata(source) + .map_err(|e| format!("inspect local artifact source {}: {e}", source.display()))? + .permissions(); + generation_permissions.set_readonly(true); + std::fs::set_permissions(&stage, generation_permissions).map_err(|e| { + format!( + "set local generation member permissions {}: {e}", + stage.display() + ) + })?; + if !files_equal(source, &stage)? { + return Err(format!( + "local artifact source changed while it was copied: {}", + source.display() + )); + } + match std::fs::hard_link(&stage, destination) { + Ok(()) => Ok(()), + Err(link_error) => match std::fs::symlink_metadata(destination) { + Ok(metadata) if metadata.is_file() && !metadata.file_type().is_symlink() => { + if files_equal(&stage, destination)? { + Ok(()) + } else { + Err(format!( + "{}: another writer installed different bytes at immutable generation member {} ({link_error})", + manifest.spec(), + destination.display() + )) + } + } + Ok(_) => Err(format!( + "{}: another writer installed a non-file at immutable generation member {} ({link_error})", + manifest.spec(), + destination.display() + )), + Err(e) => Err(format!( + "{}: publish immutable generation member {} failed ({link_error}); inspect destination also failed ({e})", + manifest.spec(), + destination.display() + )), + }, + } + })(); + drop(stage_file); + let cleanup = std::fs::remove_file(&stage) + .map_err(|e| format!("remove private local member stage {}: {e}", stage.display())); + match (copied, cleanup) { + (Ok(()), Ok(())) => Ok(()), + (Err(error), Ok(())) => Err(error), + (Ok(()), Err(cleanup)) => Err(cleanup), + (Err(error), Err(cleanup)) => Err(format!("{error}; additionally {cleanup}")), } +} - let missing_required_exports = required_exports - .iter() - .copied() - .filter(|name| !facts.exports.contains(*name)) - .collect::>(); - if !missing_required_exports.is_empty() { - failures.push(format!( - "missing required exports: {}", - missing_required_exports.join(", ") +fn reserve_local_member_stage( + parent: &Path, + package_name: &str, + session: &str, +) -> Result<(PathBuf, std::fs::File), String> { + for _ in 0..1024 { + let sequence = MIRROR_TRANSACTION_COUNTER.fetch_add(1, Ordering::Relaxed); + let stage = parent.join(format!( + ".{package_name}.{session}.member-{}-{sequence}", + std::process::id() )); + match std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&stage) + { + Ok(file) => return Ok((stage, file)), + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(e) => { + return Err(format!( + "reserve private local member stage {}: {e}", + stage.display() + )); + } + } } + Err(format!( + "could not allocate a unique local member stage below {}", + parent.display() + )) +} - let wpk_present: Vec<&str> = WPK_FORK_EXPORTS +fn declared_generation_members(manifest: &DepsManifest) -> Result, String> { + let mut members = BTreeSet::new(); + for artifact in manifest + .program_outputs .iter() - .copied() - .filter(|name| facts.exports.contains(*name)) - .collect(); - if fork_instrumentation == ForkInstrumentationPolicy::Disabled { - if !wpk_present.is_empty() { - failures.push( - "has wasm-fork-instrument exports but this output disables fork instrumentation" - .to_string(), - ); + .map(|output| output.wasm.as_str()) + .chain( + manifest + .runtime_files + .iter() + .map(|runtime_file| runtime_file.artifact.as_str()), + ) + { + let path = PathBuf::from(artifact); + if !path + .components() + .all(|component| matches!(component, Component::Normal(_))) + { + return Err(format!( + "{}: declared local generation artifact is not a portable relative path: {:?}", + manifest.spec(), + artifact + )); + } + if !members.insert(path) { + return Err(format!( + "{}: declared local generation artifact appears more than once: {:?}", + manifest.spec(), + artifact + )); } - return failures; - } - if !wpk_present.is_empty() && wpk_present.len() != WPK_FORK_EXPORTS.len() { - let missing = WPK_FORK_EXPORTS - .iter() - .copied() - .filter(|name| !wpk_present.contains(name)) - .collect::>() - .join(", "); - failures.push(format!( - "has incomplete wasm-fork-instrument exports; missing {missing}" - )); - } - if facts.imports_kernel_fork && wpk_present.len() != WPK_FORK_EXPORTS.len() { - failures.push( - "imports kernel.kernel_fork without complete wasm-fork-instrument exports".to_string(), - ); } - failures + Ok(members) } -fn validate_wasm_artifact_policy( - path: &Path, - fork_instrumentation: ForkInstrumentationPolicy, - required_exports: &[&str], -) -> Result<(), String> { - let bytes = std::fs::read(path).map_err(|e| format!("read {}: {e}", path.display()))?; - let failures = - wasm_artifact_policy_failures_for(&bytes, fork_instrumentation, required_exports); - if failures.is_empty() { - Ok(()) - } else { - Err(format!("{}: {}", path.display(), failures.join("; "))) +fn generation_member_directories(members: &BTreeSet) -> BTreeSet { + let mut directories = BTreeSet::new(); + for member in members { + let mut parent = member.parent(); + while let Some(path) = parent { + if path.as_os_str().is_empty() { + break; + } + directories.insert(path.to_path_buf()); + parent = path.parent(); + } } + directories } -fn required_exports_for_program_output( - target: &DepsManifest, - out: &crate::pkg_manifest::ProgramOutput, -) -> &'static [&'static str] { - if target.name == "kernel" && out.name == "kernel" { - wasm_posix_shared::abi::HOST_ADAPTER_REQUIRED_KERNEL_EXPORTS - } else if out.wasm.ends_with(".wasm") && target.name != "userspace" { - &EXECUTABLE_PROGRAM_REQUIRED_EXPORTS - } else { - &[] - } +fn validate_local_generation_tree( + manifest: &DepsManifest, + generation: &Path, + expected: &BTreeSet, +) -> Result { + let expected_directories = generation_member_directories(expected); + let mut present = BTreeSet::new(); + validate_local_generation_tree_inner( + manifest, + generation, + generation, + expected, + &expected_directories, + &mut present, + )?; + Ok(present.len()) } -fn validate_declared_artifact( - target: &DepsManifest, +fn validate_local_generation_tree_inner( + manifest: &DepsManifest, root: &Path, - rel: &str, - label: &str, - missing_suffix: &str, - require_regular_file: bool, -) -> Result { - let path = root.join(rel); - let metadata = std::fs::symlink_metadata(&path).map_err(|_| { + directory: &Path, + expected_files: &BTreeSet, + expected_directories: &BTreeSet, + present: &mut BTreeSet, +) -> Result<(), String> { + let metadata = std::fs::symlink_metadata(directory).map_err(|e| { format!( - "{}: declared {} output {:?} {}", - target.spec(), - label, - rel, - missing_suffix + "{}: inspect local generation directory {}: {e}", + manifest.spec(), + directory.display() ) })?; - if metadata.file_type().is_symlink() { + if !metadata.is_dir() || metadata.file_type().is_symlink() { return Err(format!( - "{}: declared {} output {:?} must not be a symlink", - target.spec(), - label, - rel + "{}: local generation path must be a real directory: {}", + manifest.spec(), + directory.display() )); } - let canonical_root = std::fs::canonicalize(root) - .map_err(|e| format!("{}: resolve package artifact root: {e}", target.spec()))?; - let resolved = std::fs::canonicalize(&path).map_err(|e| { + let entries = std::fs::read_dir(directory).map_err(|e| { format!( - "{}: resolve declared {} output {:?}: {e}", - target.spec(), - label, - rel + "{}: read local generation directory {}: {e}", + manifest.spec(), + directory.display() ) })?; - if !resolved.starts_with(&canonical_root) { - return Err(format!( - "{}: declared {} output {:?} resolves outside the package artifact root", - target.spec(), - label, - rel - )); - } - if require_regular_file && !metadata.is_file() { - return Err(format!( - "{}: declared {} output {:?} must be a regular file", - target.spec(), - label, - rel - )); - } - if !require_regular_file { - if metadata.is_file() { - return Ok(path); - } - if !metadata.is_dir() { + for entry in entries { + let entry = entry.map_err(|e| { + format!( + "{}: read local generation entry below {}: {e}", + manifest.spec(), + directory.display() + ) + })?; + let path = entry.path(); + let relative = path + .strip_prefix(root) + .map_err(|_| { + format!( + "{}: local generation entry {} escapes {}", + manifest.spec(), + path.display(), + root.display() + ) + })? + .to_path_buf(); + let metadata = std::fs::symlink_metadata(&path).map_err(|e| { + format!( + "{}: inspect local generation entry {}: {e}", + manifest.spec(), + path.display() + ) + })?; + if metadata.file_type().is_symlink() { return Err(format!( - "{}: declared {} output {:?} must be a regular file or directory", - target.spec(), - label, - rel + "{}: local generation must not contain symlinks: {}", + manifest.spec(), + path.display() )); } - let mut active_dirs = BTreeSet::new(); - let leaf_count = validate_artifact_tree(&canonical_root, &path, &mut active_dirs)?; - if leaf_count == 0 { + if metadata.is_dir() { + if !expected_directories.contains(&relative) { + return Err(format!( + "{}: local generation contains undeclared directory {}", + manifest.spec(), + relative.display() + )); + } + validate_local_generation_tree_inner( + manifest, + root, + &path, + expected_files, + expected_directories, + present, + )?; + } else if metadata.is_file() { + if !expected_files.contains(&relative) { + return Err(format!( + "{}: local generation contains undeclared file {}", + manifest.spec(), + relative.display() + )); + } + present.insert(relative); + } else { return Err(format!( - "{}: declared {} output {:?} is an empty directory and cannot round-trip through an artifact archive", - target.spec(), - label, - rel + "{}: local generation contains a special filesystem entry: {}", + manifest.spec(), + path.display() )); } } - Ok(path) + Ok(()) } -/// Validate every reachable leaf below a declared artifact directory. -/// Internal symlinks are allowed because several library packages publish -/// compatibility aliases; external/cyclic links and special files are not. -fn validate_artifact_tree( - canonical_root: &Path, - path: &Path, - active_dirs: &mut BTreeSet, -) -> Result { - let link_metadata = std::fs::symlink_metadata(path) - .map_err(|e| format!("stat package artifact {}: {e}", path.display()))?; - let resolved = std::fs::canonicalize(path) - .map_err(|e| format!("resolve package artifact {}: {e}", path.display()))?; - if !resolved.starts_with(canonical_root) { - return Err(format!( - "package artifact {} resolves outside {}", - path.display(), - canonical_root.display() - )); - } - let metadata = if link_metadata.file_type().is_symlink() { - std::fs::metadata(path) - .map_err(|e| format!("follow package artifact symlink {}: {e}", path.display()))? - } else { - link_metadata - }; - if metadata.is_file() { - return Ok(1); +fn files_equal(left: &Path, right: &Path) -> Result { + let left_metadata = std::fs::metadata(left) + .map_err(|e| format!("stat file {} for byte comparison: {e}", left.display()))?; + let right_metadata = std::fs::metadata(right) + .map_err(|e| format!("stat file {} for byte comparison: {e}", right.display()))?; + if left_metadata.len() != right_metadata.len() { + return Ok(false); + } + let mut left_file = std::io::BufReader::new( + std::fs::File::open(left) + .map_err(|e| format!("open file {} for byte comparison: {e}", left.display()))?, + ); + let mut right_file = std::io::BufReader::new( + std::fs::File::open(right) + .map_err(|e| format!("open file {} for byte comparison: {e}", right.display()))?, + ); + let mut left_buffer = [0u8; 64 * 1024]; + let mut right_buffer = [0u8; 64 * 1024]; + loop { + let left_read = std::io::Read::read(&mut left_file, &mut left_buffer) + .map_err(|e| format!("read file {} for byte comparison: {e}", left.display()))?; + let right_read = std::io::Read::read(&mut right_file, &mut right_buffer) + .map_err(|e| format!("read file {} for byte comparison: {e}", right.display()))?; + if left_read != right_read || left_buffer[..left_read] != right_buffer[..right_read] { + return Ok(false); + } + if left_read == 0 { + return Ok(true); + } } - if !metadata.is_dir() { - return Err(format!( - "package artifact {} is not a regular file, directory, or contained symlink", - path.display() - )); +} + +fn package_mirror_matches_plan(plan: &PackageClosureMirrorPlan) -> Result { + Ok(path_entry_exists(&plan.package_dir)? + && read_package_mirror_links(&plan.package_dir) + .map(|links| links == plan.expected_links()) + .unwrap_or(false)) +} + +fn scalar_mirror_matches_target(destination: &Path, target: &Path) -> Result { + match std::fs::symlink_metadata(destination) { + Ok(metadata) if metadata.file_type().is_symlink() => std::fs::read_link(destination) + .map(|actual| actual == target) + .map_err(|e| format!("read scalar mirror symlink {}: {e}", destination.display(),)), + Ok(_) => Ok(false), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(e) => Err(format!( + "inspect scalar mirror {}: {e}", + destination.display(), + )), } - if !active_dirs.insert(resolved.clone()) { - return Err(format!( - "package artifact directory symlink cycle reaches {}", - path.display() - )); +} + +fn publication_claim_exists(marker: &Path) -> Result { + match std::fs::symlink_metadata(marker) { + Ok(metadata) if metadata.is_file() && !metadata.file_type().is_symlink() => Ok(true), + Ok(_) => Err(format!( + "local generation publication claim must be a regular non-symlink file: {}", + marker.display() + )), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(e) => Err(format!( + "inspect local generation publication claim {}: {e}", + marker.display() + )), } - let mut entries = std::fs::read_dir(path) - .map_err(|e| format!("read package artifact directory {}: {e}", path.display()))? - .collect::, _>>() - .map_err(|e| format!("read package artifact directory {}: {e}", path.display()))?; - entries.sort_by_key(|entry| entry.path()); - let mut leaves = 0usize; - for entry in entries { - leaves += validate_artifact_tree(canonical_root, &entry.path(), active_dirs)?; +} + +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +enum PublicationClaim { + Created, + Existing, +} + +fn claim_local_generation_publication(marker: &Path) -> Result { + match std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(marker) + { + Ok(file) => { + file.sync_all().map_err(|e| { + format!( + "sync local generation publication claim {}: {e}", + marker.display() + ) + })?; + Ok(PublicationClaim::Created) + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { + publication_claim_exists(marker)?; + Ok(PublicationClaim::Existing) + } + Err(e) => Err(format!( + "create local generation publication claim {}: {e}", + marker.display() + )), } - active_dirs.remove(&resolved); - Ok(leaves) } -fn validate_cache_entry( - target: &DepsManifest, - dir: &Path, - arch: TargetArch, - abi_version: u32, - cache_key_sha: &str, +fn replace_mirror_symlink_no_follow( + manifest: &DepsManifest, + target: &Path, + destination: &Path, ) -> Result<(), String> { - validate_cache_artifacts(target, dir)?; - validate_cache_provenance(target, dir, arch, abi_version, cache_key_sha) + let mut transaction = LocalFileTransaction::prepare_symlink(manifest, target, destination)?; + let mut rename = |from: &Path, to: &Path| std::fs::rename(from, to); + transaction.move_existing_aside_with(manifest, &mut rename)?; + transaction.publish_with(manifest, &mut rename)?; + transaction.finish() } -fn remove_cache_entry(canonical: &Path, cache_key_sha: &str) -> Result<(), String> { - match std::fs::symlink_metadata(canonical) { - Ok(metadata) if metadata.file_type().is_symlink() || metadata.is_file() => { - std::fs::remove_file(canonical) - .map_err(|e| format!("remove stale cache path {}: {e}", canonical.display()))?; +#[cfg(any(target_vendor = "apple", target_os = "linux", target_os = "android"))] +fn rename_entry_no_replace(from: &Path, to: &Path) -> std::io::Result<()> { + rustix::fs::renameat_with( + rustix::fs::CWD, + from, + rustix::fs::CWD, + to, + rustix::fs::RenameFlags::NOREPLACE, + ) + .map_err(Into::into) +} + +#[cfg(windows)] +fn rename_entry_no_replace(from: &Path, to: &Path) -> std::io::Result<()> { + use std::os::windows::ffi::OsStrExt; + + #[link(name = "Kernel32")] + unsafe extern "system" { + fn MoveFileW(existing: *const u16, new: *const u16) -> i32; + } + + let from: Vec = from.as_os_str().encode_wide().chain(Some(0)).collect(); + let to: Vec = to.as_os_str().encode_wide().chain(Some(0)).collect(); + // SAFETY: both pointers reference NUL-terminated UTF-16 buffers for the + // duration of the call. MoveFileW omits MOVEFILE_REPLACE_EXISTING, so it + // fails atomically when the destination already exists. + if unsafe { MoveFileW(from.as_ptr(), to.as_ptr()) } == 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } +} + +#[cfg(not(any( + target_vendor = "apple", + target_os = "linux", + target_os = "android", + windows +)))] +fn rename_entry_no_replace(_from: &Path, _to: &Path) -> std::io::Result<()> { + Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "this host does not provide atomic no-replace rename", + )) +} + +#[derive(Clone, Debug, Eq, PartialEq)] +enum LocalMirrorEntryKind { + Regular { len: u64, sha256: [u8; 32] }, + Symlink { target: PathBuf }, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct LocalMirrorEntrySnapshot { + identity: PackageMirrorIdentity, + kind: LocalMirrorEntryKind, +} + +struct LocalFileTransaction { + destination: PathBuf, + transaction_root: PathBuf, + stage: PathBuf, + backup: PathBuf, + stage_snapshot: LocalMirrorEntrySnapshot, + backup_snapshot: Option, + old_moved: bool, + published: bool, + yielded_to_other_writer: bool, + finished: bool, + allow_existing_regular: bool, +} + +impl LocalFileTransaction { + #[cfg(test)] + fn prepare( + manifest: &DepsManifest, + source: &Path, + destination: &Path, + fork_instrumentation: ForkInstrumentationPolicy, + required_exports: &[&str], + ) -> Result { + let parent = destination.parent().ok_or_else(|| { + format!( + "{}: local mirror path has no parent: {}", + manifest.spec(), + destination.display() + ) + })?; + let parent = canonical_real_directory(parent, "local artifact mirror parent")?; + let file_name = destination.file_name().ok_or_else(|| { + format!( + "{}: local mirror path has no filename: {}", + manifest.spec(), + destination.display() + ) + })?; + let destination = parent.join(file_name); + let (transaction_root, stage, backup, mut stage_file, stage_identity) = + reserve_local_file_transaction(&parent, file_name)?; + let prepared = (|| { + let source_before = std::fs::symlink_metadata(source) + .map_err(|e| format!("inspect local artifact source {}: {e}", source.display()))?; + if !source_before.is_file() || source_before.file_type().is_symlink() { + return Err(format!( + "local artifact source must remain a regular non-symlink file: {}", + source.display() + )); + } + let source_identity = package_mirror_identity(&source_before)?; + let mut source_file = std::fs::File::open(source) + .map_err(|e| format!("open local artifact source {}: {e}", source.display()))?; + if package_mirror_identity(&source_file.metadata().map_err(|e| { + format!( + "inspect opened local artifact source {}: {e}", + source.display() + ) + })?)? + != source_identity + { + return Err(format!( + "local artifact source changed before it was copied: {}", + source.display() + )); + } + std::io::copy(&mut source_file, &mut stage_file).map_err(|e| { + format!( + "copy local artifact {} into private mirror stage {}: {e}", + source.display(), + stage.display() + ) + })?; + stage_file + .sync_all() + .map_err(|e| format!("sync private mirror stage {}: {e}", stage.display()))?; + std::fs::set_permissions( + &stage, + std::fs::symlink_metadata(source) + .map_err(|e| { + format!("inspect local artifact source {}: {e}", source.display()) + })? + .permissions(), + ) + .map_err(|e| { + format!( + "set private mirror stage permissions {}: {e}", + stage.display() + ) + })?; + if files_equal(source, &stage)? { + validate_wasm_artifact_policy(&stage, fork_instrumentation, required_exports) + } else { + Err(format!( + "local artifact source changed while it was copied: {}", + source.display() + )) + } + })(); + drop(stage_file); + if let Err(error) = prepared { + let cleanup = cleanup_reserved_local_stage(&transaction_root, &stage, &stage_identity); + return match cleanup { + Ok(()) => Err(error), + Err(cleanup_error) => Err(format!( + "{error}; additionally failed to clean private local-file transaction: {cleanup_error}" + )), + }; } - Ok(metadata) if metadata.is_dir() => { - std::fs::remove_dir_all(canonical) - .map_err(|e| format!("remove stale cache entry {}: {e}", canonical.display()))?; + let stage_snapshot = inspect_local_mirror_entry(&stage)?; + Ok(Self { + destination, + transaction_root, + stage, + backup, + stage_snapshot, + backup_snapshot: None, + old_moved: false, + published: false, + yielded_to_other_writer: false, + finished: false, + allow_existing_regular: true, + }) + } + + fn prepare_symlink( + manifest: &DepsManifest, + target: &Path, + destination: &Path, + ) -> Result { + let target_metadata = std::fs::symlink_metadata(target).map_err(|e| { + format!( + "{}: inspect scalar mirror target {}: {e}", + manifest.spec(), + target.display(), + ) + })?; + if !target.is_absolute() + || !target_metadata.is_file() + || target_metadata.file_type().is_symlink() + { + return Err(format!( + "{}: scalar mirror target must be an absolute regular non-symlink file: {}", + manifest.spec(), + target.display(), + )); } - Ok(_) => { + let target = std::fs::canonicalize(target).map_err(|e| { + format!( + "{}: canonicalize scalar mirror target {}: {e}", + manifest.spec(), + target.display(), + ) + })?; + let parent = destination.parent().ok_or_else(|| { + format!( + "{}: scalar mirror path has no parent: {}", + manifest.spec(), + destination.display(), + ) + })?; + let parent = canonical_real_directory(parent, "scalar mirror parent")?; + let file_name = destination.file_name().ok_or_else(|| { + format!( + "{}: scalar mirror path has no filename: {}", + manifest.spec(), + destination.display(), + ) + })?; + let destination = parent.join(file_name); + let (transaction_root, stage, backup) = + reserve_local_symlink_transaction(&parent, file_name)?; + if let Err(error) = symlink_file(&target, &stage) { + let _ = std::fs::remove_dir(&transaction_root); return Err(format!( - "refusing to remove special cache path {}", - canonical.display() + "{}: create private scalar mirror symlink {} -> {}: {error}", + manifest.spec(), + stage.display(), + target.display(), )); } - Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} - Err(e) => return Err(format!("inspect cache path {}: {e}", canonical.display())), + let stage_snapshot = match inspect_local_mirror_entry(&stage) { + Ok(snapshot) => snapshot, + Err(error) => { + let cleanup = std::fs::symlink_metadata(&stage) + .ok() + .filter(|metadata| metadata.file_type().is_symlink()) + .and_then(|_| { + (std::fs::read_link(&stage).ok().as_deref() == Some(target.as_path())) + .then(|| std::fs::remove_file(&stage)) + }); + if let Some(result) = cleanup { + let _ = result; + let _ = std::fs::remove_dir(&transaction_root); + } + return Err(format!( + "{}: inspect private scalar mirror symlink {}: {error}", + manifest.spec(), + stage.display(), + )); + } + }; + Ok(Self { + destination, + transaction_root, + stage, + backup, + stage_snapshot, + backup_snapshot: None, + old_moved: false, + published: false, + yielded_to_other_writer: false, + finished: false, + allow_existing_regular: false, + }) } - remove_cache_provenance(canonical, cache_key_sha) -} -pub(crate) fn validate_cache_artifacts(target: &DepsManifest, dir: &Path) -> Result<(), String> { - match target.kind { - ManifestKind::Library => { - for rel in &target.outputs.libs { - validate_declared_artifact( - target, - dir, - rel, - "libs", - "missing from cache entry", - true, - )?; + fn move_existing_aside_with( + &mut self, + manifest: &DepsManifest, + rename: &mut F, + ) -> Result<(), String> + where + F: FnMut(&Path, &Path) -> std::io::Result<()>, + { + let mut restore = |from: &Path, to: &Path| rename_entry_no_replace(from, to); + self.move_existing_aside_with_restore(manifest, rename, &mut restore) + } + + fn move_existing_aside_with_restore( + &mut self, + manifest: &DepsManifest, + rename: &mut F, + restore: &mut R, + ) -> Result<(), String> + where + F: FnMut(&Path, &Path) -> std::io::Result<()>, + R: FnMut(&Path, &Path) -> std::io::Result<()>, + { + match std::fs::symlink_metadata(&self.destination) { + Ok(_) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(e) => { + return Err(format!( + "{}: inspect existing local mirror {}: {e}", + manifest.spec(), + self.destination.display() + )); } - for rel in &target.outputs.headers { - validate_declared_artifact( - target, - dir, - rel, - "headers", - "missing from cache entry", - false, - )?; + } + let live_snapshot = inspect_local_mirror_entry(&self.destination).map_err(|e| { + format!( + "{}: refusing to replace non-file or unstable local mirror {}: {e}", + manifest.spec(), + self.destination.display() + ) + })?; + if !self.allow_existing_regular + && matches!(&live_snapshot.kind, LocalMirrorEntryKind::Regular { .. }) + { + return Err(format!( + "{}: refusing to replace regular file at scalar mirror {}", + manifest.spec(), + self.destination.display(), + )); + } + rename(&self.destination, &self.backup).map_err(|e| { + format!( + "{}: move existing local mirror {} aside without following it: {e}", + manifest.spec(), + self.destination.display() + ) + })?; + self.old_moved = true; + match inspect_local_mirror_entry(&self.backup) { + Ok(backup_snapshot) if backup_snapshot == live_snapshot => { + self.backup_snapshot = Some(backup_snapshot); + Ok(()) } - for rel in &target.outputs.pkgconfig { - validate_declared_artifact( - target, - dir, - rel, - "pkgconfig", - "missing from cache entry", - true, - )?; + validation => { + let detail = match validation { + Ok(_) => "entry identity or contents changed during quarantine".to_string(), + Err(e) => e, + }; + match restore(&self.backup, &self.destination) { + Ok(()) => { + self.old_moved = false; + return Err(format!( + "{}: local mirror ownership changed during quarantine; restored {} without replacing another writer and refused publication: {detail}", + manifest.spec(), + self.destination.display() + )); + } + Err(restore_error) + if restore_error.kind() == std::io::ErrorKind::AlreadyExists => + { + return Err(format!( + "{}: local mirror ownership changed during quarantine; a concurrent entry at {} was left intact and the displaced entry was preserved at {}: {detail}", + manifest.spec(), + self.destination.display(), + self.backup.display(), + )); + } + Err(restore_error) => { + return Err(format!( + "{}: local mirror ownership changed during quarantine; preserved the displaced entry at {} after no-replace restore failed: {restore_error}: {detail}", + manifest.spec(), + self.backup.display(), + )); + } + } } - for rel in &target.outputs.files { - validate_declared_artifact( - target, - dir, - rel, - "files", - "missing from cache entry", - true, - )?; + } + } + + fn publish_with(&mut self, manifest: &DepsManifest, rename: &mut F) -> Result<(), String> + where + F: FnMut(&Path, &Path) -> std::io::Result<()>, + { + let stage_kind = self.stage_snapshot.kind.clone(); + let mut publish = |stage: &Path, destination: &Path| match &stage_kind { + LocalMirrorEntryKind::Regular { .. } => std::fs::hard_link(stage, destination), + LocalMirrorEntryKind::Symlink { target } => symlink_file(target, destination), + }; + self.publish_with_operation(manifest, rename, &mut publish) + } + + fn publish_with_operation( + &mut self, + manifest: &DepsManifest, + rename: &mut F, + publish: &mut P, + ) -> Result<(), String> + where + F: FnMut(&Path, &Path) -> std::io::Result<()>, + P: FnMut(&Path, &Path) -> std::io::Result<()>, + { + let mut restore = |from: &Path, to: &Path| rename_entry_no_replace(from, to); + self.publish_with_operations(manifest, rename, publish, &mut restore) + } + + fn publish_with_operations( + &mut self, + manifest: &DepsManifest, + _rename: &mut F, + publish: &mut P, + restore: &mut R, + ) -> Result<(), String> + where + F: FnMut(&Path, &Path) -> std::io::Result<()>, + P: FnMut(&Path, &Path) -> std::io::Result<()>, + R: FnMut(&Path, &Path) -> std::io::Result<()>, + { + validate_local_mirror_entry(&self.stage, &self.stage_snapshot).map_err(|e| { + format!( + "{}: staged local mirror changed before publication: {e}", + manifest.spec() + ) + })?; + let publish_result = publish(&self.stage, &self.destination); + if let Err(publish_error) = publish_result { + match path_entry_exists(&self.destination) { + Ok(true) => { + self.yielded_to_other_writer = true; + let cleanup_error = self.cleanup_private_paths().err(); + let mut message = format!( + "{}: publish local mirror {} failed ({publish_error}); another writer installed an entry, which was left intact", + manifest.spec(), + self.destination.display() + ); + if let Some(cleanup_error) = cleanup_error { + message.push_str(&format!( + "; private transaction cleanup also failed: {cleanup_error}" + )); + } + return Err(message); + } + Ok(false) => {} + Err(inspect_error) => { + return Err(format!( + "{}: publish local mirror {} failed ({publish_error}); {inspect_error}; private quarantine was preserved", + manifest.spec(), + self.destination.display() + )); + } + } + + if self.old_moved { + let backup_snapshot = self.backup_snapshot.as_ref().ok_or_else(|| { + format!( + "{}: refusing to restore unvalidated local mirror quarantine {}", + manifest.spec(), + self.backup.display() + ) + })?; + validate_local_mirror_entry(&self.backup, backup_snapshot)?; + match restore(&self.backup, &self.destination) { + Ok(()) => { + self.old_moved = false; + self.backup_snapshot = None; + } + Err(restore_error) + if restore_error.kind() == std::io::ErrorKind::AlreadyExists => + { + self.yielded_to_other_writer = true; + let cleanup_error = self.cleanup_private_paths().err(); + let mut message = format!( + "{}: publish local mirror {} failed ({publish_error}); a concurrent writer won before rollback and was left intact", + manifest.spec(), + self.destination.display(), + ); + if let Some(cleanup_error) = cleanup_error { + message.push_str(&format!( + "; private transaction cleanup also failed: {cleanup_error}" + )); + } + return Err(message); + } + Err(restore_error) => { + return Err(format!( + "{}: publish local mirror {} failed ({publish_error}); no-replace restore of previous mirror from {} failed: {restore_error}", + manifest.spec(), + self.destination.display(), + self.backup.display(), + )); + } + } } + return Err(format!( + "{}: publish local mirror {} failed: {publish_error}", + manifest.spec(), + self.destination.display() + )); } - ManifestKind::Program => { - for out in &target.program_outputs { - let path = validate_declared_artifact( - target, - dir, - &out.wasm, - "wasm", - "missing from cache entry", - true, - )?; - validate_wasm_artifact_policy( - &path, - out.fork_instrumentation, - required_exports_for_program_output(target, out), - )?; + self.published = true; + Ok(()) + } + + fn finish(mut self) -> Result<(), String> { + self.cleanup_private_paths()?; + self.finished = true; + Ok(()) + } + + fn cleanup_private_paths(&mut self) -> Result<(), String> { + let mut failures = Vec::new(); + if let Err(e) = remove_validated_local_transaction_entry( + &self.stage, + &self.stage_snapshot, + "staged local mirror", + ) { + failures.push(e); + } + match &self.backup_snapshot { + Some(snapshot) => { + if let Err(e) = remove_validated_local_transaction_entry( + &self.backup, + snapshot, + "quarantined previous local mirror", + ) { + failures.push(e); + } + } + None => match path_entry_exists(&self.backup) { + Ok(true) => failures.push(format!( + "refusing to remove unvalidated local mirror quarantine {}", + self.backup.display() + )), + Ok(false) => {} + Err(e) => failures.push(e), + }, + } + if failures.is_empty() { + match std::fs::remove_dir(&self.transaction_root) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => failures.push(format!( + "remove empty local mirror transaction {}: {e}", + self.transaction_root.display() + )), + } + } + if failures.is_empty() { + self.backup_snapshot = None; + self.old_moved = false; + Ok(()) + } else { + Err(failures.join("; ")) + } + } + + fn restore_unpublished_backup_with(&mut self, restore: &mut R) + where + R: FnMut(&Path, &Path) -> std::io::Result<()>, + { + if self.published + || self.yielded_to_other_writer + || !self.old_moved + || self.backup_snapshot.as_ref().is_none_or(|snapshot| { + validate_local_mirror_entry(&self.backup, snapshot).is_err() + }) + { + return; + } + match restore(&self.backup, &self.destination) { + Ok(()) => { + self.old_moved = false; + self.backup_snapshot = None; } - for runtime_file in &target.runtime_files { - validate_declared_artifact( - target, - dir, - &runtime_file.artifact, - "runtime file", - "missing from cache entry", - true, - )?; + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { + self.yielded_to_other_writer = true; } + Err(_) => {} } - ManifestKind::Source => {} } - Ok(()) } -fn validate_outputs(target: &DepsManifest, out_dir: &Path) -> Result<(), String> { - match target.kind { - ManifestKind::Library => { - for rel in &target.outputs.libs { - validate_declared_artifact( - target, - out_dir, - rel, - "libs", - "not produced by build script", - true, - )?; - } - for rel in &target.outputs.headers { - validate_declared_artifact( - target, - out_dir, - rel, - "headers", - "not produced by build script", - false, - )?; - } - for rel in &target.outputs.pkgconfig { - validate_declared_artifact( - target, - out_dir, - rel, - "pkgconfig", - "not produced by build script", - true, - )?; +impl Drop for LocalFileTransaction { + fn drop(&mut self) { + if self.finished { + return; + } + let mut restore = |from: &Path, to: &Path| rename_entry_no_replace(from, to); + self.restore_unpublished_backup_with(&mut restore); + let _ = remove_validated_local_transaction_entry( + &self.stage, + &self.stage_snapshot, + "staged local mirror", + ); + if (self.published || self.yielded_to_other_writer || !self.old_moved) + && self.backup_snapshot.is_some() + { + let _ = remove_validated_local_transaction_entry( + &self.backup, + self.backup_snapshot.as_ref().unwrap(), + "quarantined previous local mirror", + ); + } + let _ = std::fs::remove_dir(&self.transaction_root); + } +} + +fn reserve_local_symlink_transaction( + parent: &Path, + file_name: &std::ffi::OsStr, +) -> Result<(PathBuf, PathBuf, PathBuf), String> { + for _ in 0..1024 { + let sequence = MIRROR_TRANSACTION_COUNTER.fetch_add(1, Ordering::Relaxed); + let transaction = format!("{}-{sequence}", std::process::id()); + let file_name = file_name.to_string_lossy(); + let transaction_root = + parent.join(format!(".{file_name}.symlink-transaction-{transaction}")); + match create_private_transaction_directory(&transaction_root) { + Ok(()) => { + return Ok(( + transaction_root.clone(), + transaction_root.join("stage"), + transaction_root.join("backup"), + )); } - for rel in &target.outputs.files { - validate_declared_artifact( - target, - out_dir, - rel, - "files", - "not produced by build script", - true, - )?; + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(e) => { + return Err(format!( + "reserve private scalar-symlink transaction {}: {e}", + transaction_root.display(), + )); } } - ManifestKind::Program => { - for out in &target.program_outputs { - let p = validate_declared_artifact( - target, - out_dir, - &out.wasm, - "wasm", - "not produced by build script", - true, - )?; - validate_wasm_artifact_policy( - &p, - out.fork_instrumentation, - required_exports_for_program_output(target, out), - )?; + } + Err(format!( + "could not allocate a unique scalar-symlink transaction below {}", + parent.display(), + )) +} + +#[cfg(test)] +fn reserve_local_file_transaction( + parent: &Path, + file_name: &std::ffi::OsStr, +) -> Result< + ( + PathBuf, + PathBuf, + PathBuf, + std::fs::File, + PackageMirrorIdentity, + ), + String, +> { + for _ in 0..1024 { + let sequence = MIRROR_TRANSACTION_COUNTER.fetch_add(1, Ordering::Relaxed); + let transaction = format!("{}-{sequence}", std::process::id()); + let file_name = file_name.to_string_lossy(); + let transaction_root = parent.join(format!(".{file_name}.local-transaction-{transaction}")); + match create_private_transaction_directory(&transaction_root) { + Ok(()) => { + let stage = transaction_root.join("stage"); + let backup = transaction_root.join("backup"); + match std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&stage) + { + Ok(file) => { + let metadata = std::fs::symlink_metadata(&stage).map_err(|e| { + format!("inspect private local file stage {}: {e}", stage.display()) + })?; + let identity = package_mirror_identity(&metadata)?; + return Ok((transaction_root, stage, backup, file, identity)); + } + Err(e) => { + let _ = std::fs::remove_dir(&transaction_root); + return Err(format!( + "create private local file stage {}: {e}", + stage.display() + )); + } + } } - for runtime_file in &target.runtime_files { - validate_declared_artifact( - target, - out_dir, - &runtime_file.artifact, - "runtime file", - "not produced by build script", - true, - )?; + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(e) => { + return Err(format!( + "reserve private local file transaction {}: {e}", + transaction_root.display() + )); } } - // No outputs to validate for source-kind (Chunk C). - ManifestKind::Source => return Ok(()), } - Ok(()) + Err(format!( + "could not allocate a unique local file transaction below {}", + parent.display() + )) } -/// Source-kind validation: the override script must have populated -/// `OUT_DIR` with *something*. Source manifests have no declared -/// outputs list (Task C.1 rejects `[outputs]` for source kind), so -/// non-emptiness is the only signal we have that the script did -/// useful work — an empty dir after a successful `bash` exit almost -/// always means the script forgot to write to `$WASM_POSIX_DEP_OUT_DIR` -/// (e.g. wrote to its own working dir, or hard-coded a path). -fn validate_source_dir_nonempty(out_dir: &Path) -> Result<(), String> { - let mut iter = - std::fs::read_dir(out_dir).map_err(|e| format!("read_dir {}: {e}", out_dir.display()))?; - if iter.next().is_none() { +fn inspect_local_mirror_entry(path: &Path) -> Result { + let before = std::fs::symlink_metadata(path) + .map_err(|e| format!("inspect local mirror entry {}: {e}", path.display()))?; + let identity = package_mirror_identity(&before)?; + let kind = if before.file_type().is_symlink() { + LocalMirrorEntryKind::Symlink { + target: std::fs::read_link(path) + .map_err(|e| format!("read local mirror symlink {}: {e}", path.display()))?, + } + } else if before.is_file() { + let mut file = std::fs::File::open(path) + .map_err(|e| format!("open local mirror entry {}: {e}", path.display()))?; + let opened = file + .metadata() + .map_err(|e| format!("inspect opened local mirror entry {}: {e}", path.display()))?; + if !opened.is_file() || package_mirror_identity(&opened)? != identity { + return Err(format!( + "local mirror entry changed before its contents were read: {}", + path.display() + )); + } + let mut hasher = Sha256::new(); + std::io::copy(&mut file, &mut hasher) + .map_err(|e| format!("hash local mirror entry {}: {e}", path.display()))?; + let hashed = file.metadata().map_err(|e| { + format!( + "reinspect opened local mirror entry {}: {e}", + path.display() + ) + })?; + if package_mirror_identity(&hashed)? != identity || hashed.len() != opened.len() { + return Err(format!( + "local mirror entry changed while its contents were read: {}", + path.display() + )); + } + LocalMirrorEntryKind::Regular { + len: opened.len(), + sha256: hasher.finalize().into(), + } + } else { return Err(format!( - "source build script left OUT_DIR empty at {}; \ - scripts MUST populate $WASM_POSIX_DEP_OUT_DIR with at \ - least one file before exiting", - out_dir.display() + "local mirror entry is not a regular file or symlink: {}", + path.display() + )); + }; + let after = std::fs::symlink_metadata(path) + .map_err(|e| format!("reinspect local mirror entry {}: {e}", path.display()))?; + if package_mirror_identity(&after)? != identity { + return Err(format!( + "local mirror entry identity changed while it was inspected: {}", + path.display() + )); + } + let actual_kind = if after.file_type().is_symlink() { + LocalMirrorEntryKind::Symlink { + target: std::fs::read_link(path) + .map_err(|e| format!("reread local mirror symlink {}: {e}", path.display()))?, + } + } else if after.is_file() { + // The file handle above already proved the exact identity and bytes. + kind.clone() + } else { + return Err(format!( + "local mirror entry type changed while it was inspected: {}", + path.display() + )); + }; + if actual_kind != kind { + return Err(format!( + "local mirror entry contents changed while it was inspected: {}", + path.display() + )); + } + Ok(LocalMirrorEntrySnapshot { identity, kind }) +} + +fn validate_local_mirror_entry( + path: &Path, + expected: &LocalMirrorEntrySnapshot, +) -> Result<(), String> { + let actual = inspect_local_mirror_entry(path)?; + if &actual == expected { + Ok(()) + } else { + Err(format!( + "local mirror identity or contents changed: {}", + path.display() + )) + } +} + +fn remove_validated_local_transaction_entry( + path: &Path, + expected: &LocalMirrorEntrySnapshot, + label: &str, +) -> Result<(), String> { + if !path_entry_exists(path)? { + return Ok(()); + } + validate_local_mirror_entry(path, expected) + .map_err(|e| format!("refusing to remove changed {label} {}: {e}", path.display()))?; + std::fs::remove_file(path) + .map_err(|e| format!("remove validated {label} {}: {e}", path.display())) +} + +#[cfg(test)] +fn cleanup_reserved_local_stage( + transaction_root: &Path, + stage: &Path, + expected_identity: &PackageMirrorIdentity, +) -> Result<(), String> { + let metadata = std::fs::symlink_metadata(stage) + .map_err(|e| format!("inspect reserved local stage {}: {e}", stage.display()))?; + if !metadata.is_file() + || metadata.file_type().is_symlink() + || &package_mirror_identity(&metadata)? != expected_identity + { + return Err(format!( + "refusing to remove changed reserved local stage {}", + stage.display() )); } + std::fs::remove_file(stage) + .map_err(|e| format!("remove reserved local stage {}: {e}", stage.display()))?; + std::fs::remove_dir(transaction_root).map_err(|e| { + format!( + "remove empty local file transaction {}: {e}", + transaction_root.display() + ) + }) +} + +/// Place symlinks under `binaries_dir/programs//` pointing at +/// each declared `[[outputs]]` artifact and `[[runtime_files]]` file in the +/// cache canonical directory. +/// +/// Layout (per arch — wasm32 and wasm64 mirror in parallel): +/// * 1 total output/runtime member: +/// `/programs//.wasm`. +/// * ≥2 total members: +/// `/programs///.wasm`. +/// * first-party kernel/userspace: `/.wasm`. +/// +/// This is the single source of truth for the symlink layout. Browser +/// demos hardcode these paths (see `apps/browser-demos/vite.config.ts` +/// and `host/src/binary-resolver.ts`), so the layout MUST NOT change +/// here without coordinating with the consumer-side import paths. +/// +/// Targets are absolute paths into the resolver cache. Any package with more +/// than one closure member owns one directory below the architecture root, so +/// its complete output/runtime closure is staged and swapped as one directory +/// transaction. One-member and first-party flat layouts retain their +/// historical replace-one-link behavior. +fn place_binaries_symlinks( + m: &DepsManifest, + canonical: &Path, + binaries_dir: &Path, + arch: TargetArch, +) -> Result<(), String> { + let outputs = &m.program_outputs; + if outputs.is_empty() { + return Err(format!("program {:?} has no [[outputs]]", m.name)); + } + let binaries_dir = canonical_real_directory(binaries_dir, "binaries publication root")?; + let programs_root = binaries_dir.join("programs"); + ensure_real_child_directory(&binaries_dir, &programs_root, "program publication root")?; + let arch_root = programs_root.join(arch.as_str()); + ensure_real_child_directory(&programs_root, &arch_root, "architecture publication root")?; + if m.uses_package_mirror_directory() { + let plan = PackageClosureMirrorPlan::validate(m, canonical, &arch_root)?; + if package_mirror_matches_plan(&plan)? { + return Ok(()); + } + return install_package_closure_mirror(plan); + } + + for out in outputs { + let src = canonical.join(&out.wasm); + let source_metadata = std::fs::symlink_metadata(&src).map_err(|e| { + format!( + "declared output {} not found in cache at {}: {e}", + out.wasm, + src.display() + ) + })?; + if !source_metadata.is_file() || source_metadata.file_type().is_symlink() { + return Err(format!( + "declared output {} is not a regular non-symlink cache file at {}", + out.wasm, + src.display() + )); + } + let dest = if m.uses_root_binary_mirror() { + binaries_dir.join(format!("{}.wasm", out.name)) + } else { + arch_root.join(m.output_dest_rel_for(out)) + }; + let dest_dir = dest + .parent() + .ok_or_else(|| format!("dest path {} has no parent", dest.display()))?; + ensure_existing_real_directory(dest_dir, "artifact publication parent")?; + if let Ok(metadata) = std::fs::symlink_metadata(&dest) { + if metadata.file_type().is_symlink() + && std::fs::read_link(&dest).ok().as_deref() == Some(src.as_path()) + { + continue; + } + if metadata.is_dir() && !metadata.file_type().is_symlink() { + return Err(format!( + "refusing to replace artifact publication directory {}", + dest.display() + )); + } + } + replace_mirror_symlink_no_follow(m, &src, &dest)?; + } + for runtime_file in &m.runtime_files { + let src = canonical.join(&runtime_file.artifact); + let metadata = std::fs::symlink_metadata(&src).map_err(|e| { + format!( + "declared runtime file {} not found in cache at {}: {e}", + runtime_file.artifact, + src.display() + ) + })?; + if !metadata.is_file() || metadata.file_type().is_symlink() { + return Err(format!( + "declared runtime file {} is not a regular non-symlink file at {}", + runtime_file.artifact, + src.display() + )); + } + let dest = arch_root.join(m.runtime_file_dest_rel_for(runtime_file)); + let dest_dir = dest + .parent() + .ok_or_else(|| format!("dest path {} has no parent", dest.display()))?; + ensure_existing_real_directory(dest_dir, "runtime-file publication parent")?; + if let Ok(metadata) = std::fs::symlink_metadata(&dest) { + if metadata.file_type().is_symlink() + && std::fs::read_link(&dest).ok().as_deref() == Some(src.as_path()) + { + continue; + } + if metadata.is_dir() && !metadata.file_type().is_symlink() { + return Err(format!( + "refusing to replace runtime-file publication directory {}", + dest.display() + )); + } + } + replace_mirror_symlink_no_follow(m, &src, &dest)?; + } Ok(()) } -/// `libcurl` → `LIBCURL`, `zlib-ng` → `ZLIB_NG`. -fn env_key(name: &str) -> String { - name.chars() - .map(|c| match c { - '-' => '_', - c => c.to_ascii_uppercase(), - }) - .collect() +#[cfg(unix)] +fn symlink_file(src: &Path, dest: &Path) -> std::io::Result<()> { + std::os::unix::fs::symlink(src, dest) } -// --------------------------------------------------------------------- -// Subcommand dispatch -// --------------------------------------------------------------------- +#[cfg(windows)] +fn symlink_file(src: &Path, dest: &Path) -> std::io::Result<()> { + std::os::windows::fs::symlink_file(src, dest) +} -/// Fallback default target architecture when neither `--arch` nor -/// `WASM_POSIX_DEFAULT_ARCH` is set. Wasm32 is the dominant target -/// today; wasm64 is opt-in via flag/env. -/// -/// Kept as a constant (rather than inlined) so tests and callers have -/// a single source of truth, and so future changes — e.g. flipping the -/// default once wasm64 is the dominant target — only have to touch -/// one site. -const DEFAULT_ARCH: TargetArch = TargetArch::Wasm32; +/// One symlink in a staged package-closure directory. +#[derive(Clone, Debug, Eq, PartialEq)] +struct PlannedMirrorLink { + /// Absolute artifact path under the one validated cache identity. + source: PathBuf, + /// Destination relative to `/programs///`. + package_relative: PathBuf, +} -/// Read the current kernel ABI version from `crates/shared`. Resolver -/// uses this as a hash input; ABI bumps therefore auto-invalidate every -/// dependent cache entry without any explicit cache-busting work. -fn current_abi_version() -> u32 { - wasm_posix_shared::ABI_VERSION +/// Fully validated package-closure mirror transaction input. +/// +/// Construction performs every fallible manifest/cache/containment/collision +/// check before the destination tree is created or changed. That ordering is +/// intentional: a missing late runtime file must not leave early output links +/// pointing at a new package identity. +#[derive(Clone, Debug)] +struct PackageClosureMirrorPlan { + package_dir: PathBuf, + links: Vec, } -/// Parse a CLI/env value into `TargetArch`. Accepts `wasm32` and -/// `wasm64`; everything else is rejected with an error message that -/// names the unknown value and lists the valid options. -pub(crate) fn parse_target_arch(s: &str) -> Result { - match s { - "wasm32" => Ok(TargetArch::Wasm32), - "wasm64" => Ok(TargetArch::Wasm64), - other => Err(format!( - "unknown --arch value {other:?}; expected wasm32 or wasm64" - )), +impl PackageClosureMirrorPlan { + fn validate( + manifest: &DepsManifest, + canonical: &Path, + arch_root: &Path, + ) -> Result { + if !matches!(manifest.kind, ManifestKind::Program) { + return Err(format!( + "{}: only program packages can populate the program mirror", + manifest.spec() + )); + } + if !manifest.uses_package_mirror_directory() { + return Err(format!( + "{}: atomic package-directory installation requires more than one declared output/runtime member", + manifest.spec() + )); + } + + // Validate the complete authored closure before even creating the + // architecture root or a staging directory. This covers Wasm policy, + // regular-file requirements, nested runtime files, and containment + // below the supplied cache root. + validate_cache_artifacts(manifest, canonical)?; + let canonical_root = std::fs::canonicalize(canonical).map_err(|e| { + format!( + "{}: resolve canonical cache identity {}: {e}", + manifest.spec(), + canonical.display() + ) + })?; + let canonical_metadata = std::fs::metadata(&canonical_root).map_err(|e| { + format!( + "{}: stat canonical cache identity {}: {e}", + manifest.spec(), + canonical_root.display() + ) + })?; + if !canonical_metadata.is_dir() { + return Err(format!( + "{}: canonical cache identity is not a directory: {}", + manifest.spec(), + canonical_root.display() + )); + } + + let mut links_by_destination: BTreeMap = BTreeMap::new(); + for output in &manifest.program_outputs { + Self::insert_link( + manifest, + &canonical_root, + &output.wasm, + manifest.output_dest_rel_for(output), + &mut links_by_destination, + )?; + } + for runtime_file in &manifest.runtime_files { + Self::insert_link( + manifest, + &canonical_root, + &runtime_file.artifact, + manifest.runtime_file_dest_rel_for(runtime_file), + &mut links_by_destination, + )?; + } + + let expected_count = manifest.program_outputs.len() + manifest.runtime_files.len(); + if links_by_destination.len() != expected_count { + return Err(format!( + "{}: resolver mirror plan contains {} unique destinations for {} declared output/runtime artifacts", + manifest.spec(), + links_by_destination.len(), + expected_count + )); + } + + Ok(Self { + package_dir: arch_root.join(&manifest.name), + links: links_by_destination + .into_iter() + .map(|(package_relative, source)| PlannedMirrorLink { + source, + package_relative, + }) + .collect(), + }) + } + + fn insert_link( + manifest: &DepsManifest, + canonical_root: &Path, + source_artifact: &str, + mirror_relative: PathBuf, + links_by_destination: &mut BTreeMap, + ) -> Result<(), String> { + let package_relative = + package_owned_relative_path(manifest, &mirror_relative).map_err(|e| { + format!( + "{}: invalid resolver mirror destination {} for artifact {:?}: {e}", + manifest.spec(), + mirror_relative.display(), + source_artifact + ) + })?; + let source = canonical_root.join(source_artifact); + let resolved_source = std::fs::canonicalize(&source).map_err(|e| { + format!( + "{}: resolve declared artifact {:?} below canonical cache identity {}: {e}", + manifest.spec(), + source_artifact, + canonical_root.display() + ) + })?; + if !resolved_source.starts_with(canonical_root) { + return Err(format!( + "{}: declared artifact {:?} resolves outside canonical cache identity {}", + manifest.spec(), + source_artifact, + canonical_root.display() + )); + } + if resolved_source != source { + return Err(format!( + "{}: declared artifact {:?} traverses a symlink inside canonical cache identity {}; resolver mirror targets must retain the exact declared artifact suffix", + manifest.spec(), + source_artifact, + canonical_root.display() + )); + } + if let Some(previous) = + links_by_destination.insert(package_relative.clone(), source.clone()) + { + return Err(format!( + "{}: resolver mirror destination {} collides between {} and {}", + manifest.spec(), + mirror_relative.display(), + previous.display(), + source.display() + )); + } + Ok(()) + } + + fn expected_links(&self) -> BTreeMap { + self.links + .iter() + .map(|link| (link.package_relative.clone(), link.source.clone())) + .collect() } } -/// Default target arch for the CLI when no `--arch` is given: -/// 1. `WASM_POSIX_DEFAULT_ARCH` env var, if set and parseable. -/// 2. Fallback to [`DEFAULT_ARCH`]. +/// Strip and validate the package-owned prefix from a resolver mirror path. /// -/// Unparseable env-var values are rejected loudly so a typo doesn't -/// silently fall through to wasm32 (which would be a confusing way to -/// debug "why did my wasm64 build land in the wrong cache slot?"). -fn default_target_arch() -> Result { - match std::env::var("WASM_POSIX_DEFAULT_ARCH") { - Ok(s) => parse_target_arch(&s).map_err(|e| format!("WASM_POSIX_DEFAULT_ARCH: {e}")), - Err(_) => Ok(DEFAULT_ARCH), +/// `Path::strip_prefix` alone is not sufficient: `package/../outside` strips +/// successfully and would escape a staging directory when joined. Requiring +/// normal components makes containment lexical as well as filesystem-checked. +fn package_owned_relative_path( + manifest: &DepsManifest, + mirror_relative: &Path, +) -> Result { + let mut components = mirror_relative.components(); + match components.next() { + Some(Component::Normal(component)) if component == manifest.name.as_str() => {} + _ => { + return Err(format!( + "path must begin with the package directory {:?}", + manifest.name + )); + } + } + + let mut package_relative = PathBuf::new(); + for component in components { + match component { + Component::Normal(component) => package_relative.push(component), + _ => { + return Err( + "path below the package directory must contain only normal components" + .to_string(), + ); + } + } } + if package_relative.as_os_str().is_empty() { + return Err("path must name an artifact below the package directory".to_string()); + } + Ok(package_relative) } -/// Extract `--arch ` / `--arch=` from `args`, leaving -/// non-flag arguments in place. Returns the parsed arch (if any) and -/// the remaining arguments. +static MIRROR_TRANSACTION_COUNTER: AtomicU64 = AtomicU64::new(0); + +/// A prepared two-rename installation of a package-owned mirror directory. /// -/// Hand-rolled rather than pulling in clap; the CLI surface is small -/// and stable. Both forms are accepted and may appear anywhere after -/// the subcommand, so `build-deps path zlib --arch=wasm64`, -/// `build-deps path --arch wasm64 zlib`, and -/// `build-deps --arch=wasm64 path zlib` all work identically. -fn extract_arch_flag(args: Vec) -> Result<(Option, Vec), String> { - let mut arch: Option = None; - let mut rest: Vec = Vec::with_capacity(args.len()); - let mut it = args.into_iter(); - while let Some(a) = it.next() { - if let Some(value) = a.strip_prefix("--arch=") { - if arch.is_some() { - return Err("--arch given more than once".to_string()); +/// A uniquely reserved private directory beside `live_dir` contains the stage +/// and backup. Sibling placement is a correctness requirement: filesystem +/// rename atomicity is only specified within one filesystem/mount. The private +/// parent is mode 0700 on Unix, so cooperating concurrent installers cannot +/// collide with or modify each other's transaction children. We do not depend +/// on rename-over-existing behavior, which differs across POSIX and Windows. +/// Instead the commit boundary is: +/// +/// 1. `live_dir -> backup_dir` (when an old entry exists); +/// 2. `stage_dir -> live_dir`. +/// +/// A pathname reader can therefore see the complete old directory, no live +/// directory in the short interval between renames, or the complete new +/// directory. It cannot see the old and new links mixed in one live directory. +/// +/// The protocol is deliberately lock-free. Before an existing live directory +/// can be moved, and again after it has moved into the private transaction, its +/// filesystem identity and complete symlink map must match. Only snapshots +/// validated inside the private parent are ever removed. If another writer +/// fills the live path between our two renames, we accept it only when its +/// *entire* declared output/runtime link set has our exact canonical targets. +/// A different winner is never removed or overwritten. A process crash can +/// leave an inert private transaction directory; scavenging without a lease +/// could delete another live writer's stage, so it is unsafe here. +#[derive(Clone, Debug, Eq, PartialEq)] +struct PackageMirrorSnapshot { + identity: PackageMirrorIdentity, + links: BTreeMap, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct PackageMirrorIdentity { + first: u64, + second: u64, +} + +struct PackageDirectoryTransaction { + live_dir: PathBuf, + transaction_root: PathBuf, + stage_dir: PathBuf, + backup_dir: PathBuf, + expected_links: BTreeMap, + stage_snapshot: PackageMirrorSnapshot, + backup_snapshot: Option, + old_moved: bool, + committed: bool, + yielded_to_other_writer: bool, + finished: bool, +} + +impl PackageDirectoryTransaction { + fn prepare(plan: PackageClosureMirrorPlan) -> Result { + let parent = plan.package_dir.parent().ok_or_else(|| { + format!( + "package mirror path has no parent: {}", + plan.package_dir.display() + ) + })?; + ensure_existing_real_directory(parent, "package mirror transaction root")?; + + let (transaction_root, stage_dir, backup_dir) = reserve_package_directory_transaction( + parent, + plan.package_dir.file_name().unwrap_or_default(), + )?; + let expected_plan_links = plan.expected_links(); + let staged = (|| { + for link in &plan.links { + let destination = stage_dir.join(&link.package_relative); + let destination_parent = destination.parent().ok_or_else(|| { + format!( + "staged package mirror path has no parent: {}", + destination.display() + ) + })?; + std::fs::create_dir_all(destination_parent).map_err(|e| { + format!( + "mkdir staged package mirror directory {}: {e}", + destination_parent.display() + ) + })?; + symlink_file(&link.source, &destination).map_err(|e| { + format!( + "symlink staged package artifact {} -> {}: {e}", + destination.display(), + link.source.display() + ) + })?; } - arch = Some(parse_target_arch(value)?); - } else if a == "--arch" { - if arch.is_some() { - return Err("--arch given more than once".to_string()); + let staged_snapshot = inspect_package_mirror_snapshot(&stage_dir)?; + let expected_links = expected_plan_links.clone(); + if staged_snapshot.links != expected_links { + return Err(format!( + "staged package mirror {} does not exactly match its validated output/runtime plan", + stage_dir.display() + )); } - let value = it - .next() - .ok_or_else(|| "--arch requires a value (wasm32 or wasm64)".to_string())?; - arch = Some(parse_target_arch(&value)?); + Ok((expected_links, staged_snapshot)) + })(); + + let (expected_links, stage_snapshot) = match staged { + Ok(prepared) => prepared, + Err(e) => { + let cleanup = cleanup_prepared_package_transaction( + &transaction_root, + &stage_dir, + &expected_plan_links, + ); + return match cleanup { + Ok(()) => Err(e), + Err(cleanup_err) => Err(format!( + "{e}; additionally failed to clean staged package mirror: {cleanup_err}" + )), + }; + } + }; + + Ok(Self { + live_dir: plan.package_dir, + transaction_root, + stage_dir, + backup_dir, + expected_links, + stage_snapshot, + backup_snapshot: None, + old_moved: false, + committed: false, + yielded_to_other_writer: false, + finished: false, + }) + } + + fn move_existing_aside_with(&mut self, rename: &mut F) -> Result<(), String> + where + F: FnMut(&Path, &Path) -> std::io::Result<()>, + { + match std::fs::symlink_metadata(&self.live_dir) { + Ok(_) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(e) => { + return Err(format!( + "inspect existing package mirror {}: {e}", + self.live_dir.display() + )); + } + } + let live_snapshot = inspect_package_mirror_snapshot(&self.live_dir).map_err(|e| { + format!( + "refusing to replace package mirror without resolver ownership proof at {}: {e}", + self.live_dir.display() + ) + })?; + rename(&self.live_dir, &self.backup_dir).map_err(|e| { + format!( + "rename existing package mirror {} -> {}: {e}", + self.live_dir.display(), + self.backup_dir.display() + ) + })?; + self.old_moved = true; + match inspect_package_mirror_snapshot(&self.backup_dir) { + Ok(backup_snapshot) if backup_snapshot == live_snapshot => { + self.backup_snapshot = Some(backup_snapshot); + } + validation => { + let detail = match validation { + Ok(_) => "the quarantined entry changed identity or contents during rename" + .to_string(), + Err(e) => e, + }; + if !path_entry_exists(&self.live_dir)? + && rename(&self.backup_dir, &self.live_dir).is_ok() + { + self.old_moved = false; + self.backup_snapshot = None; + return Err(format!( + "package mirror ownership changed during quarantine; restored {} and refused publication: {detail}", + self.live_dir.display() + )); + } + return Err(format!( + "package mirror ownership changed during quarantine; preserved the exact entry at {} and refused publication: {detail}", + self.backup_dir.display() + )); + } + } + Ok(()) + } + + fn publish_with(&mut self, rename: &mut F) -> Result<(), String> + where + F: FnMut(&Path, &Path) -> std::io::Result<()>, + { + match rename(&self.stage_dir, &self.live_dir) { + Ok(()) => { + self.committed = true; + return Ok(()); + } + Err(publish_error) => { + if path_entry_exists(&self.live_dir)? { + let winner_matches = read_package_mirror_links(&self.live_dir) + .map(|links| links == self.expected_links) + .unwrap_or(false); + self.yielded_to_other_writer = true; + let cleanup_error = self.cleanup_private_paths().err(); + if winner_matches { + self.committed = true; + return cleanup_error.map_or(Ok(()), |e| { + Err(format!( + "a concurrent writer installed the requested complete package mirror, but private transaction cleanup failed: {e}" + )) + }); + } + let mut message = format!( + "publish package mirror {} failed ({publish_error}); another writer installed a different or incomplete package directory, which was left intact", + self.live_dir.display() + ); + if let Some(cleanup_error) = cleanup_error { + message.push_str(&format!( + "; private transaction cleanup also failed: {cleanup_error}" + )); + } + return Err(message); + } + + if self.old_moved { + let Some(backup_snapshot) = &self.backup_snapshot else { + return Err(format!( + "publish package mirror {} failed ({publish_error}); refusing to restore an unvalidated quarantine at {}", + self.live_dir.display(), + self.backup_dir.display() + )); + }; + validate_package_mirror_snapshot(&self.backup_dir, backup_snapshot).map_err( + |validation_error| { + format!( + "publish package mirror {} failed ({publish_error}); refusing to restore changed quarantine {}: {validation_error}", + self.live_dir.display(), + self.backup_dir.display() + ) + }, + )?; + match rename(&self.backup_dir, &self.live_dir) { + Ok(()) => { + self.old_moved = false; + self.backup_snapshot = None; + return Err(format!( + "publish package mirror {} failed ({publish_error}); restored the previous complete package directory", + self.live_dir.display() + )); + } + Err(rollback_error) => { + return Err(format!( + "publish package mirror {} failed ({publish_error}); rollback {} -> {} also failed ({rollback_error})", + self.live_dir.display(), + self.backup_dir.display(), + self.live_dir.display() + )); + } + } + } + + Err(format!( + "publish package mirror {} failed: {publish_error}", + self.live_dir.display() + )) + } + } + } + + fn finish(mut self) -> Result<(), String> { + self.cleanup_private_paths()?; + self.finished = true; + Ok(()) + } + + fn cleanup_private_paths(&mut self) -> Result<(), String> { + let mut failures = Vec::new(); + if let Err(e) = remove_validated_package_transaction_tree( + &self.stage_dir, + &self.stage_snapshot, + "staged package mirror", + ) { + failures.push(e); + } + match &self.backup_snapshot { + Some(expected) => { + if let Err(e) = remove_validated_package_transaction_tree( + &self.backup_dir, + expected, + "quarantined previous package mirror", + ) { + failures.push(e); + } + } + None => { + if path_entry_exists(&self.backup_dir)? { + failures.push(format!( + "refusing to remove unvalidated package mirror quarantine {}", + self.backup_dir.display() + )); + } + } + } + if failures.is_empty() { + match std::fs::remove_dir(&self.transaction_root) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => { + failures.push(format!( + "remove empty package mirror transaction {}: {e}", + self.transaction_root.display() + )); + } + } + } + if failures.is_empty() { + self.old_moved = false; + self.backup_snapshot = None; + Ok(()) } else { - rest.push(a); + Err(failures.join("; ")) + } + } +} + +impl Drop for PackageDirectoryTransaction { + fn drop(&mut self) { + if self.finished { + return; + } + + // Normal Rust error paths get deterministic best-effort rollback. + // A process kill cannot run Drop; the live path nevertheless remains + // one of the documented complete-old/absent/complete-new states. + if !self.committed + && !self.yielded_to_other_writer + && self.old_moved + && !path_entry_exists(&self.live_dir).unwrap_or(true) + && self.backup_snapshot.as_ref().is_some_and(|snapshot| { + validate_package_mirror_snapshot(&self.backup_dir, snapshot).is_ok() + }) + && std::fs::rename(&self.backup_dir, &self.live_dir).is_ok() + { + self.old_moved = false; + self.backup_snapshot = None; + } + let _ = remove_validated_package_transaction_tree( + &self.stage_dir, + &self.stage_snapshot, + "staged package mirror", + ); + if (self.committed || self.yielded_to_other_writer || !self.old_moved) + && self.backup_snapshot.is_some() + { + let _ = remove_validated_package_transaction_tree( + &self.backup_dir, + self.backup_snapshot.as_ref().unwrap(), + "quarantined previous package mirror", + ); } + let _ = std::fs::remove_dir(&self.transaction_root); } - Ok((arch, rest)) } -/// Extract `--binaries-dir ` / `--binaries-dir=` from -/// `args`, leaving non-flag arguments in place. Mirrors -/// [`extract_arch_flag`]'s shape so `resolve --binaries-dir

` and -/// `--binaries-dir=

resolve` are equivalent. Only meaningful for the -/// `resolve` subcommand: when supplied, the resolver places -/// `/programs///.wasm` symlinks at -/// each declared `[[outputs]]` (see `place_binaries_symlinks`). Other -/// subcommands ignore the value. -fn extract_binaries_dir_flag(args: Vec) -> Result<(Option, Vec), String> { - let mut binaries_dir: Option = None; - let mut rest: Vec = Vec::with_capacity(args.len()); - let mut it = args.into_iter(); - while let Some(a) = it.next() { - if let Some(value) = a.strip_prefix("--binaries-dir=") { - if binaries_dir.is_some() { - return Err("--binaries-dir given more than once".to_string()); +fn install_package_closure_mirror(plan: PackageClosureMirrorPlan) -> Result<(), String> { + let mut transaction = PackageDirectoryTransaction::prepare(plan)?; + let mut rename = |from: &Path, to: &Path| std::fs::rename(from, to); + transaction.move_existing_aside_with(&mut rename)?; + transaction.publish_with(&mut rename)?; + transaction.finish() +} + +fn reserve_package_directory_transaction( + parent: &Path, + package_name: &std::ffi::OsStr, +) -> Result<(PathBuf, PathBuf, PathBuf), String> { + for _ in 0..1024 { + let sequence = MIRROR_TRANSACTION_COUNTER.fetch_add(1, Ordering::Relaxed); + let transaction = format!("{}-{sequence}", std::process::id()); + let package_name = package_name.to_string_lossy(); + let transaction_root = parent.join(format!(".{package_name}.transaction-{transaction}")); + match create_private_transaction_directory(&transaction_root) { + Ok(()) => { + let stage = transaction_root.join("stage"); + let backup = transaction_root.join("backup"); + if let Err(e) = std::fs::create_dir(&stage) { + let cleanup = std::fs::remove_dir(&transaction_root); + return Err(match cleanup { + Ok(()) => { + format!("create staged package mirror {}: {e}", stage.display()) + } + Err(cleanup_error) => format!( + "create staged package mirror {}: {e}; remove empty reservation: {cleanup_error}", + stage.display() + ), + }); + } + return Ok((transaction_root, stage, backup)); } - binaries_dir = Some(PathBuf::from(value)); - } else if a == "--binaries-dir" { - if binaries_dir.is_some() { - return Err("--binaries-dir given more than once".to_string()); + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(e) => { + return Err(format!( + "reserve package mirror transaction {}: {e}", + transaction_root.display() + )); } - let value = it - .next() - .ok_or_else(|| "--binaries-dir requires a directory path".to_string())?; - binaries_dir = Some(PathBuf::from(value)); - } else { - rest.push(a); } } - Ok((binaries_dir, rest)) + Err(format!( + "could not allocate a unique package mirror transaction below {}", + parent.display() + )) } -/// Extract `--fetch-only` from `args`, leaving non-flag arguments in place. -/// Only meaningful for `resolve`: it turns archive/source mismatches into -/// errors instead of running package build scripts. -fn extract_fetch_only_flag(args: Vec) -> (bool, Vec) { - let mut fetch_only = false; - let mut rest: Vec = Vec::with_capacity(args.len()); - for a in args { - if a == "--fetch-only" { - fetch_only = true; - } else { - rest.push(a); - } +fn path_entry_exists(path: &Path) -> Result { + match std::fs::symlink_metadata(path) { + Ok(_) => Ok(true), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(e) => Err(format!("inspect path {}: {e}", path.display())), } - (fetch_only, rest) } -pub fn run(args: Vec) -> Result<(), String> { - let (arch_flag, rest) = extract_arch_flag(args)?; - let arch = match arch_flag { - Some(a) => a, - None => default_target_arch()?, - }; - // `--binaries-dir` is `resolve`-only today, but pulling it out at - // this layer (rather than inside the `resolve` arm) keeps the flag - // location-independent: `resolve --binaries-dir x foo` and - // `--binaries-dir x resolve foo` both work, matching `--arch`'s - // shape. - let (binaries_dir, rest) = extract_binaries_dir_flag(rest)?; - let (fetch_only, rest) = extract_fetch_only_flag(rest); +#[cfg(unix)] +fn package_mirror_identity(metadata: &std::fs::Metadata) -> Result { + use std::os::unix::fs::MetadataExt; + Ok(PackageMirrorIdentity { + first: metadata.dev(), + second: metadata.ino(), + }) +} - let mut it = rest.into_iter(); - let sub = it.next().ok_or( - "usage: xtask build-deps [--arch=wasm32|wasm64] [--binaries-dir ] [--fetch-only] \ - \ - [ []]", - )?; - let target = it.next(); - // Output metadata subcommands take a second positional arg (the wasm - // basename to resolve); every other subcommand stops at one arg. Pull the - // extra slot up-front so the unexpected-arg check below still catches stray - // inputs for the simple subcommands. - let extra = it.next(); - if it.next().is_some() { - return Err(format!("build-deps {sub}: unexpected extra args")); - } +#[cfg(windows)] +fn package_mirror_identity(metadata: &std::fs::Metadata) -> Result { + use std::os::windows::fs::MetadataExt; + Ok(PackageMirrorIdentity { + first: u64::from(metadata.volume_serial_number().ok_or_else(|| { + "directory metadata does not expose a volume serial number".to_string() + })?), + second: metadata + .file_index() + .ok_or_else(|| "directory metadata does not expose a file index".to_string())?, + }) +} - let repo = repo_root(); - let registry = Registry::from_env(&repo); +#[cfg(not(any(unix, windows)))] +fn package_mirror_identity(_metadata: &std::fs::Metadata) -> Result { + Err("package mirror transactions require stable host filesystem identities".to_string()) +} - // `--binaries-dir` is only meaningful for `resolve` — surface a - // clear error rather than silently ignoring it on other - // subcommands so a typo'd `resolve` never gets papered over. - if binaries_dir.is_some() && sub != "resolve" { +/// Capture a real package directory's filesystem identity and complete +/// symlink-only contents. The identity is checked on both sides of traversal, +/// so a concurrent pathname replacement cannot be mistaken for one snapshot. +fn inspect_package_mirror_snapshot(path: &Path) -> Result { + let before = std::fs::symlink_metadata(path) + .map_err(|e| format!("inspect package mirror {}: {e}", path.display()))?; + if !before.is_dir() || before.file_type().is_symlink() { + return Err(format!( + "package mirror must be a real directory: {}", + path.display() + )); + } + let identity = package_mirror_identity(&before)?; + let links = read_package_mirror_links(path)?; + if links.is_empty() { return Err(format!( - "build-deps {sub}: --binaries-dir is only valid for `resolve`" + "package mirror has no resolver-owned symlink leaves: {}", + path.display() )); } - if fetch_only && sub != "resolve" { + let after = std::fs::symlink_metadata(path) + .map_err(|e| format!("reinspect package mirror {}: {e}", path.display()))?; + if !after.is_dir() + || after.file_type().is_symlink() + || package_mirror_identity(&after)? != identity + { return Err(format!( - "build-deps {sub}: --fetch-only is only valid for `resolve`" + "package mirror identity changed while it was inspected: {}", + path.display() )); } + Ok(PackageMirrorSnapshot { identity, links }) +} - match sub.as_str() { - "check" => { - if target.is_some() { - return Err("build-deps check: takes no arguments".into()); - } - cmd_check(®istry) +fn validate_package_mirror_snapshot( + path: &Path, + expected: &PackageMirrorSnapshot, +) -> Result<(), String> { + let actual = inspect_package_mirror_snapshot(path)?; + if &actual == expected { + Ok(()) + } else { + Err(format!( + "package mirror identity or symlink contents changed: {}", + path.display() + )) + } +} + +/// Delete only a private transaction child whose filesystem identity and exact +/// resolver-owned link map still match the captured snapshot. +fn remove_validated_package_transaction_tree( + path: &Path, + expected: &PackageMirrorSnapshot, + label: &str, +) -> Result<(), String> { + if !path_entry_exists(path)? { + return Ok(()); + } + validate_package_mirror_snapshot(path, expected) + .map_err(|e| format!("refusing to remove changed {label} {}: {e}", path.display()))?; + std::fs::remove_dir_all(path) + .map_err(|e| format!("remove validated {label} {}: {e}", path.display())) +} + +/// A preparation failure may leave only a subset of the planned symlinks and +/// their ancestor directories. Validate that subset before deleting it; any +/// regular file, special entry, unexpected link, or unexpected directory +/// leaves the private transaction quarantined for manual inspection. +fn cleanup_prepared_package_transaction( + transaction_root: &Path, + stage_dir: &Path, + expected_links: &BTreeMap, +) -> Result<(), String> { + if path_entry_exists(stage_dir)? { + let before = std::fs::symlink_metadata(stage_dir).map_err(|e| { + format!( + "inspect partial package mirror {}: {e}", + stage_dir.display() + ) + })?; + if !before.is_dir() || before.file_type().is_symlink() { + return Err(format!( + "partial package mirror is not a real directory: {}", + stage_dir.display() + )); } - "output-fork-instrumentation-for-rel" => { - let rel = target.ok_or_else(|| { - "build-deps output-fork-instrumentation-for-rel: missing " - .to_string() - })?; - if extra.is_some() { - return Err( - "build-deps output-fork-instrumentation-for-rel: unexpected extra arg".into(), - ); - } - cmd_output_fork_instrumentation_for_rel(®istry, &rel) + let identity = package_mirror_identity(&before)?; + let mut links = BTreeMap::new(); + let mut directories = BTreeSet::new(); + read_package_mirror_links_inner(stage_dir, stage_dir, &mut links, &mut directories)?; + if links + .iter() + .any(|(relative, target)| expected_links.get(relative) != Some(target)) + { + return Err(format!( + "partial package mirror contains an unexpected symlink: {}", + stage_dir.display() + )); } - _ => { - let target = target.ok_or_else(|| format!("build-deps {sub}: missing "))?; - // `target` is either a path to a package.toml (contains '/' - // or ends with .toml) or a bare name to look up in the - // registry. - let manifest = load_target(&target, ®istry)?; - match sub.as_str() { - "parse" => { - if extra.is_some() { - return Err("build-deps parse: unexpected extra arg".into()); - } - cmd_parse(&manifest) - } - "sha" => { - if extra.is_some() { - return Err("build-deps sha: unexpected extra arg".into()); - } - cmd_sha(&manifest, ®istry, arch) - } - "path" => { - if extra.is_some() { - return Err("build-deps path: unexpected extra arg".into()); - } - cmd_path(&manifest, ®istry, arch) - } - "resolve" => { - if extra.is_some() { - return Err("build-deps resolve: unexpected extra arg".into()); - } - cmd_resolve( - &manifest, - ®istry, - &repo, - arch, - binaries_dir.as_deref(), - fetch_only, - ) - } - "output-path" => { - let basename = extra.ok_or_else(|| { - "build-deps output-path: missing \ - (usage: build-deps output-path )" - .to_string() - })?; - cmd_output_path(&manifest, &basename) - } - "runtime-file-path" => { - let artifact = extra.ok_or_else(|| { - "build-deps runtime-file-path: missing \ - (usage: build-deps runtime-file-path )" - .to_string() - })?; - cmd_runtime_file_path(&manifest, &artifact) - } - "runtime-file-metadata" => { - let artifact = extra.ok_or_else(|| { - "build-deps runtime-file-metadata: missing \ - (usage: build-deps runtime-file-metadata )" - .to_string() - })?; - cmd_runtime_file_metadata(&manifest, &artifact) - } - "output-fork-instrumentation" => { - let basename = extra.ok_or_else(|| { - "build-deps output-fork-instrumentation: missing \ - (usage: build-deps output-fork-instrumentation )" - .to_string() - })?; - cmd_output_fork_instrumentation(&manifest, &basename) - } - other => Err(format!("build-deps: unknown subcommand {other:?}")), + let allowed_directories = package_mirror_link_ancestor_directories(expected_links); + if !directories.is_subset(&allowed_directories) { + return Err(format!( + "partial package mirror contains an unexpected directory: {}", + stage_dir.display() + )); + } + let after = std::fs::symlink_metadata(stage_dir).map_err(|e| { + format!( + "reinspect partial package mirror {}: {e}", + stage_dir.display() + ) + })?; + if !after.is_dir() + || after.file_type().is_symlink() + || package_mirror_identity(&after)? != identity + { + return Err(format!( + "partial package mirror identity changed while it was inspected: {}", + stage_dir.display() + )); + } + std::fs::remove_dir_all(stage_dir).map_err(|e| { + format!( + "remove validated partial package mirror {}: {e}", + stage_dir.display() + ) + })?; + } + std::fs::remove_dir(transaction_root).map_err(|e| { + format!( + "remove empty package mirror transaction {}: {e}", + transaction_root.display() + ) + }) +} + +#[cfg(test)] +fn remove_owned_transaction_path(path: &Path) -> Result<(), String> { + match std::fs::symlink_metadata(path) { + Ok(metadata) if metadata.file_type().is_symlink() || metadata.is_file() => { + std::fs::remove_file(path) + .map_err(|e| format!("remove transaction path {}: {e}", path.display())) + } + Ok(metadata) if metadata.is_dir() => std::fs::remove_dir_all(path) + .map_err(|e| format!("remove transaction directory {}: {e}", path.display())), + Ok(_) => Err(format!( + "refusing to remove special transaction path {}", + path.display() + )), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(format!("inspect transaction path {}: {e}", path.display())), + } +} + +/// Read the exact symlink leaf set below a package-owned mirror directory. +/// +/// Directories contain no regular files: every leaf must remain a link into +/// the resolver cache. Returning the full map makes concurrent-winner +/// acceptance compare every declared output and runtime file, not a sentinel. +fn read_package_mirror_links(root: &Path) -> Result, String> { + let metadata = std::fs::symlink_metadata(root) + .map_err(|e| format!("inspect package mirror {}: {e}", root.display()))?; + if !metadata.is_dir() || metadata.file_type().is_symlink() { + return Err(format!( + "package mirror must be a real directory: {}", + root.display() + )); + } + let mut links = BTreeMap::new(); + let mut directories = BTreeSet::new(); + read_package_mirror_links_inner(root, root, &mut links, &mut directories)?; + let expected_directories = package_mirror_link_ancestor_directories(&links); + if directories != expected_directories { + return Err(format!( + "package mirror {} contains directories that are not exactly the ancestors of its symlink leaves", + root.display() + )); + } + Ok(links) +} + +fn read_package_mirror_links_inner( + root: &Path, + directory: &Path, + links: &mut BTreeMap, + directories: &mut BTreeSet, +) -> Result<(), String> { + let mut entries = std::fs::read_dir(directory) + .map_err(|e| format!("read package mirror directory {}: {e}", directory.display()))? + .collect::, _>>() + .map_err(|e| format!("read package mirror directory {}: {e}", directory.display()))?; + entries.sort_by_key(|entry| entry.path()); + for entry in entries { + let path = entry.path(); + let metadata = std::fs::symlink_metadata(&path) + .map_err(|e| format!("inspect package mirror entry {}: {e}", path.display()))?; + if metadata.file_type().is_symlink() { + let relative = path.strip_prefix(root).map_err(|e| { + format!( + "package mirror entry {} is not below {}: {e}", + path.display(), + root.display() + ) + })?; + let target = std::fs::read_link(&path) + .map_err(|e| format!("read package mirror link {}: {e}", path.display()))?; + if links.insert(relative.to_path_buf(), target).is_some() { + return Err(format!( + "duplicate package mirror destination {}", + relative.display() + )); } + } else if metadata.is_dir() { + let relative = path.strip_prefix(root).map_err(|e| { + format!( + "package mirror directory {} is not below {}: {e}", + path.display(), + root.display() + ) + })?; + directories.insert(relative.to_path_buf()); + read_package_mirror_links_inner(root, &path, links, directories)?; + } else { + return Err(format!( + "package mirror entry is not a symlink or directory: {}", + path.display() + )); } } + Ok(()) } -fn load_target(target: &str, registry: &Registry) -> Result { - let looks_like_path = - target.ends_with(".toml") || target.contains('/') || target.starts_with('.'); - if looks_like_path { - // Path form: derive the package dir from the .toml path so the - // overlay (sibling `package.pr.toml`) gets honored just like - // for registry-name lookups. Falls through to the plain `load` - // when the path doesn't sit inside a parent dir (rare; a - // top-level filename has no parent). Matches `Registry::load`. - let path = Path::new(target); - match path.parent() { - Some(dir) if !dir.as_os_str().is_empty() => DepsManifest::load_with_overlay(dir), - _ => DepsManifest::load(path), +fn package_mirror_link_ancestor_directories( + links: &BTreeMap, +) -> BTreeSet { + let mut directories = BTreeSet::new(); + for relative in links.keys() { + let mut parent = relative.parent(); + while let Some(directory) = parent { + if directory.as_os_str().is_empty() { + break; + } + directories.insert(directory.to_path_buf()); + parent = directory.parent(); } - } else { - registry.load(target) } + directories } -fn cmd_parse(m: &DepsManifest) -> Result<(), String> { - println!("name = {}", m.name); - println!("version = {}", m.version); - println!("revision = {}", m.revision); - println!("source = {}", m.source.url); - println!("sha256 = {}", m.source.sha256); - println!( - "license = {}{}", - m.license.spdx, - m.license - .url - .as_deref() - .map(|u| format!(" ({u})")) - .unwrap_or_default() - ); - println!( - "depends_on= [{}]", - m.depends_on - .iter() - .map(|d| d.to_string()) - .collect::>() - .join(", ") - ); - println!( - "build = {}", - m.build_script_path(&crate::repo_root()).display() - ); - println!("outputs.libs = {:?}", m.outputs.libs); - println!("outputs.headers = {:?}", m.outputs.headers); - if !m.outputs.pkgconfig.is_empty() { - println!("outputs.pkgconfig= {:?}", m.outputs.pkgconfig); - } - if !m.outputs.files.is_empty() { - println!("outputs.files = {:?}", m.outputs.files); - } - if !m.runtime_files.is_empty() { - println!("runtime_files = {:?}", m.runtime_files); +/// Parse the argument vector for `xtask compute-cache-key-sha`. +/// +/// Required flags (order-independent, both `--flag value` and +/// `--flag=value` forms accepted): +/// --package

Path to the package directory (containing +/// `package.toml`). +/// --arch Target architecture for the cache key. +/// +/// Hand-rolled because the CLI surface is small and the existing +/// `extract_arch_flag` helper is shared with `build-deps`, where +/// `--arch` is optional and the positional arguments differ. Keeping +/// this parser focused makes the contract for the pre-flight workflow +/// (Phase B-1, Task 2) easy to read at the call site. +fn parse_compute_cache_key_sha_args(args: Vec) -> Result<(PathBuf, TargetArch), String> { + let mut package: Option = None; + let mut arch: Option = None; + let mut it = args.into_iter(); + while let Some(a) = it.next() { + if let Some(value) = a.strip_prefix("--package=") { + if package.is_some() { + return Err("--package given more than once".into()); + } + package = Some(PathBuf::from(value)); + } else if a == "--package" { + if package.is_some() { + return Err("--package given more than once".into()); + } + let value = it + .next() + .ok_or_else(|| "--package requires a directory path".to_string())?; + package = Some(PathBuf::from(value)); + } else if let Some(value) = a.strip_prefix("--arch=") { + if arch.is_some() { + return Err("--arch given more than once".into()); + } + arch = Some(parse_target_arch(value)?); + } else if a == "--arch" { + if arch.is_some() { + return Err("--arch given more than once".into()); + } + let value = it + .next() + .ok_or_else(|| "--arch requires a value (wasm32 or wasm64)".to_string())?; + arch = Some(parse_target_arch(&value)?); + } else { + return Err(format!("unexpected argument {a:?}")); + } } - Ok(()) + let package = + package.ok_or_else(|| "compute-cache-key-sha: --package is required".to_string())?; + let arch = arch + .ok_or_else(|| "compute-cache-key-sha: --arch is required".to_string())?; + Ok((package, arch)) } -fn cmd_sha(m: &DepsManifest, registry: &Registry, arch: TargetArch) -> Result<(), String> { +/// Compute the cache-key sha for the manifest at +/// `/package.toml`, resolving deps against `registry`. +/// Returns the lowercase 64-char hex string (no trailing newline) so +/// callers can either print it directly or use it programmatically. +/// +/// This is a thin wrapper around [`compute_sha`] that loads the +/// manifest, threads through the canonical `memo` / `chain` state, and +/// hex-encodes the digest. Factored out from [`run_compute_cache_key_sha`] +/// so unit tests can exercise the logic without capturing stdout. +pub(crate) fn compute_cache_key_sha_for_package( + package_dir: &Path, + registry: &Registry, + arch: TargetArch, + abi_version: u32, +) -> Result { + let manifest = DepsManifest::load_with_overlay(package_dir)?; let mut memo = BTreeMap::new(); let mut chain = Vec::new(); let sha = compute_sha( - m, + &manifest, registry, arch, - current_abi_version(), + abi_version, &mut memo, &mut chain, )?; - println!("{}", hex(&sha)); - Ok(()) + Ok(hex(&sha)) } -fn cmd_path(m: &DepsManifest, registry: &Registry, arch: TargetArch) -> Result<(), String> { - let mut memo = BTreeMap::new(); - let mut chain = Vec::new(); - let sha = compute_sha( - m, - registry, - arch, - current_abi_version(), - &mut memo, - &mut chain, - )?; - let path = canonical_path(&default_cache_root(), m, arch, &sha); - println!("{}", path.display()); +/// CLI entry point for `xtask compute-cache-key-sha`. +/// +/// Wraps the existing internal [`compute_sha`] function as a stable +/// CLI surface for Phase B-1's pre-flight workflow, which calls this +/// for every (package, arch) pair to decide which matrix entries are +/// already published and can be skipped. +/// +/// Args: +/// --package Directory containing `package.toml`. +/// --arch Target architecture. +/// +/// On success: prints exactly 64 lowercase hex chars + newline to +/// stdout. On error: returns an `Err`; the top-level `xtask` dispatch +/// in `main.rs` writes it to stderr and exits non-zero. +pub fn run_compute_cache_key_sha(args: Vec) -> Result<(), String> { + let (package_dir, arch) = parse_compute_cache_key_sha_args(args)?; + let repo = repo_root(); + let registry = Registry::from_env(&repo); + let sha = + compute_cache_key_sha_for_package(&package_dir, ®istry, arch, current_abi_version())?; + println!("{sha}"); Ok(()) } -/// `output-path `: print the relative path -/// (under `programs//`) where the resolver places this program's -/// `wasm_basename` output via `place_binaries_symlinks`. +/// Cross-consumer host-tool consistency lint. Walks the registry, +/// groups `[[host_tools]]` declarations by `name` across consumers, +/// and reports an error when consumers disagree on +/// `version_constraint` or `probe` for the same tool name. /// -/// Consumed by `scripts/install-local-binary.sh` so build scripts drop -/// their freshly-built bytes at the same path the resolver writes to. -/// Without this, the build-script-side install-local-binary path could -/// diverge from the resolver path (the case that surfaced for texlive: -/// program "texlive" with output "pdftex" — the resolver writes -/// pdftex.wasm, but install_local_binary historically wrote -/// texlive.wasm or texlive/pdftex.wasm). -fn cmd_output_path(m: &DepsManifest, wasm_basename: &str) -> Result<(), String> { - let rel = m.output_dest_rel(wasm_basename)?; - println!("{}", rel.display()); - Ok(()) -} +/// Probe defaults are normalized at parse time +/// (`HostToolProbe::default()`), so a consumer that omits `[probe]` +/// compares equal to one that writes the same defaults explicitly. +/// +/// On success: exit 0 with a one-line summary. +/// On failure: every offending group is reported in the error. +fn cmd_check(registry: &Registry) -> Result<(), String> { + let manifests = registry.walk_all()?; + check_program_package_indexes_in_context(registry, false)?; -/// `runtime-file-path `: print the mirror path -/// below `programs//` used by local and resolver materialization. -fn cmd_runtime_file_path(m: &DepsManifest, artifact: &str) -> Result<(), String> { - let rel = m.runtime_file_dest_rel(artifact)?; - println!("{}", rel.display()); - Ok(()) -} + // Group: tool_name -> Vec<(consumer_name, &HostTool)>. + let mut by_tool: BTreeMap> = BTreeMap::new(); + for (cname, m) in &manifests { + for tool in &m.host_tools { + by_tool + .entry(tool.name.clone()) + .or_default() + .push((cname.clone(), tool)); + } + } -/// Structured installation contract for VFS/image builders. JSON avoids -/// consumers scraping Debug output and keeps guest path/mode authoritative. -fn cmd_runtime_file_metadata(m: &DepsManifest, artifact: &str) -> Result<(), String> { - let value = runtime_file_metadata_value(m, artifact)?; + let tool_count = by_tool.len(); + let consumer_count = manifests + .iter() + .filter(|(_, m)| !m.host_tools.is_empty()) + .count(); + + let mut problems: Vec = Vec::new(); + for (tool, group) in &by_tool { + if group.len() < 2 { + continue; + } + // Compare each entry against the first. + let (first_consumer, first_tool) = &group[0]; + for (other_consumer, other_tool) in &group[1..] { + if first_tool.version_constraint != other_tool.version_constraint { + problems.push(format!( + "host-tool {tool:?}: inconsistent version_constraint\n - {first_consumer}: >={}\n - {other_consumer}: >={}", + first_tool.version_constraint.min, + other_tool.version_constraint.min, + )); + } + if first_tool.probe.args != other_tool.probe.args + || first_tool.probe.version_regex != other_tool.probe.version_regex + { + problems.push(format!( + "host-tool {tool:?}: inconsistent probe between {first_consumer} and {other_consumer}\n - args: {:?} vs {:?}\n - regex: {:?} vs {:?}", + first_tool.probe.args, other_tool.probe.args, + first_tool.probe.version_regex, other_tool.probe.version_regex, + )); + } + } + } + + if !problems.is_empty() { + let msg = problems.join("\n\n"); + return Err(format!("host-tool consistency check failed:\n\n{msg}")); + } println!( - "{}", - serde_json::to_string(&value).map_err(|e| format!("serialize runtime metadata: {e}"))? + "host-tool consistency: {tool_count} tool(s) across {consumer_count} consumer(s) — OK" ); Ok(()) } -fn runtime_file_metadata_value( - m: &DepsManifest, - artifact: &str, -) -> Result { - let runtime_file = m - .runtime_files - .iter() - .find(|runtime_file| runtime_file.artifact == artifact) - .ok_or_else(|| { +// --------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + use std::fs; + + fn write(dir: &Path, name: &str, version: &str, depends_on: &[&str]) { + let lib_dir = dir.join(name); + fs::create_dir_all(&lib_dir).unwrap(); + let depends = depends_on + .iter() + .map(|s| format!("{:?}", s)) + .collect::>() + .join(", "); + let text = format!( + r#" +kind = "library" +name = "{name}" +version = "{version}" +depends_on = [{depends}] + +[source] +url = "https://example.test/{name}-{version}.tar.gz" +sha256 = "{:0>64}" + +[license] +spdx = "TestLicense" + +[outputs] +libs = ["lib/lib{name}.a"] +"#, + "" + ); + fs::write(lib_dir.join("package.toml"), text).unwrap(); + } + + fn write_build_revision(dir: &Path, name: &str, revision: u32) { + fs::write( + dir.join(name).join("build.toml"), format!( - "program {:?} has no [[runtime_files]] artifact {:?}", - m.name, artifact - ) - })?; - // A runtime file is meaningful only alongside the exact executable and - // side-module outputs produced by the same program package archive. Give - // repo-side consumers the complete resolver mirror closure so they can - // select one materialization tier atomically instead of resolving each - // member independently and accidentally mixing builds. - let closure_mirror_paths: Vec = m - .program_outputs - .iter() - .map(|output| m.output_dest_rel_for(output)) - .chain( - m.runtime_files - .iter() - .map(|runtime_file| m.runtime_file_dest_rel_for(runtime_file)), + r#" +script_path = "packages/registry/{name}/build-{name}.sh" +inputs = [] +repo_url = "https://example.test/kandelo.git" +commit = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef" +revision = {revision} + +[binary] +index_url = "https://example.test/releases/download/binaries-abi-v{{abi}}/index.toml" +"# + ), ) - .collect(); - Ok(serde_json::json!({ - "artifact": runtime_file.artifact, - "guest_path": runtime_file.guest_path, - "mode": runtime_file.mode, - "mirror_path": m.runtime_file_dest_rel_for(runtime_file), - "closure_mirror_paths": closure_mirror_paths, - })) -} + .unwrap(); + } -fn cmd_output_fork_instrumentation(m: &DepsManifest, wasm_basename: &str) -> Result<(), String> { - let policy = m.output_fork_instrumentation(wasm_basename)?; - println!("{}", policy.as_str()); - Ok(()) -} + fn write_build_with_input( + dir: &Path, + name: &str, + revision: u32, + input: &str, + contents: &str, + ) { + let input_path = dir.join(name).join(input); + fs::write(&input_path, contents).unwrap(); + fs::write( + dir.join(name).join("build.toml"), + format!( + r#" +script_path = "packages/registry/{name}/build-{name}.sh" +inputs = ["packages/registry/{name}/{input}"] +repo_url = "https://example.test/kandelo.git" +commit = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef" +revision = {revision} -fn cmd_output_fork_instrumentation_for_rel( - registry: &Registry, - resolver_rel: &str, -) -> Result<(), String> { - let policy = output_fork_instrumentation_for_rel(registry, resolver_rel)?; - println!("{}", policy.as_str()); - Ok(()) -} +[binary] +index_url = "https://example.test/releases/download/binaries-abi-v{{abi}}/index.toml" +"# + ), + ) + .unwrap(); + } -fn output_fork_instrumentation_for_rel( - registry: &Registry, - resolver_rel: &str, -) -> Result { - let rel = resolver_rel - .strip_prefix("programs/wasm32/") - .or_else(|| resolver_rel.strip_prefix("programs/wasm64/")) - .or_else(|| resolver_rel.strip_prefix("programs/")) - .unwrap_or(resolver_rel); - for (_, manifest) in programs_by_name(registry)? { - for out in &manifest.program_outputs { - if manifest.output_dest_rel_for(out).to_string_lossy().as_ref() == rel { - return Ok(out.fork_instrumentation); - } - } + fn tempdir(label: &str) -> PathBuf { + let p = std::env::temp_dir() + .join("wpk-xtask-test") + .join(format!("{label}-{}", std::process::id())); + let _ = fs::remove_dir_all(&p); + fs::create_dir_all(&p).unwrap(); + p } - Ok(ForkInstrumentationPolicy::Auto) -} -fn cmd_resolve( - m: &DepsManifest, - registry: &Registry, - repo: &Path, - arch: TargetArch, - binaries_dir: Option<&Path>, - fetch_only: bool, -) -> Result<(), String> { - let cache_root = default_cache_root(); - let local_libs = repo.join("local-libs"); - let opts = ResolveOpts { - cache_root: &cache_root, - local_libs: Some(&local_libs), - force_source_build: None, - fetch_only, - repo_root: Some(repo), - // Plumb binaries_dir into ensure_built so place_binaries_symlinks - // runs for every transitive program dep, not just the target. - // The previous direct call here (post-ensure_built) only placed - // symlinks for `m`; consumer build scripts that read sibling - // package binaries via `tryResolveBinary` need the dep - // symlinks too. - binaries_dir, - }; - let path = ensure_built(m, registry, arch, current_abi_version(), &opts)?; + #[test] + fn relative_registry_roots_anchor_at_the_kandelo_repository() { + let repo = Path::new("/kandelo/source"); + assert_eq!( + resolve_registry_root(repo, "third-party/registry"), + repo.join("third-party/registry"), + ); + assert_eq!( + resolve_registry_root(repo, "/shared/registry"), + PathBuf::from("/shared/registry"), + ); + } + + #[test] + fn committed_program_package_projection_is_current() { + let registry_root = crate::repo_root().join("packages/registry"); + let registry = Registry { + roots: vec![registry_root.clone()], + }; + cmd_check_program_package_index( + ®istry_root, + ®istry_root.join("program-packages.json"), + ®istry, + ) + .unwrap(); + } + + #[test] + fn program_package_projection_excludes_root_boot_artifacts() { + let registry_root = tempdir("program-projection-root-boot-artifacts"); + write_program( + ®istry_root, + "kernel", + "1.0.0", + &[], + ":", + &[("kernel", "kandelo-kernel.wasm")], + ); + write_program( + ®istry_root, + "userspace", + "1.0.0", + &[], + ":", + &[("userspace", "wasm_posix_userspace.wasm")], + ); + write_program( + ®istry_root, + "guest-command", + "1.0.0", + &[], + ":", + &[("guest-command", "guest-command.wasm")], + ); + let registry = Registry { + roots: vec![registry_root.clone()], + }; + + let projection = program_package_index_for_root(®istry_root, ®istry).unwrap(); + assert_eq!( + projection + .packages + .keys() + .map(String::as_str) + .collect::>(), + vec!["guest-command"], + ); + } + + #[test] + fn program_projection_binds_external_programs_to_full_first_hit_dependency_context() { + let main_root = tempdir("program-projection-context-main"); + let external_root = tempdir("program-projection-context-external"); + write(&main_root, "shared", "1.0.0", &[]); + write(&main_root, "middle", "1.0.0", &["shared@1.0.0"]); + write_build_revision(&main_root, "middle", 7); + write_program( + &main_root, + "direct-program", + "1.0.0", + &["shared@1.0.0"], + ":", + &[("direct-program", "direct-program.wasm")], + ); + write_program( + &main_root, + "transitive-program", + "1.0.0", + &["middle@1.0.0"], + ":", + &[("transitive-program", "transitive-program.wasm")], + ); + let source_dir = main_root.join("source-data"); + fs::create_dir_all(&source_dir).unwrap(); + fs::write( + source_dir.join("package.toml"), + r#"kind = "source" +name = "source-data" +version = "1.0.0" +depends_on = [] +[source] +url = "https://example.test/source-data-1.0.0.tar.gz" +sha256 = "0000000000000000000000000000000000000000000000000000000000000000" +[license] +spdx = "MIT" +"#, + ) + .unwrap(); + write_program( + &external_root, + "external-program", + "1.0.0", + &["middle@1.0.0", "source-data@1.0.0"], + ":", + &[("external-program", "external-program.wasm")], + ); + write_build_revision(&external_root, "external-program", 9); + let external_manifest = external_root.join("external-program/package.toml"); + let external_text = fs::read_to_string(&external_manifest).unwrap(); + fs::write( + &external_manifest, + external_text.replace( + "version = \"1.0.0\"\n", + "version = \"1.0.0\"\narches = [\"wasm32\", \"wasm64\"]\n", + ), + ) + .unwrap(); + write(&external_root, "shared", "1.0.0", &[]); + + let main_registry = Registry { + roots: vec![main_root.clone()], + }; + let main_projection = program_package_index_for_root(&main_root, &main_registry).unwrap(); + let source_identity = &main_projection.identities["source-data"]; + assert_eq!( + source_identity.cache_keys["wasm32"], source_identity.cache_keys["wasm64"], + "source-kind package context must remain architecture independent", + ); + + let combined_registry = Registry { + roots: vec![external_root.clone(), main_root.clone()], + }; + let initial_external = + program_package_index_for_root(&external_root, &combined_registry).unwrap(); + let expected_middle = package_context_cache_keys( + &combined_registry.load("middle").unwrap(), + &combined_registry, + ) + .unwrap(); + let expected_external = package_context_cache_keys( + &combined_registry.load("external-program").unwrap(), + &combined_registry, + ) + .unwrap(); + assert_eq!( + initial_external.identities["middle"].cache_keys, expected_middle, + "the projection must use the same build.toml revision as normal dependency resolution", + ); + assert_eq!( + initial_external.identities["external-program"].cache_keys, expected_external, + "the projected program identity must use its build.toml revision", + ); + assert_eq!( + initial_external.packages["external-program"].cache_keys, expected_external, + "the program projection and top-level identity must describe one exact generation", + ); + assert_eq!( + initial_external.identities["shared"], main_projection.identities["shared"], + "an identical first-hit shadow must retain the same exact identity", + ); + assert_eq!( + initial_external.identities["middle"], main_projection.identities["middle"], + "the highest-priority index must carry lower-root identities in the exact combined context", + ); + assert_eq!( + initial_external.identities["source-data"], main_projection.identities["source-data"], + "architecture-independent lower-root sources must remain in the combined context", + ); + for package_name in ["direct-program", "transitive-program"] { + assert_eq!( + initial_external.packages[package_name], main_projection.packages[package_name], + "an identical first-hit context must carry lower-root program projections into the complete top index", + ); + } + let external_program = &initial_external.packages["external-program"]; + for arch in ["wasm32", "wasm64"] { + assert_eq!( + external_program.dependency_closures[arch] + .iter() + .map(|identity| identity.package_name.as_str()) + .collect::>(), + vec!["middle", "shared", "source-data"], + "the projected closure must include direct and transitive dependencies", + ); + let projected_source = external_program.dependency_closures[arch] + .iter() + .find(|identity| identity.package_name == "source-data") + .unwrap(); + assert_eq!(projected_source.cache_key, source_identity.cache_keys[arch],); + } - // Top-level target: ensure_built places symlinks for transitive - // deps via opts.binaries_dir, but the *target's* own symlinks land - // here so we don't recurse into "place self" inside ensure_built - // (which would also fire from archive-stage's ensure_built call, - // where placing target symlinks isn't desired). - if let Some(bdir) = binaries_dir { - if matches!(m.kind, ManifestKind::Program) && !m.program_outputs.is_empty() { - place_binaries_symlinks(m, &path, bdir, arch)?; + let external_shared = external_root.join("shared/package.toml"); + let changed = fs::read_to_string(&external_shared) + .unwrap() + .replace(&"0".repeat(64), &"1".repeat(64)); + fs::write(&external_shared, changed).unwrap(); + let changed_external = + program_package_index_for_root(&external_root, &combined_registry).unwrap(); + assert_ne!( + changed_external.identities["shared"], main_projection.identities["shared"], + "a changed first-hit shadow must carry a different contextual identity", + ); + assert_ne!( + changed_external.identities["middle"], main_projection.identities["middle"], + "a lower-root package identity must incorporate a changed transitive first-hit dependency", + ); + for package_name in ["direct-program", "transitive-program"] { + assert_ne!( + changed_external.packages[package_name].cache_keys, + main_projection.packages[package_name].cache_keys, + "a dependency-only override must rekey each affected lower-root program in the complete top projection", + ); + assert_eq!( + changed_external.packages[package_name].cache_keys, + changed_external.identities[package_name] + .cache_keys + .iter() + .filter(|(arch, _)| { + changed_external.packages[package_name] + .arches + .contains(arch) + }) + .map(|(arch, key)| (arch.clone(), key.clone())) + .collect(), + "the reprojected lower program must use its exact combined-context identity", + ); + } + for arch in ["wasm32", "wasm64"] { + let projected_shared = changed_external.packages["external-program"] + .dependency_closures[arch] + .iter() + .find(|identity| identity.package_name == "shared") + .unwrap(); + assert_eq!( + projected_shared.manifest_sha256, + changed_external.identities["shared"].manifest_sha256, + ); + assert_eq!( + projected_shared.cache_key, + changed_external.identities["shared"].cache_keys[arch], + ); + let projected_middle = changed_external.packages["external-program"] + .dependency_closures[arch] + .iter() + .find(|identity| identity.package_name == "middle") + .unwrap(); + assert_eq!( + projected_middle.manifest_sha256, + changed_external.identities["middle"].manifest_sha256, + ); + assert_eq!( + projected_middle.cache_key, + changed_external.identities["middle"].cache_keys[arch], + ); } } - println!("{}", path.display()); - Ok(()) -} + #[test] + fn registry_check_validates_each_root_index_in_its_suffix_context() { + let main_root = tempdir("program-index-check-context-main"); + let external_root = tempdir("program-index-check-context-external"); + write(&main_root, "shared", "1.0.0", &[]); + write_program( + &main_root, + "main-program", + "1.0.0", + &["shared@1.0.0"], + ":", + &[("main-program", "main-program.wasm")], + ); + write(&external_root, "shared", "1.0.0", &[]); + let external_shared = external_root.join("shared/package.toml"); + let changed = fs::read_to_string(&external_shared) + .unwrap() + .replace(&"0".repeat(64), &"1".repeat(64)); + fs::write(&external_shared, changed).unwrap(); -/// Place symlinks under `binaries_dir/programs//` pointing at -/// each declared `[[outputs]]` artifact and `[[runtime_files]]` file in the -/// cache canonical directory. -/// -/// Layout (per arch — wasm32 and wasm64 mirror in parallel): -/// * 1 output: `/programs//.wasm`. -/// * ≥2 outputs: `/programs///.wasm`. -/// * first-party kernel/userspace: `/.wasm`. -/// -/// This is the single source of truth for the symlink layout. Browser -/// demos hardcode these paths (see `apps/browser-demos/vite.config.ts` -/// and `host/src/binary-resolver.ts`), so the layout MUST NOT change -/// here without coordinating with the consumer-side import paths. -/// -/// Targets are absolute paths into the resolver cache. Replace-in-place -/// is safe (remove + symlink): symlinks are tiny and atomic, and a -/// stale link that survives an arch flip would silently route consumers -/// at the wrong arch — correctness trumps a microsecond saved on a -/// no-op. -fn place_binaries_symlinks( - m: &DepsManifest, - canonical: &Path, - binaries_dir: &Path, - arch: TargetArch, -) -> Result<(), String> { - let outputs = &m.program_outputs; - if outputs.is_empty() { - return Err(format!("program {:?} has no [[outputs]]", m.name)); - } - let arch_root = binaries_dir.join("programs").join(arch.as_str()); - for out in outputs { - let src = canonical.join(&out.wasm); - if !src.is_file() { - return Err(format!( - "declared output {} not found in cache at {}", - out.wasm, - src.display() - )); - } - let dest = if (m.name == "kernel" || m.name == "userspace") && outputs.len() == 1 { - binaries_dir.join(format!("{}.wasm", out.name)) - } else { - arch_root.join(m.output_dest_rel_for(out)) + let main_registry = Registry { + roots: vec![main_root.clone()], }; - let dest_dir = dest - .parent() - .ok_or_else(|| format!("dest path {} has no parent", dest.display()))?; - std::fs::create_dir_all(dest_dir) - .map_err(|e| format!("mkdir {}: {e}", dest_dir.display()))?; - // Replace-in-place: remove any existing entry (file or - // symlink), then create a fresh symlink. Skipping the remove - // step would cause `symlink` to fail with EEXIST. - if dest.exists() || dest.symlink_metadata().is_ok() { - let _ = std::fs::remove_file(&dest); - } - std::os::unix::fs::symlink(&src, &dest) - .map_err(|e| format!("symlink {} -> {}: {e}", dest.display(), src.display()))?; + fs::write( + main_root.join("program-packages.json"), + serialize_program_package_index(&main_root, &main_registry).unwrap(), + ) + .unwrap(); + + let combined_registry = Registry { + roots: vec![external_root.clone(), main_root.clone()], + }; + fs::write( + external_root.join("program-packages.json"), + serialize_program_package_index(&external_root, &combined_registry).unwrap(), + ) + .unwrap(); + + check_program_package_indexes_in_context(&combined_registry, true).expect( + "a lower root's suffix-context index must remain valid when a higher root shadows its dependency", + ); } - for runtime_file in &m.runtime_files { - let src = canonical.join(&runtime_file.artifact); - let metadata = std::fs::symlink_metadata(&src).map_err(|e| { - format!( - "declared runtime file {} not found in cache at {}: {e}", - runtime_file.artifact, - src.display() + + #[test] + fn source_context_check_requires_an_index_for_each_existing_registry_root() { + let existing_root = tempdir("program-index-context-required"); + let missing_root = existing_root.with_extension("absent"); + let _ = fs::remove_dir_all(&missing_root); + let registry = Registry { + roots: vec![missing_root, existing_root.clone()], + }; + + let error = check_program_package_indexes_in_context(®istry, true).unwrap_err(); + assert!( + error.contains("missing") && error.contains("program-packages.json"), + "got: {error}" + ); + + fs::write( + existing_root.join("program-packages.json"), + serialize_program_package_index( + &existing_root, + &Registry { + roots: vec![existing_root.clone()], + }, ) - })?; - if !metadata.is_file() || metadata.file_type().is_symlink() { - return Err(format!( - "declared runtime file {} is not a regular non-symlink file at {}", - runtime_file.artifact, - src.display() - )); - } - let dest = arch_root.join(m.runtime_file_dest_rel_for(runtime_file)); - let dest_dir = dest - .parent() - .ok_or_else(|| format!("dest path {} has no parent", dest.display()))?; - std::fs::create_dir_all(dest_dir) - .map_err(|e| format!("mkdir {}: {e}", dest_dir.display()))?; - if dest.exists() || dest.symlink_metadata().is_ok() { - let _ = std::fs::remove_file(&dest); - } - std::os::unix::fs::symlink(&src, &dest) - .map_err(|e| format!("symlink {} -> {}: {e}", dest.display(), src.display()))?; + .unwrap(), + ) + .unwrap(); + check_program_package_indexes_in_context(®istry, true) + .expect("nonexistent roots are skipped and every existing root has a fresh index"); } - Ok(()) -} -/// Parse the argument vector for `xtask compute-cache-key-sha`. -/// -/// Required flags (order-independent, both `--flag value` and -/// `--flag=value` forms accepted): -/// --package Path to the package directory (containing -/// `package.toml`). -/// --arch Target architecture for the cache key. -/// -/// Hand-rolled because the CLI surface is small and the existing -/// `extract_arch_flag` helper is shared with `build-deps`, where -/// `--arch` is optional and the positional arguments differ. Keeping -/// this parser focused makes the contract for the pre-flight workflow -/// (Phase B-1, Task 2) easy to read at the call site. -fn parse_compute_cache_key_sha_args(args: Vec) -> Result<(PathBuf, TargetArch), String> { - let mut package: Option = None; - let mut arch: Option = None; - let mut it = args.into_iter(); - while let Some(a) = it.next() { - if let Some(value) = a.strip_prefix("--package=") { - if package.is_some() { - return Err("--package given more than once".into()); - } - package = Some(PathBuf::from(value)); - } else if a == "--package" { - if package.is_some() { - return Err("--package given more than once".into()); - } - let value = it - .next() - .ok_or_else(|| "--package requires a directory path".to_string())?; - package = Some(PathBuf::from(value)); - } else if let Some(value) = a.strip_prefix("--arch=") { - if arch.is_some() { - return Err("--arch given more than once".into()); - } - arch = Some(parse_target_arch(value)?); - } else if a == "--arch" { - if arch.is_some() { - return Err("--arch given more than once".into()); - } - let value = it - .next() - .ok_or_else(|| "--arch requires a value (wasm32 or wasm64)".to_string())?; - arch = Some(parse_target_arch(&value)?); - } else { - return Err(format!("unexpected argument {a:?}")); - } - } - let package = - package.ok_or_else(|| "compute-cache-key-sha: --package is required".to_string())?; - let arch = arch - .ok_or_else(|| "compute-cache-key-sha: --arch is required".to_string())?; - Ok((package, arch)) -} + #[test] + fn source_context_check_rejects_revision_input_and_transitive_input_mutations() { + let root = tempdir("program-index-context-source-freshness"); + write(&root, "dependency", "1.0.0", &[]); + write_build_with_input(&root, "dependency", 1, "recipe.txt", "dependency-one\n"); + write_program( + &root, + "command", + "1.0.0", + &["dependency@1.0.0"], + ":", + &[("command", "command.wasm")], + ); + write_build_with_input(&root, "command", 1, "recipe.txt", "command-one\n"); + let registry = Registry { + roots: vec![root.clone()], + }; + let index = root.join("program-packages.json"); + let refresh = || { + fs::write( + &index, + serialize_program_package_index(&root, ®istry).unwrap(), + ) + .unwrap(); + }; + let assert_stale = |reason: &str| { + let error = check_program_package_indexes_in_context(®istry, true).unwrap_err(); + assert!(error.contains("is stale"), "{reason}: got {error}"); + }; -/// Compute the cache-key sha for the manifest at -/// `/package.toml`, resolving deps against `registry`. -/// Returns the lowercase 64-char hex string (no trailing newline) so -/// callers can either print it directly or use it programmatically. -/// -/// This is a thin wrapper around [`compute_sha`] that loads the -/// manifest, threads through the canonical `memo` / `chain` state, and -/// hex-encodes the digest. Factored out from [`run_compute_cache_key_sha`] -/// so unit tests can exercise the logic without capturing stdout. -pub(crate) fn compute_cache_key_sha_for_package( - package_dir: &Path, - registry: &Registry, - arch: TargetArch, - abi_version: u32, -) -> Result { - let manifest = DepsManifest::load_with_overlay(package_dir)?; - let mut memo = BTreeMap::new(); - let mut chain = Vec::new(); - let sha = compute_sha( - &manifest, - registry, - arch, - abi_version, - &mut memo, - &mut chain, - )?; - Ok(hex(&sha)) -} + refresh(); + let command_build = root.join("command/build.toml"); + let original_command_build = fs::read_to_string(&command_build).unwrap(); + fs::write( + &command_build, + original_command_build.replace("revision = 1", "revision = 2"), + ) + .unwrap(); + assert_stale("program build.toml revision mutation"); -/// CLI entry point for `xtask compute-cache-key-sha`. -/// -/// Wraps the existing internal [`compute_sha`] function as a stable -/// CLI surface for Phase B-1's pre-flight workflow, which calls this -/// for every (package, arch) pair to decide which matrix entries are -/// already published and can be skipped. -/// -/// Args: -/// --package Directory containing `package.toml`. -/// --arch Target architecture. -/// -/// On success: prints exactly 64 lowercase hex chars + newline to -/// stdout. On error: returns an `Err`; the top-level `xtask` dispatch -/// in `main.rs` writes it to stderr and exits non-zero. -pub fn run_compute_cache_key_sha(args: Vec) -> Result<(), String> { - let (package_dir, arch) = parse_compute_cache_key_sha_args(args)?; - let repo = repo_root(); - let registry = Registry::from_env(&repo); - let sha = - compute_cache_key_sha_for_package(&package_dir, ®istry, arch, current_abi_version())?; - println!("{sha}"); - Ok(()) -} + fs::write(&command_build, &original_command_build).unwrap(); + refresh(); + fs::write(root.join("command/recipe.txt"), "command-two\n").unwrap(); + assert_stale("program declared build input mutation"); -/// Cross-consumer host-tool consistency lint. Walks the registry, -/// groups `[[host_tools]]` declarations by `name` across consumers, -/// and reports an error when consumers disagree on -/// `version_constraint` or `probe` for the same tool name. -/// -/// Probe defaults are normalized at parse time -/// (`HostToolProbe::default()`), so a consumer that omits `[probe]` -/// compares equal to one that writes the same defaults explicitly. -/// -/// On success: exit 0 with a one-line summary. -/// On failure: every offending group is reported in the error. -fn cmd_check(registry: &Registry) -> Result<(), String> { - let manifests = registry.walk_all()?; + fs::write(root.join("command/recipe.txt"), "command-one\n").unwrap(); + refresh(); + fs::write(root.join("dependency/recipe.txt"), "dependency-two\n").unwrap(); + assert_stale("transitive dependency declared build input mutation"); + } + + #[test] + fn complete_top_projection_excludes_a_lower_program_shadowed_by_a_non_program() { + let main_root = tempdir("program-projection-non-program-shadow-main"); + let external_root = tempdir("program-projection-non-program-shadow-external"); + write_program( + &main_root, + "same-name", + "1.0.0", + &[], + ":", + &[("same-name", "same-name.wasm")], + ); + write(&external_root, "same-name", "2.0.0", &[]); - // Group: tool_name -> Vec<(consumer_name, &HostTool)>. - let mut by_tool: BTreeMap> = BTreeMap::new(); - for (cname, m) in &manifests { - for tool in &m.host_tools { - by_tool - .entry(tool.name.clone()) - .or_default() - .push((cname.clone(), tool)); - } + let main_registry = Registry { + roots: vec![main_root.clone()], + }; + let main_projection = program_package_index_for_root(&main_root, &main_registry).unwrap(); + assert!(main_projection.packages.contains_key("same-name")); + + let combined_registry = Registry { + roots: vec![external_root.clone(), main_root.clone()], + }; + let combined_projection = + program_package_index_for_root(&external_root, &combined_registry).unwrap(); + assert!( + combined_projection.identities.contains_key("same-name"), + "the first-hit non-program still needs a contextual identity", + ); + assert!( + !combined_projection.packages.contains_key("same-name"), + "a lower program must not survive a higher first-hit non-program shadow", + ); } - let tool_count = by_tool.len(); - let consumer_count = manifests - .iter() - .filter(|(_, m)| !m.host_tools.is_empty()) - .count(); + #[test] + fn program_projection_generation_requires_its_root_to_be_first_existing() { + let main_root = tempdir("program-projection-root-order-main"); + let external_root = tempdir("program-projection-root-order-external"); + write(&main_root, "main-library", "1.0.0", &[]); + write(&external_root, "external-library", "1.0.0", &[]); + let combined_registry = Registry { + roots: vec![external_root.clone(), main_root.clone()], + }; - let mut problems: Vec = Vec::new(); - for (tool, group) in &by_tool { - if group.len() < 2 { - continue; - } - // Compare each entry against the first. - let (first_consumer, first_tool) = &group[0]; - for (other_consumer, other_tool) in &group[1..] { - if first_tool.version_constraint != other_tool.version_constraint { - problems.push(format!( - "host-tool {tool:?}: inconsistent version_constraint\n - {first_consumer}: >={}\n - {other_consumer}: >={}", - first_tool.version_constraint.min, - other_tool.version_constraint.min, - )); - } - if first_tool.probe.args != other_tool.probe.args - || first_tool.probe.version_regex != other_tool.probe.version_regex - { - problems.push(format!( - "host-tool {tool:?}: inconsistent probe between {first_consumer} and {other_consumer}\n - args: {:?} vs {:?}\n - regex: {:?} vs {:?}", - first_tool.probe.args, other_tool.probe.args, - first_tool.probe.version_regex, other_tool.probe.version_regex, - )); + let error = program_package_index_for_root(&main_root, &combined_registry).unwrap_err(); + assert!( + error.contains("not the highest-priority existing configured registry root") + && error.contains(&external_root.display().to_string()), + "got: {error}", + ); + } + + #[test] + fn program_package_projection_publication_never_exposes_partial_json() { + let root = tempdir("program-projection-atomic-readers"); + let output = root.join("program-packages.json"); + write_program_package_index_atomically(&output, br#"{"generation":0}"#).unwrap(); + let stop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let reader_stop = std::sync::Arc::clone(&stop); + let reader_output = output.clone(); + let reader = std::thread::spawn(move || { + while !reader_stop.load(Ordering::Acquire) { + let bytes = fs::read(&reader_output).unwrap(); + let value: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert!(value["generation"].as_u64().is_some()); } + }); + + for generation in 1..=100 { + let json = format!(r#"{{"generation":{generation}}}"#); + write_program_package_index_atomically(&output, json.as_bytes()).unwrap(); } + stop.store(true, Ordering::Release); + reader.join().unwrap(); + assert_eq!( + serde_json::from_slice::(&fs::read(&output).unwrap()).unwrap()["generation"], + 100 + ); } - if !problems.is_empty() { - let msg = problems.join("\n\n"); - return Err(format!("host-tool consistency check failed:\n\n{msg}")); + #[test] + fn program_package_projection_publish_failure_preserves_the_old_index() { + let root = tempdir("program-projection-publish-failure"); + let output = root.join("program-packages.json"); + fs::write(&output, b"{\"generation\":\"old\"}\n").unwrap(); + let mut fail_replace = |_from: &Path, _to: &Path| { + Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "injected projection publish failure", + )) + }; + + let error = write_program_package_index_atomically_with( + &output, + b"{\"generation\":\"new\"}\n", + &mut fail_replace, + ) + .unwrap_err(); + assert!( + error.contains("injected projection publish failure"), + "got: {error}" + ); + assert_eq!(fs::read(&output).unwrap(), b"{\"generation\":\"old\"}\n"); + assert!(fs::read_dir(&root).unwrap().all(|entry| { + !entry + .unwrap() + .file_name() + .to_string_lossy() + .contains(".index-transaction-") + })); } - println!( - "host-tool consistency: {tool_count} tool(s) across {consumer_count} consumer(s) — OK" - ); - Ok(()) -} -// --------------------------------------------------------------------- -// Tests -// --------------------------------------------------------------------- + #[test] + fn program_package_projection_source_change_after_staging_preserves_the_old_index() { + let root = tempdir("program-projection-source-change"); + let output = root.join("program-packages.json"); + fs::write(&output, b"{\"generation\":\"old\"}\n").unwrap(); + let mut refresh_changed_source = || Ok(b"{\"generation\":\"newer-source\"}\n".to_vec()); + let mut replace = |from: &Path, to: &Path| fs::rename(from, to); + + let error = write_program_package_index_atomically_with_source( + &output, + b"{\"generation\":\"staged\"}\n", + &mut refresh_changed_source, + &mut replace, + ) + .unwrap_err(); + assert!( + error.contains("registry changed after") + || error.contains("refresh program package index"), + "got: {error}" + ); + assert_eq!(fs::read(&output).unwrap(), b"{\"generation\":\"old\"}\n"); + } -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - use std::fs; + #[test] + fn program_package_projection_stale_writer_preserves_a_newer_target() { + let root = tempdir("program-projection-target-cas"); + let output = root.join("program-packages.json"); + let newer_output = output.clone(); + let staged = b"{\"generation\":\"staged\"}\n".to_vec(); + let staged_for_refresh = staged.clone(); + let mut publish_newer_target = move || { + fs::write(&newer_output, b"{\"generation\":\"newer\"}\n").unwrap(); + Ok(staged_for_refresh.clone()) + }; + let mut replace = |from: &Path, to: &Path| fs::rename(from, to); - fn write(dir: &Path, name: &str, version: &str, depends_on: &[&str]) { - let lib_dir = dir.join(name); - fs::create_dir_all(&lib_dir).unwrap(); - let depends = depends_on - .iter() - .map(|s| format!("{:?}", s)) - .collect::>() - .join(", "); - let text = format!( - r#" -kind = "library" -name = "{name}" -version = "{version}" -depends_on = [{depends}] + let error = write_program_package_index_atomically_with_source( + &output, + &staged, + &mut publish_newer_target, + &mut replace, + ) + .unwrap_err(); + assert!( + error.contains("target appeared before publication"), + "got: {error}" + ); + assert_eq!(fs::read(&output).unwrap(), b"{\"generation\":\"newer\"}\n"); + } + + #[test] + fn program_package_projection_holds_its_writer_lock_through_replacement() { + let root = tempdir("program-projection-locked-replace"); + let output = root.join("program-packages.json"); + fs::write(&output, b"{\"generation\":\"old\"}\n").unwrap(); + let lock_path = program_package_index_lock_path( + &root, + output.file_name().expect("program index filename"), + ); + let mut replace_after_validation = |from: &Path, to: &Path| { + // The replacement callback runs after the target snapshot has been + // validated. A competing generator must still be unable to enter + // its publication boundary at this exact point. + let competing_writer = fs::OpenOptions::new() + .read(true) + .write(true) + .open(&lock_path)?; + assert!( + matches!( + competing_writer.try_lock(), + Err(std::fs::TryLockError::WouldBlock) + ), + "a second writer acquired the publication lock after validation but before replacement", + ); + fs::rename(from, to) + }; + + write_program_package_index_atomically_with( + &output, + b"{\"generation\":\"new\"}\n", + &mut replace_after_validation, + ) + .unwrap(); + + assert_eq!(fs::read(&output).unwrap(), b"{\"generation\":\"new\"}\n"); + } + + #[test] + fn program_package_projection_never_deletes_a_substituted_private_stage() { + let root = tempdir("program-projection-substituted-stage"); + let output = root.join("program-packages.json"); + let displaced_stage = root.join("displaced-stage"); + fs::write(&output, b"{\"generation\":\"old\"}\n").unwrap(); + let mut substitute_stage = |from: &Path, _to: &Path| { + fs::rename(from, &displaced_stage)?; + fs::write(from, b"user replacement")?; + Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "injected failure after stage substitution", + )) + }; + + let error = write_program_package_index_atomically_with( + &output, + b"{\"generation\":\"new\"}\n", + &mut substitute_stage, + ) + .unwrap_err(); + assert!( + error.contains("injected failure after stage substitution") + && error.contains("refusing to remove changed"), + "got: {error}" + ); + assert_eq!(fs::read(&output).unwrap(), b"{\"generation\":\"old\"}\n"); + let transaction_root = fs::read_dir(&root) + .unwrap() + .map(|entry| entry.unwrap().path()) + .find(|path| { + path.file_name() + .unwrap() + .to_string_lossy() + .contains(".index-transaction-") + }) + .unwrap(); + assert_eq!( + fs::read(transaction_root.join("index")).unwrap(), + b"user replacement" + ); + remove_owned_transaction_path(&transaction_root).unwrap(); + } + + #[cfg(unix)] + #[test] + fn program_package_projection_publish_replaces_a_racing_symlink_not_its_target() { + let root = tempdir("program-projection-racing-symlink"); + let output = root.join("program-packages.json"); + let displaced = root.join("old-index"); + let outside = root.join("outside"); + fs::write(&output, b"{\"generation\":\"old\"}\n").unwrap(); + fs::write(&outside, b"outside").unwrap(); + let mut substitute_then_replace = |from: &Path, to: &Path| { + fs::rename(to, &displaced)?; + symlink_file(&outside, to)?; + fs::rename(from, to) + }; + + write_program_package_index_atomically_with( + &output, + b"{\"generation\":\"new\"}\n", + &mut substitute_then_replace, + ) + .unwrap(); + assert_eq!(fs::read(&output).unwrap(), b"{\"generation\":\"new\"}\n"); + assert_eq!(fs::read(&outside).unwrap(), b"outside"); + assert_eq!(fs::read(&displaced).unwrap(), b"{\"generation\":\"old\"}\n"); + } + #[test] + fn program_package_projection_rejects_a_registry_mutation_between_snapshots() { + let registry_root = tempdir("program-projection-registry-mutation"); + let package = registry_root.join("changing-program"); + fs::create_dir_all(&package).unwrap(); + let manifest_path = package.join("package.toml"); + let manifest = |version: &str| { + format!( + r#"kind = "program" +name = "changing-program" +version = "{version}" +depends_on = [] [source] -url = "https://example.test/{name}-{version}.tar.gz" -sha256 = "{:0>64}" +url = "https://example.test/changing-program-{version}.tar.gz" +sha256 = "0000000000000000000000000000000000000000000000000000000000000000" +[license] +spdx = "MIT" +[[outputs]] +name = "changing-command" +wasm = "changing-command.wasm" +"#, + ) + }; + fs::write(&manifest_path, manifest("1.0")).unwrap(); + let registry = Registry { + roots: vec![registry_root.clone()], + }; + let mut mutate = || fs::write(&manifest_path, manifest("2.0")).unwrap(); + + let error = program_package_index_for_root_with(®istry_root, ®istry, &mut mutate) + .unwrap_err(); + assert!( + error.contains("registry changed while generating"), + "got: {error}", + ); + } + #[test] + fn program_package_projection_rejects_cross_package_mirror_collisions() { + let registry = tempdir("program-projection-collision"); + for package in ["first", "second"] { + let directory = registry.join(package); + fs::create_dir_all(&directory).unwrap(); + fs::write( + directory.join("package.toml"), + format!( + r#"kind = "program" +name = "{package}" +version = "1.0" +depends_on = [] +[source] +url = "https://example.test/{package}.tar.gz" +sha256 = "0000000000000000000000000000000000000000000000000000000000000000" [license] -spdx = "TestLicense" +spdx = "MIT" +[[outputs]] +name = "shared" +wasm = "shared.wasm" +"# + ), + ) + .unwrap(); + } -[outputs] -libs = ["lib/lib{name}.a"] -"#, - "" + let selected = Registry { + roots: vec![registry.clone()], + }; + let error = program_package_index_for_root(®istry, &selected).unwrap_err(); + assert!( + error.contains("conflict") && error.contains("shared.wasm"), + "got: {error}" ); - fs::write(lib_dir.join("package.toml"), text).unwrap(); } - fn tempdir(label: &str) -> PathBuf { - let p = std::env::temp_dir() - .join("wpk-xtask-test") - .join(format!("{label}-{}", std::process::id())); - let _ = fs::remove_dir_all(&p); - fs::create_dir_all(&p).unwrap(); - p + #[test] + fn program_package_projection_rejects_file_directory_mirror_collisions() { + let registry = tempdir("program-projection-file-directory-collision"); + let scalar = registry.join("scalar-owner"); + fs::create_dir_all(&scalar).unwrap(); + fs::write( + scalar.join("package.toml"), + r#"kind = "program" +name = "scalar-owner" +version = "1.0" +depends_on = [] +[source] +url = "https://example.test/scalar.tar.gz" +sha256 = "0000000000000000000000000000000000000000000000000000000000000000" +[license] +spdx = "MIT" +[[outputs]] +name = "directory-owner" +wasm = "artifact" +"#, + ) + .unwrap(); + let directory = registry.join("directory-owner"); + fs::create_dir_all(&directory).unwrap(); + fs::write( + directory.join("package.toml"), + r#"kind = "program" +name = "directory-owner" +version = "1.0" +depends_on = [] +[source] +url = "https://example.test/directory.tar.gz" +sha256 = "0000000000000000000000000000000000000000000000000000000000000000" +[license] +spdx = "MIT" +[[outputs]] +name = "first" +wasm = "first.wasm" +[[outputs]] +name = "second" +wasm = "second.wasm" +"#, + ) + .unwrap(); + + let selected = Registry { + roots: vec![registry.clone()], + }; + let error = program_package_index_for_root(®istry, &selected).unwrap_err(); + assert!( + error.contains("programs/wasm32/directory-owner") + && error.contains("programs/wasm32/directory-owner/first.wasm"), + "got: {error}" + ); } fn fixture_git(repo: &Path, args: &[&str]) -> String { @@ -4917,17 +9747,11 @@ index_url = "https://example.test/releases/binaries-abi-v{abi}/index.toml" "#, ) .unwrap(); - let registry = Registry { - roots: vec![root], - }; + let registry = Registry { roots: vec![root] }; - let actual = compute_cache_key_sha_for_package( - &package, - ®istry, - TargetArch::Wasm32, - TEST_ABI, - ) - .unwrap(); + let actual = + compute_cache_key_sha_for_package(&package, ®istry, TargetArch::Wasm32, TEST_ABI) + .unwrap(); // Golden produced by the resolver before build.toml learned the // optional [[git_inputs]] section. Merely adding that schema must not @@ -5045,8 +9869,51 @@ index_url = "https://example.test/releases/binaries-abi-v{abi}/index.toml" let after = global_package_build_input_digests_for(&root, &["toolchain.txt"]).unwrap(); assert_ne!( - before[0].digest, after[0].digest, - "global build input content changes must change its digest" + before[0].digest, after[0].digest, + "global build input content changes must change its digest" + ); + } + + #[test] + fn program_projection_cache_keys_change_with_global_toolchain_inputs() { + let root = tempdir("program-projection-global-build-inputs"); + write_program( + &root, + "global-input-command", + "1.0.0", + &[], + ":", + &[("global-input-command", "global-input-command.wasm")], + ); + let registry = Registry { + roots: vec![root.clone()], + }; + let manifest = registry.load("global-input-command").unwrap(); + let before_inputs = vec![BuildInputDigest { + label: "toolchain.txt".to_string(), + digest: [1; 32], + }]; + let after_inputs = vec![BuildInputDigest { + label: "toolchain.txt".to_string(), + digest: [2; 32], + }]; + + let before = package_context_cache_keys_with_global_toolchain_inputs( + &manifest, + ®istry, + Some(&before_inputs), + ) + .unwrap(); + let after = package_context_cache_keys_with_global_toolchain_inputs( + &manifest, + ®istry, + Some(&after_inputs), + ) + .unwrap(); + + assert_ne!( + before, after, + "the cache identities serialized into program-packages.json must change with global toolchain inputs", ); } @@ -5298,6 +10165,162 @@ index_url = "https://example.test/releases/download/binaries-abi-v{abi}/index.to assert!(err.contains("nope.txt"), "got: {err}"); } + #[test] + fn canonical_build_inputs_follow_first_hit_package_ownership() { + let repo = tempdir("canonical-build-input-main-repo"); + let main_root = repo.join("packages/registry"); + let external_root = tempdir("canonical-build-input-external"); + write(&external_root, "consumer", "1.0.0", &[]); + write(&external_root, "shadowed", "1.0.0", &[]); + write(&main_root, "shadowed", "1.0.0", &[]); + write(&external_root, "shared-helper", "1.0.0", &[]); + write(&main_root, "shared-helper", "1.0.0", &[]); + + fs::write( + external_root.join("shadowed/recipe.txt"), + "external shadow\n", + ) + .unwrap(); + fs::write(main_root.join("shadowed/recipe.txt"), "main shadow\n").unwrap(); + fs::write( + external_root.join("shared-helper/cross-package.txt"), + "external helper\n", + ) + .unwrap(); + fs::write( + main_root.join("shared-helper/cross-package.txt"), + "main helper\n", + ) + .unwrap(); + fs::write( + external_root.join("shared-helper/legacy.txt"), + "legacy external helper\n", + ) + .unwrap(); + + let registry = Registry { + roots: vec![external_root.clone(), main_root.clone()], + }; + let external_shadow = registry.load("shadowed").unwrap(); + let consumer = registry.load("consumer").unwrap(); + + let selected_shadow = resolve_build_input_path_from_repo( + &external_shadow, + ®istry, + "packages/registry/shadowed/recipe.txt", + &repo, + ) + .unwrap(); + assert_eq!( + selected_shadow, + external_root.join("shadowed/recipe.txt"), + "a canonical input owned by a first-hit external package must hash external bytes", + ); + + let selected_cross_package = resolve_build_input_path_from_repo( + &consumer, + ®istry, + "packages/registry/shared-helper/cross-package.txt", + &repo, + ) + .unwrap(); + assert_eq!( + selected_cross_package, + external_root.join("shared-helper/cross-package.txt"), + "canonical cross-package helpers must follow the same ordered first-hit roots", + ); + fs::write( + external_root.join("consumer/build.toml"), + r#" +script_path = "packages/registry/consumer/build-consumer.sh" +inputs = ["packages/registry/shared-helper/cross-package.txt"] +repo_url = "https://example.test/external.git" +commit = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef" +revision = 1 + +[binary] +index_url = "https://example.test/releases/download/binaries-abi-v{abi}/index.toml" +"#, + ) + .unwrap(); + let digests = build_input_digests(&consumer, ®istry).unwrap(); + assert_eq!( + digests[0].digest, + hash_build_input(&selected_cross_package).unwrap(), + "cache-key input hashing must consume the selected external cross-package bytes", + ); + + fs::remove_file(&selected_cross_package).unwrap(); + let missing_selected_input = resolve_build_input_path_from_repo( + &consumer, + ®istry, + "packages/registry/shared-helper/cross-package.txt", + &repo, + ) + .unwrap_err(); + assert!( + missing_selected_input.contains("first-hit registry package") + && missing_selected_input.contains("lower-priority package roots were not consulted"), + "a selected external package must not be completed with a lower package's file: {missing_selected_input}", + ); + + let legacy_registry_relative = resolve_build_input_path_from_repo( + &consumer, + ®istry, + "shared-helper/legacy.txt", + &repo, + ) + .unwrap(); + assert_eq!( + legacy_registry_relative, + external_root.join("shared-helper/legacy.txt"), + "existing registry-relative third-party input paths remain supported", + ); + fs::remove_file(&legacy_registry_relative).unwrap(); + fs::write( + main_root.join("shared-helper/legacy.txt"), + "legacy main helper\n", + ) + .unwrap(); + let missing_legacy_input = resolve_build_input_path_from_repo( + &consumer, + ®istry, + "shared-helper/legacy.txt", + &repo, + ) + .unwrap_err(); + assert!( + missing_legacy_input.contains("first-hit registry package") + && missing_legacy_input.contains("lower-priority package roots were not consulted"), + "legacy registry-relative inputs must use the same package-level selection: {missing_legacy_input}", + ); + + write(&main_root, "main-only-helper", "1.0.0", &[]); + fs::write( + main_root.join("main-only-helper/owned.txt"), + "main selected helper\n", + ) + .unwrap(); + fs::create_dir_all(external_root.join("main-only-helper")).unwrap(); + fs::write( + external_root.join("main-only-helper/owned.txt"), + "unclaimed external directory\n", + ) + .unwrap(); + let selected_main_package = resolve_build_input_path_from_repo( + &consumer, + ®istry, + "packages/registry/main-only-helper/owned.txt", + &repo, + ) + .unwrap(); + assert_eq!( + selected_main_package, + main_root.join("main-only-helper/owned.txt"), + "a higher directory without package.toml cannot shadow the first package manifest in main", + ); + } + #[test] fn compute_cache_key_sha_is_deterministic_across_invocations() { let root = tempdir("ckcs-deterministic"); @@ -6481,47 +11504,204 @@ mkdir -p $WASM_POSIX_DEP_OUT_DIR/lib && touch $WASM_POSIX_DEP_OUT_DIR/lib/libT.a &format!("160000,{first_commit},vendor/submodule"), ], ); - fixture_git(&source, &["commit", "--quiet", "-m", "gitlink"]); - let commit = fixture_git(&source, &["rev-parse", "HEAD"]); - let canonical = root.join("cache/programs/shell"); - fs::create_dir_all(canonical.parent().unwrap()).unwrap(); - let err = ProvisionedGitInputs::provision_declarations( - "shell@0.1.0", - &canonical, - vec![GitBuildInput { - name: "tap".to_string(), - repository: format!("file://{}", source.display()), - commit, - }], - ) - .unwrap_err(); - assert!(err.contains("submodule gitlinks"), "got: {err}"); + fixture_git(&source, &["commit", "--quiet", "-m", "gitlink"]); + let commit = fixture_git(&source, &["rev-parse", "HEAD"]); + let canonical = root.join("cache/programs/shell"); + fs::create_dir_all(canonical.parent().unwrap()).unwrap(); + let err = ProvisionedGitInputs::provision_declarations( + "shell@0.1.0", + &canonical, + vec![GitBuildInput { + name: "tap".to_string(), + repository: format!("file://{}", source.display()), + commit, + }], + ) + .unwrap_err(); + assert!(err.contains("submodule gitlinks"), "got: {err}"); + } + + #[test] + fn ensure_built_fails_when_declared_output_missing() { + let root = tempdir("built-missing-out"); + let cache = tempdir("built-missing-cache"); + write_lib( + &root, + "libC", + "1.0.0", + &[], + // Script succeeds but does NOT create the declared lib. + r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR""#, + r#"[outputs] +libs = ["lib/libC.a"] +"#, + ); + let reg = Registry { + roots: vec![root.clone()], + }; + let m = reg.load("libC").unwrap(); + + let err = + ensure_built(&m, ®, TEST_ARCH, TEST_ABI, &resolve_opts(&cache, None)).unwrap_err(); + assert!(err.contains("not produced"), "got: {err}"); + // Temp dir was cleaned up; canonical path does not exist. + let sha = compute_sha( + &m, + ®, + TEST_ARCH, + TEST_ABI, + &mut BTreeMap::new(), + &mut Vec::new(), + ) + .unwrap(); + let canonical = canonical_path(&cache, &m, TEST_ARCH, &sha); + assert!( + !canonical.exists(), + "canonical cache dir must not exist on failure" + ); + + // No leftover temp dirs in the libs/ directory. + if let Ok(rd) = std::fs::read_dir(cache.join("libs")) { + let leftovers: Vec<_> = rd.collect(); + for l in &leftovers { + let e = l.as_ref().unwrap(); + assert!( + !e.file_name().to_string_lossy().contains(".tmp-"), + "found leftover: {:?}", + e.file_name() + ); + } + } + } + + #[test] + fn ensure_built_fails_when_declared_runtime_file_missing() { + let root = tempdir("built-missing-runtime-file"); + let cache = tempdir("built-missing-runtime-file-cache"); + write_lib( + &root, + "libRuntimeMissing", + "1.0.0", + &[], + r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR/lib" +touch "$WASM_POSIX_DEP_OUT_DIR/lib/libRuntimeMissing.a""#, + r#"[outputs] +libs = ["lib/libRuntimeMissing.a"] +files = ["share/runtime.dat"] +"#, + ); + let reg = Registry { roots: vec![root] }; + let m = reg.load("libRuntimeMissing").unwrap(); + + let err = + ensure_built(&m, ®, TEST_ARCH, TEST_ABI, &resolve_opts(&cache, None)).unwrap_err(); + assert!(err.contains("declared files output"), "got: {err}"); + assert!(err.contains("share/runtime.dat"), "got: {err}"); + } + + #[test] + fn ensure_built_accepts_declared_runtime_file() { + let root = tempdir("built-runtime-file"); + let cache = tempdir("built-runtime-file-cache"); + write_lib( + &root, + "libRuntime", + "1.0.0", + &[], + r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR/lib" "$WASM_POSIX_DEP_OUT_DIR/share" +touch "$WASM_POSIX_DEP_OUT_DIR/lib/libRuntime.a" +printf runtime > "$WASM_POSIX_DEP_OUT_DIR/share/runtime.dat""#, + r#"[outputs] +libs = ["lib/libRuntime.a"] +files = ["share/runtime.dat"] +"#, + ); + let reg = Registry { roots: vec![root] }; + let m = reg.load("libRuntime").unwrap(); + + let path = + ensure_built(&m, ®, TEST_ARCH, TEST_ABI, &resolve_opts(&cache, None)).unwrap(); + assert_eq!( + std::fs::read_to_string(path.join("share/runtime.dat")).unwrap(), + "runtime" + ); + } + + #[test] + fn ensure_built_rejects_declared_runtime_file_directory() { + let root = tempdir("built-runtime-file-directory"); + let cache = tempdir("built-runtime-file-directory-cache"); + write_lib( + &root, + "libRuntimeDirectory", + "1.0.0", + &[], + r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR/lib" "$WASM_POSIX_DEP_OUT_DIR/share/runtime.dat" +touch "$WASM_POSIX_DEP_OUT_DIR/lib/libRuntimeDirectory.a""#, + r#"[outputs] +libs = ["lib/libRuntimeDirectory.a"] +files = ["share/runtime.dat"] +"#, + ); + let reg = Registry { roots: vec![root] }; + let m = reg.load("libRuntimeDirectory").unwrap(); + + let err = + ensure_built(&m, ®, TEST_ARCH, TEST_ABI, &resolve_opts(&cache, None)).unwrap_err(); + assert!(err.contains("must be a regular file"), "got: {err}"); + } + + #[cfg(unix)] + #[test] + fn ensure_built_rejects_declared_runtime_file_symlink_escape() { + let root = tempdir("built-runtime-file-symlink-escape"); + let cache = tempdir("built-runtime-file-symlink-escape-cache"); + let outside = root.join("outside.dat"); + std::fs::write(&outside, b"outside").unwrap(); + write_lib( + &root, + "libRuntimeSymlinkEscape", + "1.0.0", + &[], + &format!( + r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR/lib" "$WASM_POSIX_DEP_OUT_DIR/share" +touch "$WASM_POSIX_DEP_OUT_DIR/lib/libRuntimeSymlinkEscape.a" +ln -s {:?} "$WASM_POSIX_DEP_OUT_DIR/share/runtime.dat""#, + outside + ), + r#"[outputs] +libs = ["lib/libRuntimeSymlinkEscape.a"] +files = ["share/runtime.dat"] +"#, + ); + let reg = Registry { roots: vec![root] }; + let m = reg.load("libRuntimeSymlinkEscape").unwrap(); + + let err = + ensure_built(&m, ®, TEST_ARCH, TEST_ABI, &resolve_opts(&cache, None)).unwrap_err(); + assert!(err.contains("must not be a symlink"), "got: {err}"); } #[test] - fn ensure_built_fails_when_declared_output_missing() { - let root = tempdir("built-missing-out"); - let cache = tempdir("built-missing-cache"); + fn ensure_built_fails_when_script_exits_nonzero() { + let root = tempdir("built-badexit"); + let cache = tempdir("built-badexit-cache"); write_lib( &root, - "libC", + "libD", "1.0.0", &[], - // Script succeeds but does NOT create the declared lib. - r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR""#, + "echo boom >&2\nexit 37", r#"[outputs] -libs = ["lib/libC.a"] +libs = ["lib/libD.a"] "#, ); - let reg = Registry { - roots: vec![root.clone()], - }; - let m = reg.load("libC").unwrap(); + let reg = Registry { roots: vec![root] }; + let m = reg.load("libD").unwrap(); let err = ensure_built(&m, ®, TEST_ARCH, TEST_ABI, &resolve_opts(&cache, None)).unwrap_err(); - assert!(err.contains("not produced"), "got: {err}"); - // Temp dir was cleaned up; canonical path does not exist. + assert!(err.contains("exited"), "got: {err}"); let sha = compute_sha( &m, ®, @@ -6531,154 +11711,329 @@ libs = ["lib/libC.a"] &mut Vec::new(), ) .unwrap(); - let canonical = canonical_path(&cache, &m, TEST_ARCH, &sha); + assert!(!canonical_path(&cache, &m, TEST_ARCH, &sha).exists()); + } + + /// Regression: build-script stdout must NOT leak to xtask's stdout. + /// + /// `cmd_resolve` consumers shell-capture xtask's stdout to read the + /// canonical cache path: + /// `PREFIX="$(cargo run -- build-deps resolve )"`. If the bash + /// subprocess's stdout were inherited (the default), every chatty + /// `echo` in the build script would land on xtask's stdout ahead of + /// the final `println!(path)`, and consumers would capture the + /// build log instead of the path. + /// + /// The `build_into_cache` fix dups xtask's stderr fd into the bash + /// subprocess's stdout. We can't easily intercept `println!` from + /// inside a unit test, but we *can* verify the underlying mechanism + /// works: spawn a child whose stdout is redirected to an OwnedFd + /// (the same `Stdio::from(OwnedFd)` shape `build_into_cache` uses), + /// and confirm the output arrives there — proving libstd routes the + /// child's fd 1 to that fd and not to the test's own stdout. + #[test] + fn build_script_stdout_redirect_to_owned_fd_works() { + use std::io::Read; + use std::os::unix::net::UnixStream; + + // UnixStream::pair gives us two endpoints with full read+write, + // both as `OwnedFd` via Into. We hand the bash subprocess one + // end as its stdout and read from the other. This mirrors the + // production shape: build_into_cache hands bash an OwnedFd + // cloned from xtask's stderr; here we hand bash an OwnedFd + // cloned from a socketpair endpoint. Both flow through the + // same `Stdio::from(OwnedFd)` impl in libstd. + let (parent, child) = UnixStream::pair().expect("socketpair"); + let child_fd: std::os::fd::OwnedFd = child.into(); + let stdio = Stdio::from(child_fd); + + let mut cmd = Command::new("bash"); + cmd.arg("-c"); + cmd.arg("echo BUILD_SCRIPT_STDOUT_LINE_THAT_MUST_NOT_LEAK; echo line2; echo line3"); + cmd.stdout(stdio); + let status = cmd.status().expect("spawn bash"); + assert!(status.success(), "bash exit: {status}"); + + // Read the redirected output. We must drop our local handle on + // the child's write side first so the read end sees EOF — which + // is automatic here: child_fd was moved into Stdio, so once + // the child process exits, the only remaining write reference + // is gone. + drop(cmd); + let mut reader = parent; + let mut buf = String::new(); + reader.read_to_string(&mut buf).expect("read socketpair"); assert!( - !canonical.exists(), - "canonical cache dir must not exist on failure" + buf.contains("BUILD_SCRIPT_STDOUT_LINE_THAT_MUST_NOT_LEAK"), + "redirected stdout missing marker; got: {buf:?}" ); + assert!(buf.contains("line2"), "got: {buf:?}"); + assert!(buf.contains("line3"), "got: {buf:?}"); + } - // No leftover temp dirs in the libs/ directory. - if let Ok(rd) = std::fs::read_dir(cache.join("libs")) { - let leftovers: Vec<_> = rd.collect(); - for l in &leftovers { - let e = l.as_ref().unwrap(); - assert!( - !e.file_name().to_string_lossy().contains(".tmp-"), - "found leftover: {:?}", - e.file_name() - ); - } - } + /// Regression companion: confirm the exact pattern used inside + /// `build_into_cache` — `std::io::stderr().as_fd().try_clone_to_owned()` + /// followed by `Stdio::from(OwnedFd)` — does not panic and does + /// produce a usable Stdio. We can't observe the redirected output + /// here (it would land on the test runner's stderr, which the + /// runner captures and drops on success), but we *can* verify the + /// dup-fd mechanism succeeds and the bash child runs successfully + /// with that Stdio. A regression that broke try_clone_to_owned or + /// the From for Stdio impl would surface here. + #[test] + fn build_into_cache_stderr_dup_pattern_does_not_panic() { + let stderr_dup = std::io::stderr() + .as_fd() + .try_clone_to_owned() + .expect("dup stderr fd"); + let mut cmd = Command::new("bash"); + cmd.arg("-c").arg("echo running >&2; exit 0"); + cmd.stdout(Stdio::from(stderr_dup)); + let status = cmd.status().expect("spawn bash"); + assert!(status.success(), "bash exit: {status}"); } #[test] - fn ensure_built_fails_when_declared_runtime_file_missing() { - let root = tempdir("built-missing-runtime-file"); - let cache = tempdir("built-missing-runtime-file-cache"); + fn local_libs_override_wins() { + let root = tempdir("override-reg"); + let cache = tempdir("override-cache"); + let local = tempdir("override-local"); write_lib( &root, - "libRuntimeMissing", + "libE", "1.0.0", &[], - r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR/lib" -touch "$WASM_POSIX_DEP_OUT_DIR/lib/libRuntimeMissing.a""#, + // If this ran we'd fail the test: override must prevent it. + "exit 99", r#"[outputs] -libs = ["lib/libRuntimeMissing.a"] -files = ["share/runtime.dat"] +libs = ["lib/libE.a"] "#, ); + let override_build = local.join("libE").join("build"); + std::fs::create_dir_all(override_build.join("lib")).unwrap(); + std::fs::write(override_build.join("lib/libE.a"), b"").unwrap(); + let reg = Registry { roots: vec![root] }; - let m = reg.load("libRuntimeMissing").unwrap(); + let m = reg.load("libE").unwrap(); - let err = - ensure_built(&m, ®, TEST_ARCH, TEST_ABI, &resolve_opts(&cache, None)).unwrap_err(); - assert!(err.contains("declared files output"), "got: {err}"); - assert!(err.contains("share/runtime.dat"), "got: {err}"); + let path = ensure_built( + &m, + ®, + TEST_ARCH, + TEST_ABI, + &resolve_opts(&cache, Some(&local)), + ) + .unwrap(); + assert_eq!(path, override_build); } #[test] - fn ensure_built_accepts_declared_runtime_file() { - let root = tempdir("built-runtime-file"); - let cache = tempdir("built-runtime-file-cache"); + fn transitive_deps_are_built_and_exposed_via_env() { + let root = tempdir("transitive-reg"); + let cache = tempdir("transitive-cache"); + + // libFoo produces a stamp header; libBar consumes it via env var. write_lib( &root, - "libRuntime", + "libFoo", "1.0.0", &[], - r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR/lib" "$WASM_POSIX_DEP_OUT_DIR/share" -touch "$WASM_POSIX_DEP_OUT_DIR/lib/libRuntime.a" -printf runtime > "$WASM_POSIX_DEP_OUT_DIR/share/runtime.dat""#, + r#" +mkdir -p "$WASM_POSIX_DEP_OUT_DIR/include" +echo "foo header body" > "$WASM_POSIX_DEP_OUT_DIR/include/foo.h" +"#, + r#"[outputs] +headers = ["include/foo.h"] +"#, + ); + write_lib( + &root, + "libBar", + "1.0.0", + &["libFoo@1.0.0"], + r#" +test -n "${WASM_POSIX_DEP_LIBFOO_DIR:-}" || { echo "LIBFOO_DIR not set" >&2; exit 1; } +test -f "$WASM_POSIX_DEP_LIBFOO_DIR/include/foo.h" || { echo "foo.h missing" >&2; exit 1; } +mkdir -p "$WASM_POSIX_DEP_OUT_DIR/lib" +cp "$WASM_POSIX_DEP_LIBFOO_DIR/include/foo.h" "$WASM_POSIX_DEP_OUT_DIR/lib/libBar.a" +"#, + r#"[outputs] +libs = ["lib/libBar.a"] +"#, + ); + let reg = Registry { roots: vec![root] }; + let bar = reg.load("libBar").unwrap(); + let bar_path = + ensure_built(&bar, ®, TEST_ARCH, TEST_ABI, &resolve_opts(&cache, None)).unwrap(); + + let pseudo = std::fs::read_to_string(bar_path.join("lib/libBar.a")).unwrap(); + assert_eq!(pseudo.trim(), "foo header body"); + } + + #[test] + fn env_key_canonicalises_hyphens_and_case() { + assert_eq!(env_key("libcurl"), "LIBCURL"); + assert_eq!(env_key("zlib-ng"), "ZLIB_NG"); + assert_eq!(env_key("Foo-Bar-Baz"), "FOO_BAR_BAZ"); + } + + // --- pkgconfig / libtool archive path rewriting --- + // + // autoconf bakes `--prefix` into generated `.pc` / `.la` files at + // configure time. Our build scripts configure with + // `--prefix=$WASM_POSIX_DEP_OUT_DIR` — the temp dir. After the + // atomic rename into the canonical cache path, those baked-in + // strings point at a temp directory that no longer exists. The + // resolver must rewrite them before (or as part of) the install + // so downstream `pkg-config` / `libtool` consumers see a valid + // path. These tests pin that behaviour. + + #[test] + fn pkgconfig_prefix_is_rewritten_to_canonical_path() { + let root = tempdir("pc-rewrite-reg"); + let cache = tempdir("pc-rewrite-cache"); + write_lib( + &root, + "libPc", + "1.0.0", + &[], + // Bakes `prefix=$WASM_POSIX_DEP_OUT_DIR` into the .pc + // file — the same mistake autoconf makes. + r#" +mkdir -p "$WASM_POSIX_DEP_OUT_DIR/lib/pkgconfig" +touch "$WASM_POSIX_DEP_OUT_DIR/lib/libPc.a" +cat > "$WASM_POSIX_DEP_OUT_DIR/lib/pkgconfig/libPc.pc" <` substring; got:\n{pc}" ); } #[test] - fn ensure_built_rejects_declared_runtime_file_directory() { - let root = tempdir("built-runtime-file-directory"); - let cache = tempdir("built-runtime-file-directory-cache"); + fn libtool_archive_libdir_is_rewritten_to_canonical_path() { + let root = tempdir("la-rewrite-reg"); + let cache = tempdir("la-rewrite-cache"); write_lib( &root, - "libRuntimeDirectory", + "libLa", "1.0.0", &[], - r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR/lib" "$WASM_POSIX_DEP_OUT_DIR/share/runtime.dat" -touch "$WASM_POSIX_DEP_OUT_DIR/lib/libRuntimeDirectory.a""#, + // libtool writes `libdir='/lib'` — same problem. + r#" +mkdir -p "$WASM_POSIX_DEP_OUT_DIR/lib" +touch "$WASM_POSIX_DEP_OUT_DIR/lib/libLa.a" +cat > "$WASM_POSIX_DEP_OUT_DIR/lib/libLa.la" <` substring; got:\n{la}" + ); } - #[cfg(unix)] #[test] - fn ensure_built_rejects_declared_runtime_file_symlink_escape() { - let root = tempdir("built-runtime-file-symlink-escape"); - let cache = tempdir("built-runtime-file-symlink-escape-cache"); - let outside = root.join("outside.dat"); - std::fs::write(&outside, b"outside").unwrap(); + fn pkgconfig_symlinks_survive_the_rewrite() { + // libpng and ncurses install `lib{png,png16}.pc` plus a + // `libpng.pc → libpng16.pc` symlink. The rewrite must not + // follow the symlink (that would rewrite the real file + // twice) and must not turn the symlink into a regular file. + let root = tempdir("pc-symlink-reg"); + let cache = tempdir("pc-symlink-cache"); write_lib( &root, - "libRuntimeSymlinkEscape", + "libSym", "1.0.0", &[], - &format!( - r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR/lib" "$WASM_POSIX_DEP_OUT_DIR/share" -touch "$WASM_POSIX_DEP_OUT_DIR/lib/libRuntimeSymlinkEscape.a" -ln -s {:?} "$WASM_POSIX_DEP_OUT_DIR/share/runtime.dat""#, - outside - ), + r#" +mkdir -p "$WASM_POSIX_DEP_OUT_DIR/lib/pkgconfig" +touch "$WASM_POSIX_DEP_OUT_DIR/lib/libSym.a" +cat > "$WASM_POSIX_DEP_OUT_DIR/lib/pkgconfig/libSym1.pc" <&2\nexit 37", - r#"[outputs] -libs = ["lib/libD.a"] -"#, - ); + fn canonical_path_layout() { + let root = tempdir("cache-path"); + write(&root, "zlib", "1.3.1", &[]); let reg = Registry { roots: vec![root] }; - let m = reg.load("libD").unwrap(); - - let err = - ensure_built(&m, ®, TEST_ARCH, TEST_ABI, &resolve_opts(&cache, None)).unwrap_err(); - assert!(err.contains("exited"), "got: {err}"); + let m = reg.load("zlib").unwrap(); let sha = compute_sha( &m, ®, @@ -6688,1094 +12043,1297 @@ libs = ["lib/libD.a"] &mut Vec::new(), ) .unwrap(); - assert!(!canonical_path(&cache, &m, TEST_ARCH, &sha).exists()); + let cache = PathBuf::from("/tmp/testcache"); + let path = canonical_path(&cache, &m, TEST_ARCH, &sha); + + let parent = path.parent().unwrap(); + assert_eq!(parent, cache.join("libs")); + let name = path.file_name().unwrap().to_string_lossy().into_owned(); + // The path includes the arch segment between revN and the full cache key. + assert!(name.starts_with("zlib-1.3.1-rev1-wasm32-"), "got {name}"); + let key = name.rsplit('-').next().unwrap(); + assert_eq!(key.len(), 64); + assert!(key.chars().all(|c| c.is_ascii_hexdigit())); + assert_eq!(key, hex(&sha)); } - /// Regression: build-script stdout must NOT leak to xtask's stdout. - /// - /// `cmd_resolve` consumers shell-capture xtask's stdout to read the - /// canonical cache path: - /// `PREFIX="$(cargo run -- build-deps resolve )"`. If the bash - /// subprocess's stdout were inherited (the default), every chatty - /// `echo` in the build script would land on xtask's stdout ahead of - /// the final `println!(path)`, and consumers would capture the - /// build log instead of the path. - /// - /// The `build_into_cache` fix dups xtask's stderr fd into the bash - /// subprocess's stdout. We can't easily intercept `println!` from - /// inside a unit test, but we *can* verify the underlying mechanism - /// works: spawn a child whose stdout is redirected to an OwnedFd - /// (the same `Stdio::from(OwnedFd)` shape `build_into_cache` uses), - /// and confirm the output arrives there — proving libstd routes the - /// child's fd 1 to that fd and not to the test's own stdout. #[test] - fn build_script_stdout_redirect_to_owned_fd_works() { - use std::io::Read; - use std::os::unix::net::UnixStream; - - // UnixStream::pair gives us two endpoints with full read+write, - // both as `OwnedFd` via Into. We hand the bash subprocess one - // end as its stdout and read from the other. This mirrors the - // production shape: build_into_cache hands bash an OwnedFd - // cloned from xtask's stderr; here we hand bash an OwnedFd - // cloned from a socketpair endpoint. Both flow through the - // same `Stdio::from(OwnedFd)` impl in libstd. - let (parent, child) = UnixStream::pair().expect("socketpair"); - let child_fd: std::os::fd::OwnedFd = child.into(); - let stdio = Stdio::from(child_fd); - - let mut cmd = Command::new("bash"); - cmd.arg("-c"); - cmd.arg("echo BUILD_SCRIPT_STDOUT_LINE_THAT_MUST_NOT_LEAK; echo line2; echo line3"); - cmd.stdout(stdio); - let status = cmd.status().expect("spawn bash"); - assert!(status.success(), "bash exit: {status}"); - - // Read the redirected output. We must drop our local handle on - // the child's write side first so the read end sees EOF — which - // is automatic here: child_fd was moved into Stdio, so once - // the child process exits, the only remaining write reference - // is gone. - drop(cmd); - let mut reader = parent; - let mut buf = String::new(); - reader.read_to_string(&mut buf).expect("read socketpair"); - assert!( - buf.contains("BUILD_SCRIPT_STDOUT_LINE_THAT_MUST_NOT_LEAK"), - "redirected stdout missing marker; got: {buf:?}" + fn source_kind_canonical_path_omits_arch() { + let dir = tempdir("source-canonical"); + let m = parse_source_manifest(&dir); + let sha = [0u8; 32]; + let cache = PathBuf::from("/cache"); + let path = canonical_path(&cache, &m, TargetArch::Wasm32, &sha); + assert_eq!( + path, + PathBuf::from(format!( + "/cache/sources/pcre2-source-10.42-rev1-{}", + hex(&sha) + )) ); - assert!(buf.contains("line2"), "got: {buf:?}"); - assert!(buf.contains("line3"), "got: {buf:?}"); } - /// Regression companion: confirm the exact pattern used inside - /// `build_into_cache` — `std::io::stderr().as_fd().try_clone_to_owned()` - /// followed by `Stdio::from(OwnedFd)` — does not panic and does - /// produce a usable Stdio. We can't observe the redirected output - /// here (it would land on the test runner's stderr, which the - /// runner captures and drops on success), but we *can* verify the - /// dup-fd mechanism succeeds and the bash child runs successfully - /// with that Stdio. A regression that broke try_clone_to_owned or - /// the From for Stdio impl would surface here. #[test] - fn build_into_cache_stderr_dup_pattern_does_not_panic() { - let stderr_dup = std::io::stderr() - .as_fd() - .try_clone_to_owned() - .expect("dup stderr fd"); - let mut cmd = Command::new("bash"); - cmd.arg("-c").arg("echo running >&2; exit 0"); - cmd.stdout(Stdio::from(stderr_dup)); - let status = cmd.status().expect("spawn bash"); - assert!(status.success(), "bash exit: {status}"); + fn canonical_path_does_not_alias_keys_with_the_same_archive_prefix() { + let dir = tempdir("canonical-full-key"); + let m = parse_source_manifest(&dir); + let mut first = [0u8; 32]; + let mut second = [0u8; 32]; + first[..4].copy_from_slice(&[0xde, 0xad, 0xbe, 0xef]); + second[..4].copy_from_slice(&[0xde, 0xad, 0xbe, 0xef]); + first[31] = 1; + second[31] = 2; + + let first_path = canonical_path(Path::new("/cache"), &m, TEST_ARCH, &first); + let second_path = canonical_path(Path::new("/cache"), &m, TEST_ARCH, &second); + + assert_ne!(first_path, second_path); + assert!(first_path.to_string_lossy().ends_with(&hex(&first))); + assert!(second_path.to_string_lossy().ends_with(&hex(&second))); + } + + fn parse_source_manifest(dir: &Path) -> DepsManifest { + let text = r#" +kind = "source" +name = "pcre2-source" +version = "10.42" + +[source] +url = "https://example.test/pcre2.tar.bz2" +sha256 = "0000000000000000000000000000000000000000000000000000000000000000" + +[license] +spdx = "BSD-3-Clause" +"#; + DepsManifest::parse(text, dir.to_path_buf()).unwrap() } #[test] - fn local_libs_override_wins() { - let root = tempdir("override-reg"); - let cache = tempdir("override-cache"); - let local = tempdir("override-local"); + fn resolve_with_arch_wasm64_uses_different_cache_path() { + let root = tempdir("arch-flag"); + let cache = tempdir("arch-cache"); write_lib( &root, - "libE", + "libA", "1.0.0", &[], - // If this ran we'd fail the test: override must prevent it. - "exit 99", + r#" +mkdir -p "$WASM_POSIX_DEP_OUT_DIR/lib" +touch "$WASM_POSIX_DEP_OUT_DIR/lib/libA.a" +"#, r#"[outputs] -libs = ["lib/libE.a"] +libs = ["lib/libA.a"] "#, ); - let override_build = local.join("libE").join("build"); - std::fs::create_dir_all(override_build.join("lib")).unwrap(); - std::fs::write(override_build.join("lib/libE.a"), b"").unwrap(); - let reg = Registry { roots: vec![root] }; - let m = reg.load("libE").unwrap(); + let m = reg.load("libA").unwrap(); - let path = ensure_built( + let p32 = ensure_built( &m, ®, - TEST_ARCH, + TargetArch::Wasm32, TEST_ABI, - &resolve_opts(&cache, Some(&local)), + &resolve_opts(&cache, None), ) .unwrap(); - assert_eq!(path, override_build); + let p64 = ensure_built( + &m, + ®, + TargetArch::Wasm64, + TEST_ABI, + &resolve_opts(&cache, None), + ) + .unwrap(); + assert_ne!(p32, p64); + assert!( + p32.to_string_lossy().contains("wasm32"), + "wasm32 path missing arch segment: {}", + p32.display() + ); + assert!( + p64.to_string_lossy().contains("wasm64"), + "wasm64 path missing arch segment: {}", + p64.display() + ); } #[test] - fn transitive_deps_are_built_and_exposed_via_env() { - let root = tempdir("transitive-reg"); - let cache = tempdir("transitive-cache"); + fn parse_target_arch_accepts_known_values() { + assert_eq!(parse_target_arch("wasm32").unwrap(), TargetArch::Wasm32); + assert_eq!(parse_target_arch("wasm64").unwrap(), TargetArch::Wasm64); + } - // libFoo produces a stamp header; libBar consumes it via env var. + #[test] + fn parse_target_arch_rejects_unknown_values() { + let err = parse_target_arch("x86_64").unwrap_err(); + assert!(err.contains("x86_64"), "got: {err}"); + assert!( + err.contains("wasm32") && err.contains("wasm64"), + "error should list valid options; got: {err}" + ); + } + + /// `WASM_POSIX_DEP_PKG_CONFIG_PATH` is a colon-joined list of every + /// transitively-resolved lib's `lib/pkgconfig/` directory. Consumers + /// (e.g., wget, git) prepend it to `PKG_CONFIG_PATH` so pkg-config + /// can chase `Requires.private` chains across the whole dep graph + /// without each consumer hand-rolling per-dep search paths. + /// + /// The test sets up a 3-level chain: + /// libFoo (no deps, ships pkgconfig) + /// <- libBar (deps libFoo, ships pkgconfig) + /// <- libBaz (deps libBar — libFoo is transitive only) + /// + /// libBaz's build script asserts that `WASM_POSIX_DEP_PKG_CONFIG_PATH` + /// contains BOTH libFoo's and libBar's pkgconfig dirs. Order is not + /// fixed — we match either ordering via case patterns. + #[test] + fn pkg_config_path_includes_transitive_lib_pkgconfig() { + let root = tempdir("pcpath-reg"); + let cache = tempdir("pcpath-cache"); + + // libFoo: produces a .pc file. No deps. write_lib( &root, "libFoo", "1.0.0", &[], r#" +mkdir -p "$WASM_POSIX_DEP_OUT_DIR/lib/pkgconfig" mkdir -p "$WASM_POSIX_DEP_OUT_DIR/include" -echo "foo header body" > "$WASM_POSIX_DEP_OUT_DIR/include/foo.h" +touch "$WASM_POSIX_DEP_OUT_DIR/include/foo.h" +cat > "$WASM_POSIX_DEP_OUT_DIR/lib/pkgconfig/libFoo.pc" <<'PCEOF' +Name: libFoo +Version: 1.0.0 +PCEOF "#, r#"[outputs] headers = ["include/foo.h"] +pkgconfig = ["lib/pkgconfig/libFoo.pc"] "#, ); + + // libBar: depends on libFoo, also produces a .pc file. write_lib( &root, "libBar", "1.0.0", &["libFoo@1.0.0"], r#" -test -n "${WASM_POSIX_DEP_LIBFOO_DIR:-}" || { echo "LIBFOO_DIR not set" >&2; exit 1; } -test -f "$WASM_POSIX_DEP_LIBFOO_DIR/include/foo.h" || { echo "foo.h missing" >&2; exit 1; } -mkdir -p "$WASM_POSIX_DEP_OUT_DIR/lib" -cp "$WASM_POSIX_DEP_LIBFOO_DIR/include/foo.h" "$WASM_POSIX_DEP_OUT_DIR/lib/libBar.a" -"#, - r#"[outputs] -libs = ["lib/libBar.a"] -"#, - ); - let reg = Registry { roots: vec![root] }; - let bar = reg.load("libBar").unwrap(); - let bar_path = - ensure_built(&bar, ®, TEST_ARCH, TEST_ABI, &resolve_opts(&cache, None)).unwrap(); - - let pseudo = std::fs::read_to_string(bar_path.join("lib/libBar.a")).unwrap(); - assert_eq!(pseudo.trim(), "foo header body"); - } - - #[test] - fn env_key_canonicalises_hyphens_and_case() { - assert_eq!(env_key("libcurl"), "LIBCURL"); - assert_eq!(env_key("zlib-ng"), "ZLIB_NG"); - assert_eq!(env_key("Foo-Bar-Baz"), "FOO_BAR_BAZ"); - } - - // --- pkgconfig / libtool archive path rewriting --- - // - // autoconf bakes `--prefix` into generated `.pc` / `.la` files at - // configure time. Our build scripts configure with - // `--prefix=$WASM_POSIX_DEP_OUT_DIR` — the temp dir. After the - // atomic rename into the canonical cache path, those baked-in - // strings point at a temp directory that no longer exists. The - // resolver must rewrite them before (or as part of) the install - // so downstream `pkg-config` / `libtool` consumers see a valid - // path. These tests pin that behaviour. - - #[test] - fn pkgconfig_prefix_is_rewritten_to_canonical_path() { - let root = tempdir("pc-rewrite-reg"); - let cache = tempdir("pc-rewrite-cache"); - write_lib( - &root, - "libPc", - "1.0.0", - &[], - // Bakes `prefix=$WASM_POSIX_DEP_OUT_DIR` into the .pc - // file — the same mistake autoconf makes. - r#" mkdir -p "$WASM_POSIX_DEP_OUT_DIR/lib/pkgconfig" -touch "$WASM_POSIX_DEP_OUT_DIR/lib/libPc.a" -cat > "$WASM_POSIX_DEP_OUT_DIR/lib/pkgconfig/libPc.pc" < "$WASM_POSIX_DEP_OUT_DIR/lib/pkgconfig/libBar.pc" <<'PCEOF' +Name: libBar Version: 1.0.0 -Libs: -L\${libdir} -lPc +Requires: libFoo PCEOF "#, r#"[outputs] -libs = ["lib/libPc.a"] -pkgconfig = ["lib/pkgconfig/libPc.pc"] +headers = ["include/bar.h"] +pkgconfig = ["lib/pkgconfig/libBar.pc"] "#, ); - let reg = Registry { roots: vec![root] }; - let m = reg.load("libPc").unwrap(); - - let canonical = - ensure_built(&m, ®, TEST_ARCH, TEST_ABI, &resolve_opts(&cache, None)).unwrap(); - - let pc = std::fs::read_to_string(canonical.join("lib/pkgconfig/libPc.pc")).unwrap(); - assert!( - pc.contains(&format!("prefix={}", canonical.display())), - "pkgconfig prefix must point at the canonical cache path; got:\n{pc}" - ); - assert!( - !pc.contains(".tmp-"), - "pkgconfig must not contain any `.tmp-` substring; got:\n{pc}" - ); - } - #[test] - fn libtool_archive_libdir_is_rewritten_to_canonical_path() { - let root = tempdir("la-rewrite-reg"); - let cache = tempdir("la-rewrite-cache"); + // libBaz: depends on libBar (libFoo is transitive). Build script + // asserts WASM_POSIX_DEP_PKG_CONFIG_PATH contains both libFoo + // and libBar pkgconfig dirs (order-insensitive). write_lib( &root, - "libLa", + "libBaz", "1.0.0", - &[], - // libtool writes `libdir='/lib'` — same problem. + &["libBar@1.0.0"], r#" +test -n "${WASM_POSIX_DEP_PKG_CONFIG_PATH:-}" || { + echo "WASM_POSIX_DEP_PKG_CONFIG_PATH unset" >&2 + exit 1 +} +case "$WASM_POSIX_DEP_PKG_CONFIG_PATH" in + *libFoo*lib/pkgconfig*libBar*lib/pkgconfig*) : ;; + *libBar*lib/pkgconfig*libFoo*lib/pkgconfig*) : ;; + *) + echo "WASM_POSIX_DEP_PKG_CONFIG_PATH does not contain both libFoo and libBar pkgconfig dirs:" >&2 + echo " $WASM_POSIX_DEP_PKG_CONFIG_PATH" >&2 + exit 1 + ;; +esac mkdir -p "$WASM_POSIX_DEP_OUT_DIR/lib" -touch "$WASM_POSIX_DEP_OUT_DIR/lib/libLa.a" -cat > "$WASM_POSIX_DEP_OUT_DIR/lib/libLa.la" <` substring; got:\n{la}" - ); + let reg = Registry { roots: vec![root] }; + let m = reg.load("libBaz").unwrap(); + ensure_built(&m, ®, TEST_ARCH, TEST_ABI, &resolve_opts(&cache, None)).unwrap(); } - #[test] - fn pkgconfig_symlinks_survive_the_rewrite() { - // libpng and ncurses install `lib{png,png16}.pc` plus a - // `libpng.pc → libpng16.pc` symlink. The rewrite must not - // follow the symlink (that would rewrite the real file - // twice) and must not turn the symlink into a regular file. - let root = tempdir("pc-symlink-reg"); - let cache = tempdir("pc-symlink-cache"); + /// Libs without a `lib/pkgconfig/` directory (e.g., ncurses ships a + /// `.pc` file optionally; some libs ship none at all) must be SKIPPED + /// when composing `WASM_POSIX_DEP_PKG_CONFIG_PATH`. Otherwise we'd + /// hand pkg-config a list of nonexistent search paths, which clutters + /// diagnostics and (for some pkg-config versions) errors out. + #[test] + fn pkg_config_path_skips_libs_without_pkgconfig_dir() { + let root = tempdir("pcpath-skip-reg"); + let cache = tempdir("pcpath-skip-cache"); + + // libNoPc: ships only a header — no pkgconfig. write_lib( &root, - "libSym", + "libNoPc", "1.0.0", &[], r#" -mkdir -p "$WASM_POSIX_DEP_OUT_DIR/lib/pkgconfig" -touch "$WASM_POSIX_DEP_OUT_DIR/lib/libSym.a" -cat > "$WASM_POSIX_DEP_OUT_DIR/lib/pkgconfig/libSym1.pc" <&2 + echo " $WASM_POSIX_DEP_PKG_CONFIG_PATH" >&2 + exit 1 + ;; +esac +mkdir -p "$WASM_POSIX_DEP_OUT_DIR/lib" +touch "$WASM_POSIX_DEP_OUT_DIR/lib/libConsumer.a" +"#, + r#"[outputs] +libs = ["lib/libConsumer.a"] +"#, ); - assert!(!real.contains(".tmp-")); - // Reading via the symlink produces the same (rewritten) text. - let via_link = std::fs::read_to_string(canonical.join("lib/pkgconfig/libSym.pc")).unwrap(); - assert_eq!(real, via_link); + let reg = Registry { roots: vec![root] }; + let m = reg.load("libConsumer").unwrap(); + ensure_built(&m, ®, TEST_ARCH, TEST_ABI, &resolve_opts(&cache, None)).unwrap(); + } - // The symlink is still a symlink — we didn't overwrite it - // with a regular file during the rewrite. - let meta = std::fs::symlink_metadata(canonical.join("lib/pkgconfig/libSym.pc")).unwrap(); - assert!( - meta.file_type().is_symlink(), - "pkgconfig symlink must survive as a symlink after rewrite" - ); + // --- Remote-fetch integration tests (Task A.9) ------------------- + // + // These exercise the full `[binary]` resolution path with a + // hand-crafted .tar.zst archive served over a `file://` URL — + // the same code path as production HTTP fetches, but without a + // real network or HTTP server. Each test verifies one outcome: + // + // * happy path — archive is sha-, arch-, abi-, cache_key-valid → + // resolver installs without invoking the build script; + // * sha mismatch / arch mismatch / abi mismatch / cache_key + // mismatch — resolver logs and falls through to source build. + // + // The build script writes a sentinel `via-build` file. Its presence + // in the canonical cache means the source build ran; its absence + // (with the artifacts otherwise installed) means the remote fetch + // succeeded. + + fn sha256_hex(bytes: &[u8]) -> String { + let mut h = Sha256::new(); + h.update(bytes); + let out: [u8; 32] = h.finalize().into(); + hex(&out) } - #[test] - fn canonical_path_layout() { - let root = tempdir("cache-path"); - write(&root, "zlib", "1.3.1", &[]); - let reg = Registry { roots: vec![root] }; - let m = reg.load("zlib").unwrap(); - let sha = compute_sha( - &m, - ®, - TEST_ARCH, - TEST_ABI, - &mut BTreeMap::new(), - &mut Vec::new(), - ) - .unwrap(); - let cache = PathBuf::from("/tmp/testcache"); - let path = canonical_path(&cache, &m, TEST_ARCH, &sha); + /// Build the archived `manifest.toml` text for a library named + /// `name`. `arch` and `abi_versions` and `cache_key_sha` populate + /// the `[compatibility]` block. Output declaration is `lib/out.a` + /// to match `write_lib_with_build_toml`. + fn archived_manifest_text( + name: &str, + arch: &str, + abi_versions: &[u32], + cache_key_sha: &str, + ) -> String { + let abi_csv = abi_versions + .iter() + .map(|v| v.to_string()) + .collect::>() + .join(", "); + format!( + r#" +kind = "library" +name = "{name}" +version = "1.0.0" +revision = 1 +depends_on = [] - let parent = path.parent().unwrap(); - assert_eq!(parent, cache.join("libs")); - let name = path.file_name().unwrap().to_string_lossy().into_owned(); - // The path includes the arch segment between revN and the full cache key. - assert!(name.starts_with("zlib-1.3.1-rev1-wasm32-"), "got {name}"); - let key = name.rsplit('-').next().unwrap(); - assert_eq!(key.len(), 64); - assert!(key.chars().all(|c| c.is_ascii_hexdigit())); - assert_eq!(key, hex(&sha)); +[source] +url = "https://example.test/{name}-1.0.0.tar.gz" +sha256 = "{:0>64}" + +[license] +spdx = "TestLicense" + +[outputs] +libs = ["lib/out.a"] + +[compatibility] +target_arch = "{arch}" +abi_versions = [{abi_csv}] +cache_key_sha = "{cache_key_sha}" +"#, + "" + ) } - #[test] - fn source_kind_canonical_path_omits_arch() { - let dir = tempdir("source-canonical"); - let m = parse_source_manifest(&dir); - let sha = [0u8; 32]; - let cache = PathBuf::from("/cache"); - let path = canonical_path(&cache, &m, TargetArch::Wasm32, &sha); - assert_eq!( - path, - PathBuf::from(format!( - "/cache/sources/pcre2-source-10.42-rev1-{}", - hex(&sha) - )) - ); + fn archived_program_manifest_text( + name: &str, + output_name: &str, + output_wasm: &str, + arch: &str, + abi_versions: &[u32], + cache_key_sha: &str, + ) -> String { + let abi_csv = abi_versions + .iter() + .map(|v| v.to_string()) + .collect::>() + .join(", "); + format!( + r#" +kind = "program" +name = "{name}" +version = "1.0.0" +revision = 1 +depends_on = [] + +[source] +url = "https://example.test/{name}-1.0.0.tar.gz" +sha256 = "{:0>64}" + +[license] +spdx = "TestLicense" + +[[outputs]] +name = "{output_name}" +wasm = "{output_wasm}" + +[compatibility] +target_arch = "{arch}" +abi_versions = [{abi_csv}] +cache_key_sha = "{cache_key_sha}" +"#, + "" + ) } - #[test] - fn canonical_path_does_not_alias_keys_with_the_same_archive_prefix() { - let dir = tempdir("canonical-full-key"); - let m = parse_source_manifest(&dir); - let mut first = [0u8; 32]; - let mut second = [0u8; 32]; - first[..4].copy_from_slice(&[0xde, 0xad, 0xbe, 0xef]); - second[..4].copy_from_slice(&[0xde, 0xad, 0xbe, 0xef]); - first[31] = 1; - second[31] = 2; + fn archived_program_runtime_manifest_text( + name: &str, + arch: &str, + abi_versions: &[u32], + cache_key_sha: &str, + ) -> String { + let abi_csv = abi_versions + .iter() + .map(|value| value.to_string()) + .collect::>() + .join(", "); + format!( + r#" +kind = "program" +name = "{name}" +version = "1.0.0" +revision = 1 +depends_on = [] - let first_path = canonical_path(Path::new("/cache"), &m, TEST_ARCH, &first); - let second_path = canonical_path(Path::new("/cache"), &m, TEST_ARCH, &second); +[source] +url = "https://example.test/{name}.tar.gz" +sha256 = "{:0>64}" + +[license] +spdx = "MIT" + +[[outputs]] +name = "{name}" +wasm = "{name}.wasm" - assert_ne!(first_path, second_path); - assert!(first_path.to_string_lossy().ends_with(&hex(&first))); - assert!(second_path.to_string_lossy().ends_with(&hex(&second))); +[[runtime_files]] +artifact = "icu.dat" +guest_path = "/usr/lib/php/icu.dat" + +[compatibility] +target_arch = "{arch}" +abi_versions = [{abi_csv}] +cache_key_sha = "{cache_key_sha}" +"#, + "" + ) } - fn parse_source_manifest(dir: &Path) -> DepsManifest { - let text = r#" -kind = "source" -name = "pcre2-source" -version = "10.42" + /// Write a source `package.toml` + sibling `build.toml` for + /// index-lookup-based resolution tests. The `build.toml`'s + /// `[binary]` block points at `index_url` (typically a `file://` + /// URL to a staged `index.toml`). The build script drops a + /// `via-build` sentinel so fall-through tests can detect that the + /// source build ran instead of the index fetch. + fn write_lib_with_build_toml(root: &Path, name: &str, index_url: &str) { + let lib_dir = root.join(name); + std::fs::create_dir_all(&lib_dir).unwrap(); + + let deps_toml = format!( + r#" +kind = "library" +name = "{name}" +version = "1.0.0" +depends_on = [] [source] -url = "https://example.test/pcre2.tar.bz2" -sha256 = "0000000000000000000000000000000000000000000000000000000000000000" +url = "https://example.test/{name}-1.0.0.tar.gz" +sha256 = "{:0>64}" [license] -spdx = "BSD-3-Clause" -"#; - DepsManifest::parse(text, dir.to_path_buf()).unwrap() - } +spdx = "TestLicense" - #[test] - fn resolve_with_arch_wasm64_uses_different_cache_path() { - let root = tempdir("arch-flag"); - let cache = tempdir("arch-cache"); - write_lib( - &root, - "libA", - "1.0.0", - &[], - r#" -mkdir -p "$WASM_POSIX_DEP_OUT_DIR/lib" -touch "$WASM_POSIX_DEP_OUT_DIR/lib/libA.a" -"#, - r#"[outputs] -libs = ["lib/libA.a"] +[outputs] +libs = ["lib/out.a"] "#, + "" ); - let reg = Registry { roots: vec![root] }; - let m = reg.load("libA").unwrap(); + std::fs::write(lib_dir.join("package.toml"), deps_toml).unwrap(); - let p32 = ensure_built( - &m, - ®, - TargetArch::Wasm32, - TEST_ABI, - &resolve_opts(&cache, None), - ) - .unwrap(); - let p64 = ensure_built( - &m, - ®, - TargetArch::Wasm64, - TEST_ABI, - &resolve_opts(&cache, None), - ) - .unwrap(); - assert_ne!(p32, p64); - assert!( - p32.to_string_lossy().contains("wasm32"), - "wasm32 path missing arch segment: {}", - p32.display() - ); - assert!( - p64.to_string_lossy().contains("wasm64"), - "wasm64 path missing arch segment: {}", - p64.display() + let build_toml = format!( + r#" +script_path = "packages/registry/{name}/build-{name}.sh" +repo_url = "https://example.test/repo.git" +commit = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef" + +[binary] +index_url = "{index_url}" +"# ); - } + std::fs::write(lib_dir.join("build.toml"), build_toml).unwrap(); - #[test] - fn parse_target_arch_accepts_known_values() { - assert_eq!(parse_target_arch("wasm32").unwrap(), TargetArch::Wasm32); - assert_eq!(parse_target_arch("wasm64").unwrap(), TargetArch::Wasm64); + let script = "#!/bin/bash\nset -euo pipefail\n\ +mkdir -p \"$WASM_POSIX_DEP_OUT_DIR/lib\"\n\ +echo BUILD > \"$WASM_POSIX_DEP_OUT_DIR/lib/out.a\"\n\ +touch \"$WASM_POSIX_DEP_OUT_DIR/via-build\"\n"; + let script_path = lib_dir.join(format!("build-{name}.sh")); + std::fs::write(&script_path, script).unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut p = std::fs::metadata(&script_path).unwrap().permissions(); + p.set_mode(0o755); + std::fs::set_permissions(&script_path, p).unwrap(); + } } - #[test] - fn parse_target_arch_rejects_unknown_values() { - let err = parse_target_arch("x86_64").unwrap_err(); - assert!(err.contains("x86_64"), "got: {err}"); - assert!( - err.contains("wasm32") && err.contains("wasm64"), - "error should list valid options; got: {err}" + fn write_program_build_toml(root: &Path, name: &str, index_url: &str) { + let dir = root.join(name); + let build_toml = format!( + r#" +script_path = "packages/registry/{name}/build-{name}.sh" +repo_url = "https://example.test/repo.git" +commit = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef" + +[binary] +index_url = "{index_url}" +"# ); + std::fs::write(dir.join("build.toml"), build_toml).unwrap(); } - /// `WASM_POSIX_DEP_PKG_CONFIG_PATH` is a colon-joined list of every - /// transitively-resolved lib's `lib/pkgconfig/` directory. Consumers - /// (e.g., wget, git) prepend it to `PKG_CONFIG_PATH` so pkg-config - /// can chase `Requires.private` chains across the whole dep graph - /// without each consumer hand-rolling per-dep search paths. - /// - /// The test sets up a 3-level chain: - /// libFoo (no deps, ships pkgconfig) - /// <- libBar (deps libFoo, ships pkgconfig) - /// <- libBaz (deps libBar — libFoo is transitive only) - /// - /// libBaz's build script asserts that `WASM_POSIX_DEP_PKG_CONFIG_PATH` - /// contains BOTH libFoo's and libBar's pkgconfig dirs. Order is not - /// fixed — we match either ordering via case patterns. - #[test] - fn pkg_config_path_includes_transitive_lib_pkgconfig() { - let root = tempdir("pcpath-reg"); - let cache = tempdir("pcpath-cache"); - - // libFoo: produces a .pc file. No deps. - write_lib( - &root, - "libFoo", + fn write_runtime_program_with_index(root: &Path, name: &str, index_url: &str) { + write_program( + root, + name, "1.0.0", &[], - r#" -mkdir -p "$WASM_POSIX_DEP_OUT_DIR/lib/pkgconfig" -mkdir -p "$WASM_POSIX_DEP_OUT_DIR/include" -touch "$WASM_POSIX_DEP_OUT_DIR/include/foo.h" -cat > "$WASM_POSIX_DEP_OUT_DIR/lib/pkgconfig/libFoo.pc" <<'PCEOF' -Name: libFoo -Version: 1.0.0 -PCEOF -"#, - r#"[outputs] -headers = ["include/foo.h"] -pkgconfig = ["lib/pkgconfig/libFoo.pc"] -"#, + &format!( + r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR" +printf '\x00asm\x01\x00\x00\x00\x01\x05\x01\x60\x00\x01\x7f\x03\x02\x01\x00\x07\x1a\x02\x0d__abi_version\x00\x00\x06_start\x00\x00\x0a\x06\x01\x04\x00\x41\x00\x0b' > "$WASM_POSIX_DEP_OUT_DIR/{name}.wasm" +printf RUNTIME-BYTES > "$WASM_POSIX_DEP_OUT_DIR/icu.dat" +touch "$WASM_POSIX_DEP_OUT_DIR/via-build""#, + ), + &[(name, &format!("{name}.wasm"))], ); + append_program_runtime_file(root, name, "icu.dat", "/usr/lib/php/icu.dat"); + let build_toml = format!( + r#"script_path = "{name}/build-{name}.sh" +repo_url = "https://example.test/repo.git" +commit = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef" - // libBar: depends on libFoo, also produces a .pc file. - write_lib( - &root, - "libBar", - "1.0.0", - &["libFoo@1.0.0"], - r#" -mkdir -p "$WASM_POSIX_DEP_OUT_DIR/lib/pkgconfig" -mkdir -p "$WASM_POSIX_DEP_OUT_DIR/include" -touch "$WASM_POSIX_DEP_OUT_DIR/include/bar.h" -cat > "$WASM_POSIX_DEP_OUT_DIR/lib/pkgconfig/libBar.pc" <<'PCEOF' -Name: libBar -Version: 1.0.0 -Requires: libFoo -PCEOF +[binary] +index_url = "{index_url}" "#, - r#"[outputs] -headers = ["include/bar.h"] -pkgconfig = ["lib/pkgconfig/libBar.pc"] + ); + fs::write(root.join(name).join("build.toml"), build_toml).unwrap(); + } + + /// Stage an `index.toml` at `path` declaring `name@1.0.0` with a + /// single Success entry for `arch` pointing at `archive_url` with + /// the given `archive_sha256` and `cache_key_sha`. Mirrors what + /// `xtask index-update` will produce in CI; tests use this to + /// short-circuit a real publish pipeline. + fn stage_index_toml( + path: &Path, + name: &str, + arch: TargetArch, + archive_url: &str, + archive_sha256: &str, + cache_key_sha: &str, + ) { + let arch_str = arch.as_str(); + let content = format!( + r#"abi_version = {abi} +generated_at = "2026-05-13T00:00:00Z" +generator = "test" + +[[packages]] +name = "{name}" +version = "1.0.0" +revision = 1 + +[packages.binary.{arch_str}] +status = "success" +archive_url = "{archive_url}" +archive_sha256 = "{archive_sha256}" +cache_key_sha = "{cache_key_sha}" +built_at = "2026-05-13T00:00:00Z" +built_by = "test" "#, + abi = TEST_ABI, ); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(path, content).unwrap(); + } + + // Resolver tests for the index-lookup binary fetch path. Each + // test stages a real archive + index.toml on disk (file:// URLs + // throughout — no network required), writes a source + // package.toml + sibling build.toml that points at the staged + // index, and exercises the resolver's path under one specific + // verification condition. + // + // Fall-through tests assert that the source build's `via-build` + // sentinel appears in the cache (proving the resolver gave up on + // the index path and ran the build script); the happy-path test + // asserts the archive's bytes landed AND `via-build` is absent. + + #[test] + fn direct_pr_overlay_fetch_installs_archive_before_build_toml_index() { + let root = tempdir("direct-overlay-reg"); + let cache = tempdir("direct-overlay-cache"); + let archive_dir = tempdir("direct-overlay-archive"); - // libBaz: depends on libBar (libFoo is transitive). Build script - // asserts WASM_POSIX_DEP_PKG_CONFIG_PATH contains both libFoo - // and libBar pkgconfig dirs (order-insensitive). write_lib( &root, - "libBaz", + "libOverlay", "1.0.0", - &["libBar@1.0.0"], + &[], r#" -test -n "${WASM_POSIX_DEP_PKG_CONFIG_PATH:-}" || { - echo "WASM_POSIX_DEP_PKG_CONFIG_PATH unset" >&2 - exit 1 -} -case "$WASM_POSIX_DEP_PKG_CONFIG_PATH" in - *libFoo*lib/pkgconfig*libBar*lib/pkgconfig*) : ;; - *libBar*lib/pkgconfig*libFoo*lib/pkgconfig*) : ;; - *) - echo "WASM_POSIX_DEP_PKG_CONFIG_PATH does not contain both libFoo and libBar pkgconfig dirs:" >&2 - echo " $WASM_POSIX_DEP_PKG_CONFIG_PATH" >&2 - exit 1 - ;; -esac mkdir -p "$WASM_POSIX_DEP_OUT_DIR/lib" -touch "$WASM_POSIX_DEP_OUT_DIR/lib/libBaz.a" +echo BUILD > "$WASM_POSIX_DEP_OUT_DIR/lib/out.a" +touch "$WASM_POSIX_DEP_OUT_DIR/via-build" "#, r#"[outputs] -libs = ["lib/libBaz.a"] +libs = ["lib/out.a"] "#, ); + let reg_without_overlay = Registry { + roots: vec![root.clone()], + }; + let m_without_overlay = reg_without_overlay.load("libOverlay").unwrap(); + let cache_key_hex = hex(&compute_sha( + &m_without_overlay, + ®_without_overlay, + TEST_ARCH, + TEST_ABI, + &mut BTreeMap::new(), + &mut Vec::new(), + ) + .unwrap()); + + let manifest_text = + archived_manifest_text("libOverlay", "wasm32", &[TEST_ABI], &cache_key_hex); + let archive_bytes = crate::remote_fetch::build_test_archive( + &manifest_text, + &[("lib/out.a", b"FROM-OVERLAY")], + ); + let archive_sha_hex = sha256_hex(&archive_bytes); + let archive_path = archive_dir.join("libOverlay-1.0.0.tar.zst"); + std::fs::write(&archive_path, &archive_bytes).unwrap(); + let archive_url = format!("file://{}", archive_path.display()); + + std::fs::write( + root.join("libOverlay/package.pr.toml"), + format!( + r#" +[binary.wasm32] +archive_url = "{archive_url}" +archive_sha256 = "{archive_sha_hex}" +"# + ), + ) + .unwrap(); + let reg = Registry { roots: vec![root] }; - let m = reg.load("libBaz").unwrap(); - ensure_built(&m, ®, TEST_ARCH, TEST_ABI, &resolve_opts(&cache, None)).unwrap(); + let m = reg.load("libOverlay").unwrap(); + let path = + ensure_built(&m, ®, TEST_ARCH, TEST_ABI, &resolve_opts(&cache, None)).unwrap(); + + assert_eq!( + std::fs::read(path.join("lib/out.a")).unwrap(), + b"FROM-OVERLAY" + ); + assert!( + !path.join("via-build").exists(), + "direct package.pr.toml overlay should bypass the source build" + ); } - /// Libs without a `lib/pkgconfig/` directory (e.g., ncurses ships a - /// `.pc` file optionally; some libs ship none at all) must be SKIPPED - /// when composing `WASM_POSIX_DEP_PKG_CONFIG_PATH`. Otherwise we'd - /// hand pkg-config a list of nonexistent search paths, which clutters - /// diagnostics and (for some pkg-config versions) errors out. #[test] - fn pkg_config_path_skips_libs_without_pkgconfig_dir() { - let root = tempdir("pcpath-skip-reg"); - let cache = tempdir("pcpath-skip-cache"); + fn index_fetch_installs_archive_when_sha_arch_abi_cachekey_all_match() { + let root = tempdir("idx-happy-reg"); + let cache = tempdir("idx-happy-cache"); + let archive_dir = tempdir("idx-happy-archive"); + let index_dir = tempdir("idx-happy-index"); - // libNoPc: ships only a header — no pkgconfig. + // Compute the cache_key_sha the resolver will produce for the + // (fixed-shape) source manifest. cache_key_sha hashes + // name/version/revision/source/arch/abi/dep-shas. This + // fixture's build.toml declares no extra cache inputs, so it + // does not affect compute_sha beyond the revision already + // loaded onto the manifest. + let throwaway_root = tempdir("idx-happy-pre"); write_lib( - &root, - "libNoPc", + &throwaway_root, + "libIdx", "1.0.0", &[], - r#" -mkdir -p "$WASM_POSIX_DEP_OUT_DIR/include" -touch "$WASM_POSIX_DEP_OUT_DIR/include/nopc.h" -"#, - r#"[outputs] -headers = ["include/nopc.h"] -"#, + "true", + "[outputs]\nlibs = [\"lib/out.a\"]\n", + ); + let pre_reg = Registry { + roots: vec![throwaway_root.clone()], + }; + let pre_m = pre_reg.load("libIdx").unwrap(); + let pre_sha = compute_sha( + &pre_m, + &pre_reg, + TEST_ARCH, + TEST_ABI, + &mut BTreeMap::new(), + &mut Vec::new(), + ) + .unwrap(); + let cache_key_hex = hex(&pre_sha); + let _ = std::fs::remove_dir_all(&throwaway_root); + + // Build a real archive whose internal manifest matches arch + // + abi + cache_key. + let manifest_text = archived_manifest_text("libIdx", "wasm32", &[TEST_ABI], &cache_key_hex); + let archive_bytes = crate::remote_fetch::build_test_archive( + &manifest_text, + &[("lib/out.a", b"\x00\x01\x02FAKE")], ); + let archive_sha_hex = sha256_hex(&archive_bytes); + let archive_path = archive_dir.join("libIdx-1.0.0.tar.zst"); + std::fs::write(&archive_path, &archive_bytes).unwrap(); + let archive_url = format!("file://{}", archive_path.display()); - // libConsumer: depends on libNoPc. Asserts that - // WASM_POSIX_DEP_PKG_CONFIG_PATH does NOT contain libNoPc's path, - // even as an empty entry. Empty string is acceptable. - write_lib( - &root, - "libConsumer", - "1.0.0", - &["libNoPc@1.0.0"], - r#" -# Set defaults so set -u doesn't trip. -: "${WASM_POSIX_DEP_PKG_CONFIG_PATH:=}" -case "$WASM_POSIX_DEP_PKG_CONFIG_PATH" in - *libNoPc*) - echo "WASM_POSIX_DEP_PKG_CONFIG_PATH must skip libs without pkgconfig dirs:" >&2 - echo " $WASM_POSIX_DEP_PKG_CONFIG_PATH" >&2 - exit 1 - ;; -esac -mkdir -p "$WASM_POSIX_DEP_OUT_DIR/lib" -touch "$WASM_POSIX_DEP_OUT_DIR/lib/libConsumer.a" -"#, - r#"[outputs] -libs = ["lib/libConsumer.a"] -"#, + let index_path = index_dir.join("index.toml"); + stage_index_toml( + &index_path, + "libIdx", + TargetArch::Wasm32, + &archive_url, + &archive_sha_hex, + &cache_key_hex, ); + let index_url = format!("file://{}", index_path.display()); + write_lib_with_build_toml(&root, "libIdx", &index_url); let reg = Registry { roots: vec![root] }; - let m = reg.load("libConsumer").unwrap(); - ensure_built(&m, ®, TEST_ARCH, TEST_ABI, &resolve_opts(&cache, None)).unwrap(); - } - - // --- Remote-fetch integration tests (Task A.9) ------------------- - // - // These exercise the full `[binary]` resolution path with a - // hand-crafted .tar.zst archive served over a `file://` URL — - // the same code path as production HTTP fetches, but without a - // real network or HTTP server. Each test verifies one outcome: - // - // * happy path — archive is sha-, arch-, abi-, cache_key-valid → - // resolver installs without invoking the build script; - // * sha mismatch / arch mismatch / abi mismatch / cache_key - // mismatch — resolver logs and falls through to source build. - // - // The build script writes a sentinel `via-build` file. Its presence - // in the canonical cache means the source build ran; its absence - // (with the artifacts otherwise installed) means the remote fetch - // succeeded. + let m = reg.load("libIdx").unwrap(); + let path = + ensure_built(&m, ®, TEST_ARCH, TEST_ABI, &resolve_opts(&cache, None)).unwrap(); - fn sha256_hex(bytes: &[u8]) -> String { - let mut h = Sha256::new(); - h.update(bytes); - let out: [u8; 32] = h.finalize().into(); - hex(&out) + // Artifact installed at the canonical cache path with the + // archive's bytes. + assert!(path.starts_with(cache.join("libs"))); + let lib_bytes = std::fs::read(path.join("lib/out.a")).unwrap(); + assert_eq!(lib_bytes, b"\x00\x01\x02FAKE"); + // Build script did NOT run. + assert!( + !path.join("via-build").exists(), + "index fetch should bypass the source build" + ); + // Manifest + artifacts dir stripped during reshape. + assert!(!path.join("manifest.toml").exists()); + assert!(!path.join("artifacts").exists()); } - /// Build the archived `manifest.toml` text for a library named - /// `name`. `arch` and `abi_versions` and `cache_key_sha` populate - /// the `[compatibility]` block. Output declaration is `lib/out.a` - /// to match `write_lib_with_build_toml`. - fn archived_manifest_text( - name: &str, - arch: &str, - abi_versions: &[u32], - cache_key_sha: &str, - ) -> String { - let abi_csv = abi_versions - .iter() - .map(|v| v.to_string()) - .collect::>() - .join(", "); - format!( - r#" -kind = "library" -name = "{name}" -version = "1.0.0" -revision = 1 -depends_on = [] - -[source] -url = "https://example.test/{name}-1.0.0.tar.gz" -sha256 = "{:0>64}" - -[license] -spdx = "TestLicense" + #[test] + fn index_fetch_falls_through_on_index_toml_abi_mismatch() { + let root = tempdir("idx-index-abi-fail-reg"); + let cache = tempdir("idx-index-abi-fail-cache"); + let archive_dir = tempdir("idx-index-abi-fail-archive"); + let index_dir = tempdir("idx-index-abi-fail-index"); -[outputs] -libs = ["lib/out.a"] + let index_path = index_dir.join("index.toml"); + let index_url = format!("file://{}", index_path.display()); + write_lib_with_build_toml(&root, "libIdxTopAbi", &index_url); -[compatibility] -target_arch = "{arch}" -abi_versions = [{abi_csv}] -cache_key_sha = "{cache_key_sha}" -"#, - "" + let reg = Registry { + roots: vec![root.clone()], + }; + let m = reg.load("libIdxTopAbi").unwrap(); + let cache_key_hex = hex(&compute_sha( + &m, + ®, + TEST_ARCH, + TEST_ABI, + &mut BTreeMap::new(), + &mut Vec::new(), ) - } - - fn archived_program_manifest_text( - name: &str, - output_name: &str, - output_wasm: &str, - arch: &str, - abi_versions: &[u32], - cache_key_sha: &str, - ) -> String { - let abi_csv = abi_versions - .iter() - .map(|v| v.to_string()) - .collect::>() - .join(", "); - format!( - r#" -kind = "program" -name = "{name}" -version = "1.0.0" -revision = 1 -depends_on = [] + .unwrap()); -[source] -url = "https://example.test/{name}-1.0.0.tar.gz" -sha256 = "{:0>64}" + let manifest_text = + archived_manifest_text("libIdxTopAbi", "wasm32", &[TEST_ABI], &cache_key_hex); + let archive_bytes = + crate::remote_fetch::build_test_archive(&manifest_text, &[("lib/out.a", b"REMOTE")]); + let archive_sha_hex = sha256_hex(&archive_bytes); + let archive_path = archive_dir.join("libIdxTopAbi-1.0.0.tar.zst"); + std::fs::write(&archive_path, &archive_bytes).unwrap(); + let archive_url = format!("file://{}", archive_path.display()); -[license] -spdx = "TestLicense" + let arch_str = TEST_ARCH.as_str(); + std::fs::write( + &index_path, + format!( + r#"abi_version = {} +generated_at = "2026-05-13T00:00:00Z" +generator = "test" -[[outputs]] -name = "{output_name}" -wasm = "{output_wasm}" +[[packages]] +name = "libIdxTopAbi" +version = "1.0.0" +revision = 1 -[compatibility] -target_arch = "{arch}" -abi_versions = [{abi_csv}] -cache_key_sha = "{cache_key_sha}" +[packages.binary.{arch_str}] +status = "success" +archive_url = "{archive_url}" +archive_sha256 = "{archive_sha_hex}" +cache_key_sha = "{cache_key_hex}" "#, - "" + TEST_ABI + 1 + ), ) - } + .unwrap(); - fn archived_program_runtime_manifest_text( - name: &str, - arch: &str, - abi_versions: &[u32], - cache_key_sha: &str, - ) -> String { - let abi_csv = abi_versions - .iter() - .map(|value| value.to_string()) - .collect::>() - .join(", "); - format!( - r#" -kind = "program" -name = "{name}" -version = "1.0.0" -revision = 1 -depends_on = [] + let path = + ensure_built(&m, ®, TEST_ARCH, TEST_ABI, &resolve_opts(&cache, None)).unwrap(); -[source] -url = "https://example.test/{name}.tar.gz" -sha256 = "{:0>64}" + assert!( + path.join("via-build").exists(), + "top-level index ABI mismatch must fall through to source build" + ); + let lib = std::fs::read(path.join("lib/out.a")).unwrap(); + assert_ne!(lib, b"REMOTE", "remote bytes must not have been installed"); + } -[license] -spdx = "MIT" + #[test] + fn binaries_dir_program_fetch_does_not_require_built_deps() { + let root = tempdir("prog-bdir-remote-first-reg"); + let cache = tempdir("prog-bdir-remote-first-cache"); + let bin_dir = tempdir("prog-bdir-remote-first-bin"); + let archive_dir = tempdir("prog-bdir-remote-first-archive"); + let index_dir = tempdir("prog-bdir-remote-first-index"); -[[outputs]] -name = "{name}" -wasm = "{name}.wasm" + write_program( + &root, + "baddep", + "1.0.0", + &[], + "echo baddep source build should not run >&2; exit 42", + &[("baddep", "baddep.wasm")], + ); + write_program( + &root, + "progIdx", + "1.0.0", + &["baddep@1.0.0"], + "echo progIdx source build should not run >&2; exit 43", + &[("progIdx", "progIdx.wasm")], + ); -[[runtime_files]] -artifact = "icu.dat" -guest_path = "/usr/lib/php/icu.dat" + let index_path = index_dir.join("index.toml"); + let index_url = format!("file://{}", index_path.display()); + write_program_build_toml(&root, "progIdx", &index_url); -[compatibility] -target_arch = "{arch}" -abi_versions = [{abi_csv}] -cache_key_sha = "{cache_key_sha}" -"#, - "" + let reg = Registry { + roots: vec![root.clone()], + }; + let m = reg.load("progIdx").unwrap(); + let cache_key_hex = hex(&compute_sha( + &m, + ®, + TEST_ARCH, + TEST_ABI, + &mut BTreeMap::new(), + &mut Vec::new(), ) + .unwrap()); + + let manifest_text = archived_program_manifest_text( + "progIdx", + "progIdx", + "progIdx.wasm", + "wasm32", + &[TEST_ABI], + &cache_key_hex, + ); + let prog_wasm = minimal_executable_wasm(); + let archive_bytes = crate::remote_fetch::build_test_archive( + &manifest_text, + &[("progIdx.wasm", prog_wasm.as_slice())], + ); + let archive_sha_hex = sha256_hex(&archive_bytes); + let archive_path = archive_dir.join("progIdx-1.0.0.tar.zst"); + std::fs::write(&archive_path, &archive_bytes).unwrap(); + let archive_url = format!("file://{}", archive_path.display()); + stage_index_toml( + &index_path, + "progIdx", + TargetArch::Wasm32, + &archive_url, + &archive_sha_hex, + &cache_key_hex, + ); + + let opts = ResolveOpts { + cache_root: &cache, + local_libs: None, + force_source_build: None, + fetch_only: false, + repo_root: Some(&root), + binaries_dir: Some(&bin_dir), + }; + let path = ensure_built(&m, ®, TEST_ARCH, TEST_ABI, &opts).unwrap(); + + assert_eq!(std::fs::read(path.join("progIdx.wasm")).unwrap(), prog_wasm); + let baddep_cached = std::fs::read_dir(cache.join("programs")) + .unwrap() + .filter_map(Result::ok) + .any(|entry| entry.file_name().to_string_lossy().starts_with("baddep-")); + assert!( + !baddep_cached, + "binary materialization should not have source-built baddep first" + ); } - /// Write a source `package.toml` + sibling `build.toml` for - /// index-lookup-based resolution tests. The `build.toml`'s - /// `[binary]` block points at `index_url` (typically a `file://` - /// URL to a staged `index.toml`). The build script drops a - /// `via-build` sentinel so fall-through tests can detect that the - /// source build ran instead of the index fetch. - fn write_lib_with_build_toml(root: &Path, name: &str, index_url: &str) { - let lib_dir = root.join(name); - std::fs::create_dir_all(&lib_dir).unwrap(); + #[test] + fn fetched_program_runtime_file_matches_source_mirror_layout() { + let root = tempdir("runtime-fetch-parity-reg"); + let remote_cache = tempdir("runtime-fetch-parity-remote-cache"); + let source_cache = tempdir("runtime-fetch-parity-source-cache"); + let remote_bin = tempdir("runtime-fetch-parity-remote-bin"); + let source_bin = tempdir("runtime-fetch-parity-source-bin"); + let archive_dir = tempdir("runtime-fetch-parity-archive"); + let index_dir = tempdir("runtime-fetch-parity-index"); + let index_path = index_dir.join("index.toml"); + let index_url = format!("file://{}", index_path.display()); + let name = "runtimeFetched"; + write_runtime_program_with_index(&root, name, &index_url); - let deps_toml = format!( - r#" -kind = "library" -name = "{name}" -version = "1.0.0" -depends_on = [] + let reg = Registry { + roots: vec![root.clone()], + }; + let manifest = reg.load(name).unwrap(); + let cache_key_hex = hex(&compute_sha( + &manifest, + ®, + TEST_ARCH, + TEST_ABI, + &mut BTreeMap::new(), + &mut Vec::new(), + ) + .unwrap()); + let archived_manifest = archived_program_runtime_manifest_text( + name, + TEST_ARCH.as_str(), + &[TEST_ABI], + &cache_key_hex, + ); + let wasm_name = format!("{name}.wasm"); + let wasm_bytes = b"\x00asm\x01\x00\x00\x00\x01\x05\x01\x60\x00\x01\x7f\x03\x02\x01\x00\x07\x1a\x02\x0d__abi_version\x00\x00\x06_start\x00\x00\x0a\x06\x01\x04\x00\x41\x00\x0b"; + let archive_bytes = crate::remote_fetch::build_test_archive( + &archived_manifest, + &[ + (wasm_name.as_str(), wasm_bytes.as_slice()), + ("icu.dat", b"RUNTIME-BYTES"), + ], + ); + let archive_path = archive_dir.join(format!("{name}-1.0.0.tar.zst")); + fs::write(&archive_path, &archive_bytes).unwrap(); + stage_index_toml( + &index_path, + name, + TEST_ARCH, + &format!("file://{}", archive_path.display()), + &sha256_hex(&archive_bytes), + &cache_key_hex, + ); -[source] -url = "https://example.test/{name}-1.0.0.tar.gz" -sha256 = "{:0>64}" + let remote_opts = ResolveOpts { + cache_root: &remote_cache, + local_libs: None, + force_source_build: None, + fetch_only: false, + repo_root: Some(&root), + binaries_dir: Some(&remote_bin), + }; + let remote_path = ensure_built(&manifest, ®, TEST_ARCH, TEST_ABI, &remote_opts).unwrap(); + assert!(!remote_path.join("via-build").exists()); + place_binaries_symlinks(&manifest, &remote_path, &remote_bin, TEST_ARCH).unwrap(); -[license] -spdx = "TestLicense" + let force = BTreeSet::from([name.to_string()]); + let source_opts = ResolveOpts { + cache_root: &source_cache, + local_libs: None, + force_source_build: Some(&force), + fetch_only: false, + repo_root: Some(&root), + binaries_dir: Some(&source_bin), + }; + let source_path = ensure_built(&manifest, ®, TEST_ARCH, TEST_ABI, &source_opts).unwrap(); + assert!(source_path.join("via-build").exists()); + place_binaries_symlinks(&manifest, &source_path, &source_bin, TEST_ARCH).unwrap(); -[outputs] -libs = ["lib/out.a"] -"#, - "" + let mirror_rel = Path::new("programs/wasm32").join(name).join("icu.dat"); + assert_eq!( + fs::read(remote_bin.join(&mirror_rel)).unwrap(), + b"RUNTIME-BYTES" ); - std::fs::write(lib_dir.join("package.toml"), deps_toml).unwrap(); - - let build_toml = format!( - r#" -script_path = "packages/registry/{name}/build-{name}.sh" -repo_url = "https://example.test/repo.git" -commit = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef" - -[binary] -index_url = "{index_url}" -"# + assert_eq!( + fs::read(source_bin.join(&mirror_rel)).unwrap(), + b"RUNTIME-BYTES" ); - std::fs::write(lib_dir.join("build.toml"), build_toml).unwrap(); - - let script = "#!/bin/bash\nset -euo pipefail\n\ -mkdir -p \"$WASM_POSIX_DEP_OUT_DIR/lib\"\n\ -echo BUILD > \"$WASM_POSIX_DEP_OUT_DIR/lib/out.a\"\n\ -touch \"$WASM_POSIX_DEP_OUT_DIR/via-build\"\n"; - let script_path = lib_dir.join(format!("build-{name}.sh")); - std::fs::write(&script_path, script).unwrap(); - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let mut p = std::fs::metadata(&script_path).unwrap().permissions(); - p.set_mode(0o755); - std::fs::set_permissions(&script_path, p).unwrap(); - } } - fn write_program_build_toml(root: &Path, name: &str, index_url: &str) { - let dir = root.join(name); - let build_toml = format!( - r#" -script_path = "packages/registry/{name}/build-{name}.sh" -repo_url = "https://example.test/repo.git" -commit = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef" + #[test] + fn incomplete_fetched_runtime_file_falls_back_or_fails_fetch_only() { + let root = tempdir("runtime-fetch-incomplete-reg"); + let fallback_cache = tempdir("runtime-fetch-incomplete-fallback-cache"); + let fetch_only_cache = tempdir("runtime-fetch-incomplete-only-cache"); + let archive_dir = tempdir("runtime-fetch-incomplete-archive"); + let index_dir = tempdir("runtime-fetch-incomplete-index"); + let index_path = index_dir.join("index.toml"); + let index_url = format!("file://{}", index_path.display()); + let name = "runtimeIncomplete"; + write_runtime_program_with_index(&root, name, &index_url); -[binary] -index_url = "{index_url}" -"# + let reg = Registry { + roots: vec![root.clone()], + }; + let manifest = reg.load(name).unwrap(); + let cache_key_hex = hex(&compute_sha( + &manifest, + ®, + TEST_ARCH, + TEST_ABI, + &mut BTreeMap::new(), + &mut Vec::new(), + ) + .unwrap()); + let archived_manifest = archived_program_runtime_manifest_text( + name, + TEST_ARCH.as_str(), + &[TEST_ABI], + &cache_key_hex, ); - std::fs::write(dir.join("build.toml"), build_toml).unwrap(); - } - - fn write_runtime_program_with_index(root: &Path, name: &str, index_url: &str) { - write_program( - root, + let wasm_name = format!("{name}.wasm"); + let wasm_bytes = b"\x00asm\x01\x00\x00\x00\x01\x05\x01\x60\x00\x01\x7f\x03\x02\x01\x00\x07\x1a\x02\x0d__abi_version\x00\x00\x06_start\x00\x00\x0a\x06\x01\x04\x00\x41\x00\x0b"; + let archive_bytes = crate::remote_fetch::build_test_archive( + &archived_manifest, + &[(wasm_name.as_str(), wasm_bytes.as_slice())], + ); + let archive_path = archive_dir.join(format!("{name}-1.0.0.tar.zst")); + fs::write(&archive_path, &archive_bytes).unwrap(); + stage_index_toml( + &index_path, name, - "1.0.0", - &[], - &format!( - r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR" -printf '\x00asm\x01\x00\x00\x00\x01\x05\x01\x60\x00\x01\x7f\x03\x02\x01\x00\x07\x1a\x02\x0d__abi_version\x00\x00\x06_start\x00\x00\x0a\x06\x01\x04\x00\x41\x00\x0b' > "$WASM_POSIX_DEP_OUT_DIR/{name}.wasm" -printf RUNTIME-BYTES > "$WASM_POSIX_DEP_OUT_DIR/icu.dat" -touch "$WASM_POSIX_DEP_OUT_DIR/via-build""#, - ), - &[(name, &format!("{name}.wasm"))], + TEST_ARCH, + &format!("file://{}", archive_path.display()), + &sha256_hex(&archive_bytes), + &cache_key_hex, ); - append_program_runtime_file(root, name, "icu.dat", "/usr/lib/php/icu.dat"); - let build_toml = format!( - r#"script_path = "{name}/build-{name}.sh" -repo_url = "https://example.test/repo.git" -commit = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef" -[binary] -index_url = "{index_url}" -"#, + let fallback = ensure_built( + &manifest, + ®, + TEST_ARCH, + TEST_ABI, + &resolve_opts(&fallback_cache, None), + ) + .unwrap(); + assert!(fallback.join("via-build").exists()); + assert_eq!( + fs::read(fallback.join("icu.dat")).unwrap(), + b"RUNTIME-BYTES" ); - fs::write(root.join(name).join("build.toml"), build_toml).unwrap(); - } - /// Stage an `index.toml` at `path` declaring `name@1.0.0` with a - /// single Success entry for `arch` pointing at `archive_url` with - /// the given `archive_sha256` and `cache_key_sha`. Mirrors what - /// `xtask index-update` will produce in CI; tests use this to - /// short-circuit a real publish pipeline. - fn stage_index_toml( - path: &Path, - name: &str, - arch: TargetArch, - archive_url: &str, - archive_sha256: &str, - cache_key_sha: &str, - ) { - let arch_str = arch.as_str(); - let content = format!( - r#"abi_version = {abi} -generated_at = "2026-05-13T00:00:00Z" -generator = "test" + let fetch_only_opts = ResolveOpts { + cache_root: &fetch_only_cache, + local_libs: None, + force_source_build: None, + fetch_only: true, + repo_root: Some(&root), + binaries_dir: None, + }; + let err = ensure_built(&manifest, ®, TEST_ARCH, TEST_ABI, &fetch_only_opts).unwrap_err(); + assert!(err.contains("fetch-only"), "got: {err}"); + assert!( + !canonical_path( + &fetch_only_cache, + &manifest, + TEST_ARCH, + &compute_sha( + &manifest, + ®, + TEST_ARCH, + TEST_ABI, + &mut BTreeMap::new(), + &mut Vec::new(), + ) + .unwrap(), + ) + .join("via-build") + .exists() + ); + } -[[packages]] -name = "{name}" -version = "1.0.0" -revision = 1 + #[test] + fn index_fetch_falls_through_on_archive_sha_mismatch() { + let root = tempdir("idx-shafail-reg"); + let cache = tempdir("idx-shafail-cache"); + let archive_dir = tempdir("idx-shafail-archive"); + let index_dir = tempdir("idx-shafail-index"); -[packages.binary.{arch_str}] -status = "success" -archive_url = "{archive_url}" -archive_sha256 = "{archive_sha256}" -cache_key_sha = "{cache_key_sha}" -built_at = "2026-05-13T00:00:00Z" -built_by = "test" -"#, - abi = TEST_ABI, + // Build a real archive but advertise the WRONG sha in the index. + let manifest_text = archived_manifest_text( + "libIdxSha", + "wasm32", + &[TEST_ABI], + // cache_key_sha is irrelevant: we never get past the sha + // check. Fill with a valid-shaped dummy so parse_archived + // wouldn't complain (defence in depth). + &"a".repeat(64), ); - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent).unwrap(); - } - std::fs::write(path, content).unwrap(); - } + let archive_bytes = + crate::remote_fetch::build_test_archive(&manifest_text, &[("lib/out.a", b"REMOTE")]); + let archive_path = archive_dir.join("libIdxSha-1.0.0.tar.zst"); + std::fs::write(&archive_path, &archive_bytes).unwrap(); + let archive_url = format!("file://{}", archive_path.display()); - // Resolver tests for the index-lookup binary fetch path. Each - // test stages a real archive + index.toml on disk (file:// URLs - // throughout — no network required), writes a source - // package.toml + sibling build.toml that points at the staged - // index, and exercises the resolver's path under one specific - // verification condition. - // - // Fall-through tests assert that the source build's `via-build` - // sentinel appears in the cache (proving the resolver gave up on - // the index path and ran the build script); the happy-path test - // asserts the archive's bytes landed AND `via-build` is absent. + let bogus_sha = "0".repeat(64); + let index_path = index_dir.join("index.toml"); + stage_index_toml( + &index_path, + "libIdxSha", + TargetArch::Wasm32, + &archive_url, + &bogus_sha, + &"a".repeat(64), + ); + let index_url = format!("file://{}", index_path.display()); + write_lib_with_build_toml(&root, "libIdxSha", &index_url); - #[test] - fn direct_pr_overlay_fetch_installs_archive_before_build_toml_index() { - let root = tempdir("direct-overlay-reg"); - let cache = tempdir("direct-overlay-cache"); - let archive_dir = tempdir("direct-overlay-archive"); + let reg = Registry { roots: vec![root] }; + let m = reg.load("libIdxSha").unwrap(); + let path = + ensure_built(&m, ®, TEST_ARCH, TEST_ABI, &resolve_opts(&cache, None)).unwrap(); - write_lib( - &root, - "libOverlay", - "1.0.0", - &[], - r#" -mkdir -p "$WASM_POSIX_DEP_OUT_DIR/lib" -echo BUILD > "$WASM_POSIX_DEP_OUT_DIR/lib/out.a" -touch "$WASM_POSIX_DEP_OUT_DIR/via-build" -"#, - r#"[outputs] -libs = ["lib/out.a"] -"#, + // Source build ran. + assert!( + path.join("via-build").exists(), + "sha mismatch must fall through to source build" ); + let lib = std::fs::read(path.join("lib/out.a")).unwrap(); + assert_ne!(lib, b"REMOTE", "remote bytes must not have been installed"); + } - let reg_without_overlay = Registry { - roots: vec![root.clone()], - }; - let m_without_overlay = reg_without_overlay.load("libOverlay").unwrap(); - let cache_key_hex = hex(&compute_sha( - &m_without_overlay, - ®_without_overlay, - TEST_ARCH, - TEST_ABI, - &mut BTreeMap::new(), - &mut Vec::new(), - ) - .unwrap()); + #[test] + fn index_fetch_falls_through_on_target_arch_mismatch() { + let root = tempdir("idx-archfail-reg"); + let cache = tempdir("idx-archfail-cache"); + let archive_dir = tempdir("idx-archfail-archive"); + let index_dir = tempdir("idx-archfail-index"); + // Archive's internal compatibility block declares wasm64 — + // resolver requests wasm32 (TEST_ARCH). The index entry + // points the wasm32 slot at this archive (an + // archive-staging bug a real CI would never produce, but + // the resolver must defend against it). let manifest_text = - archived_manifest_text("libOverlay", "wasm32", &[TEST_ABI], &cache_key_hex); - let archive_bytes = crate::remote_fetch::build_test_archive( - &manifest_text, - &[("lib/out.a", b"FROM-OVERLAY")], - ); - let archive_sha_hex = sha256_hex(&archive_bytes); - let archive_path = archive_dir.join("libOverlay-1.0.0.tar.zst"); + archived_manifest_text("libIdxArch", "wasm64", &[TEST_ABI], &"a".repeat(64)); + let archive_bytes = + crate::remote_fetch::build_test_archive(&manifest_text, &[("lib/out.a", b"REMOTE")]); + let archive_sha = sha256_hex(&archive_bytes); + let archive_path = archive_dir.join("libIdxArch-1.0.0.tar.zst"); std::fs::write(&archive_path, &archive_bytes).unwrap(); let archive_url = format!("file://{}", archive_path.display()); - std::fs::write( - root.join("libOverlay/package.pr.toml"), - format!( - r#" -[binary.wasm32] -archive_url = "{archive_url}" -archive_sha256 = "{archive_sha_hex}" -"# - ), - ) - .unwrap(); + let index_path = index_dir.join("index.toml"); + stage_index_toml( + &index_path, + "libIdxArch", + TargetArch::Wasm32, + &archive_url, + &archive_sha, + &"a".repeat(64), + ); + let index_url = format!("file://{}", index_path.display()); + write_lib_with_build_toml(&root, "libIdxArch", &index_url); let reg = Registry { roots: vec![root] }; - let m = reg.load("libOverlay").unwrap(); - let path = - ensure_built(&m, ®, TEST_ARCH, TEST_ABI, &resolve_opts(&cache, None)).unwrap(); + let m = reg.load("libIdxArch").unwrap(); + let path = ensure_built( + &m, + ®, + TEST_ARCH, // wasm32 + TEST_ABI, + &resolve_opts(&cache, None), + ) + .unwrap(); - assert_eq!( - std::fs::read(path.join("lib/out.a")).unwrap(), - b"FROM-OVERLAY" - ); assert!( - !path.join("via-build").exists(), - "direct package.pr.toml overlay should bypass the source build" + path.join("via-build").exists(), + "arch mismatch must fall through to source build" ); } #[test] - fn index_fetch_installs_archive_when_sha_arch_abi_cachekey_all_match() { - let root = tempdir("idx-happy-reg"); - let cache = tempdir("idx-happy-cache"); - let archive_dir = tempdir("idx-happy-archive"); - let index_dir = tempdir("idx-happy-index"); + fn index_fetch_falls_through_on_abi_mismatch() { + let root = tempdir("idx-abifail-reg"); + let cache = tempdir("idx-abifail-cache"); + let archive_dir = tempdir("idx-abifail-archive"); + let index_dir = tempdir("idx-abifail-index"); - // Compute the cache_key_sha the resolver will produce for the - // (fixed-shape) source manifest. cache_key_sha hashes - // name/version/revision/source/arch/abi/dep-shas. This - // fixture's build.toml declares no extra cache inputs, so it - // does not affect compute_sha beyond the revision already - // loaded onto the manifest. - let throwaway_root = tempdir("idx-happy-pre"); - write_lib( - &throwaway_root, - "libIdx", - "1.0.0", - &[], - "true", - "[outputs]\nlibs = [\"lib/out.a\"]\n", + // Archive supports only ABI 999 — resolver passes TEST_ABI. + let manifest_text = archived_manifest_text("libIdxAbi", "wasm32", &[999], &"a".repeat(64)); + let archive_bytes = + crate::remote_fetch::build_test_archive(&manifest_text, &[("lib/out.a", b"REMOTE")]); + let archive_sha = sha256_hex(&archive_bytes); + let archive_path = archive_dir.join("libIdxAbi-1.0.0.tar.zst"); + std::fs::write(&archive_path, &archive_bytes).unwrap(); + let archive_url = format!("file://{}", archive_path.display()); + + let index_path = index_dir.join("index.toml"); + stage_index_toml( + &index_path, + "libIdxAbi", + TargetArch::Wasm32, + &archive_url, + &archive_sha, + &"a".repeat(64), ); - let pre_reg = Registry { - roots: vec![throwaway_root.clone()], - }; - let pre_m = pre_reg.load("libIdx").unwrap(); - let pre_sha = compute_sha( - &pre_m, - &pre_reg, + let index_url = format!("file://{}", index_path.display()); + write_lib_with_build_toml(&root, "libIdxAbi", &index_url); + + let reg = Registry { roots: vec![root] }; + let m = reg.load("libIdxAbi").unwrap(); + let path = ensure_built( + &m, + ®, TEST_ARCH, - TEST_ABI, - &mut BTreeMap::new(), - &mut Vec::new(), + TEST_ABI, // not in [999] + &resolve_opts(&cache, None), ) .unwrap(); - let cache_key_hex = hex(&pre_sha); - let _ = std::fs::remove_dir_all(&throwaway_root); - // Build a real archive whose internal manifest matches arch - // + abi + cache_key. - let manifest_text = archived_manifest_text("libIdx", "wasm32", &[TEST_ABI], &cache_key_hex); - let archive_bytes = crate::remote_fetch::build_test_archive( - &manifest_text, - &[("lib/out.a", b"\x00\x01\x02FAKE")], + assert!( + path.join("via-build").exists(), + "abi mismatch must fall through to source build" ); - let archive_sha_hex = sha256_hex(&archive_bytes); - let archive_path = archive_dir.join("libIdx-1.0.0.tar.zst"); + } + + #[test] + fn index_fetch_falls_through_on_cache_key_mismatch() { + let root = tempdir("idx-ckfail-reg"); + let cache = tempdir("idx-ckfail-cache"); + let archive_dir = tempdir("idx-ckfail-archive"); + let index_dir = tempdir("idx-ckfail-index"); + + // Archive's internal compat.cache_key_sha is well-formed but + // doesn't match what compute_sha would produce for this lib. + let wrong_ck = "f".repeat(64); + let manifest_text = archived_manifest_text("libIdxCk", "wasm32", &[TEST_ABI], &wrong_ck); + let archive_bytes = + crate::remote_fetch::build_test_archive(&manifest_text, &[("lib/out.a", b"REMOTE")]); + let archive_sha = sha256_hex(&archive_bytes); + let archive_path = archive_dir.join("libIdxCk-1.0.0.tar.zst"); std::fs::write(&archive_path, &archive_bytes).unwrap(); let archive_url = format!("file://{}", archive_path.display()); let index_path = index_dir.join("index.toml"); stage_index_toml( &index_path, - "libIdx", + "libIdxCk", TargetArch::Wasm32, &archive_url, - &archive_sha_hex, - &cache_key_hex, + &archive_sha, + &wrong_ck, ); let index_url = format!("file://{}", index_path.display()); - write_lib_with_build_toml(&root, "libIdx", &index_url); + write_lib_with_build_toml(&root, "libIdxCk", &index_url); let reg = Registry { roots: vec![root] }; - let m = reg.load("libIdx").unwrap(); + let m = reg.load("libIdxCk").unwrap(); let path = ensure_built(&m, ®, TEST_ARCH, TEST_ABI, &resolve_opts(&cache, None)).unwrap(); - // Artifact installed at the canonical cache path with the - // archive's bytes. - assert!(path.starts_with(cache.join("libs"))); - let lib_bytes = std::fs::read(path.join("lib/out.a")).unwrap(); - assert_eq!(lib_bytes, b"\x00\x01\x02FAKE"); - // Build script did NOT run. assert!( - !path.join("via-build").exists(), - "index fetch should bypass the source build" + path.join("via-build").exists(), + "cache_key_sha mismatch must fall through to source build" ); - // Manifest + artifacts dir stripped during reshape. - assert!(!path.join("manifest.toml").exists()); - assert!(!path.join("artifacts").exists()); } #[test] - fn index_fetch_falls_through_on_index_toml_abi_mismatch() { - let root = tempdir("idx-index-abi-fail-reg"); - let cache = tempdir("idx-index-abi-fail-cache"); - let archive_dir = tempdir("idx-index-abi-fail-archive"); - let index_dir = tempdir("idx-index-abi-fail-index"); + fn fetch_only_rejects_missing_index_entry_without_source_build() { + let root = tempdir("fetch-only-missing-reg"); + let cache = tempdir("fetch-only-missing-cache"); + let index_dir = tempdir("fetch-only-missing-index"); let index_path = index_dir.join("index.toml"); + std::fs::write( + &index_path, + format!( + r#"abi_version = {TEST_ABI} +generated_at = "2026-06-09T00:00:00Z" +generator = "test" +"# + ), + ) + .unwrap(); let index_url = format!("file://{}", index_path.display()); - write_lib_with_build_toml(&root, "libIdxTopAbi", &index_url); + write_lib_with_build_toml(&root, "libFetchOnly", &index_url); let reg = Registry { roots: vec![root.clone()], }; - let m = reg.load("libIdxTopAbi").unwrap(); - let cache_key_hex = hex(&compute_sha( + let m = reg.load("libFetchOnly").unwrap(); + let sha = compute_sha( &m, ®, TEST_ARCH, @@ -7783,768 +13341,1912 @@ archive_sha256 = "{archive_sha_hex}" &mut BTreeMap::new(), &mut Vec::new(), ) - .unwrap()); + .unwrap(); + let canonical = canonical_path(&cache, &m, TEST_ARCH, &sha); + let opts = ResolveOpts { + cache_root: &cache, + local_libs: None, + force_source_build: None, + fetch_only: true, + repo_root: Some(&root), + binaries_dir: None, + }; - let manifest_text = - archived_manifest_text("libIdxTopAbi", "wasm32", &[TEST_ABI], &cache_key_hex); - let archive_bytes = - crate::remote_fetch::build_test_archive(&manifest_text, &[("lib/out.a", b"REMOTE")]); - let archive_sha_hex = sha256_hex(&archive_bytes); - let archive_path = archive_dir.join("libIdxTopAbi-1.0.0.tar.zst"); - std::fs::write(&archive_path, &archive_bytes).unwrap(); - let archive_url = format!("file://{}", archive_path.display()); + let err = ensure_built(&m, ®, TEST_ARCH, TEST_ABI, &opts).unwrap_err(); + assert!(err.contains("fetch-only resolve"), "got: {err}"); + assert!( + !canonical.join("via-build").exists(), + "fetch-only must not run the source build script" + ); + } - let arch_str = TEST_ARCH.as_str(); - std::fs::write( - &index_path, + // --- kind = "program" resolver tests (Task B.2) --- + + /// Create a `kind = "program"` package.toml + build-.sh pair. + /// Mirrors `write_lib` but emits `[[outputs]]` array-of-tables. + fn write_program( + root: &Path, + name: &str, + version: &str, + deps: &[&str], + build_script_body: &str, + outputs: &[(&str, &str)], + ) { + let dir = root.join(name); + fs::create_dir_all(&dir).unwrap(); + let depends_on = deps + .iter() + .map(|d| format!("\"{}\"", d)) + .collect::>() + .join(", "); + let mut outputs_toml = String::new(); + for (n, w) in outputs { + outputs_toml.push_str(&format!("[[outputs]]\nname = \"{n}\"\nwasm = \"{w}\"\n\n")); + } + fs::write( + dir.join("package.toml"), format!( - r#"abi_version = {} -generated_at = "2026-05-13T00:00:00Z" -generator = "test" + r#"kind = "program" +name = "{name}" +version = "{version}" +depends_on = [{depends_on}] +[source] +url = "https://example.test/{name}.tar.gz" +sha256 = "0000000000000000000000000000000000000000000000000000000000000000" +[license] +spdx = "MIT" +{outputs_toml}"#, + ), + ) + .unwrap(); + let script_path = dir.join(format!("build-{name}.sh")); + fs::write( + &script_path, + format!("#!/bin/bash\nset -e\n{build_script_body}\n"), + ) + .unwrap(); -[[packages]] -name = "libIdxTopAbi" -version = "1.0.0" -revision = 1 + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut p = std::fs::metadata(&script_path).unwrap().permissions(); + p.set_mode(0o755); + std::fs::set_permissions(&script_path, p).unwrap(); + } + } -[packages.binary.{arch_str}] -status = "success" -archive_url = "{archive_url}" -archive_sha256 = "{archive_sha_hex}" -cache_key_sha = "{cache_key_hex}" + fn append_program_runtime_file(root: &Path, name: &str, artifact: &str, guest_path: &str) { + let manifest_path = root.join(name).join("package.toml"); + let mut text = fs::read_to_string(&manifest_path).unwrap(); + text.push_str(&format!( + "\n[[runtime_files]]\nartifact = {artifact:?}\nguest_path = {guest_path:?}\n" + )); + fs::write(manifest_path, text).unwrap(); + } + + #[test] + fn canonical_path_uses_programs_subdir_for_program_kind() { + let m = DepsManifest::parse( + r#"kind = "program" +name = "vim" +version = "9.1.0900" +[source] +url = "https://x.test/vim.tar.gz" +sha256 = "0000000000000000000000000000000000000000000000000000000000000000" +[license] +spdx = "Vim" +[[outputs]] +name = "vim" +wasm = "vim.wasm" "#, - TEST_ABI + 1 - ), + PathBuf::from("/x"), ) .unwrap(); + let sha = [0u8; 32]; + let p = canonical_path(Path::new("/cache"), &m, TargetArch::Wasm32, &sha); + let s = p.to_string_lossy(); + assert!(s.contains("/programs/"), "got: {s}"); + assert!(s.contains("vim-9.1.0900-rev1-wasm32-"), "got: {s}"); + } + + #[test] + fn build_validates_program_wasm_outputs_present() { + let root = tempdir("prog-out-pass"); + let cache = tempdir("prog-out-pass-cache"); + write_program( + &root, + "tinyprog", + "0.1.0", + &[], + // Build script writes the declared wasm. + r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR" && printf '\x00asm\x01\x00\x00\x00\x01\x05\x01\x60\x00\x01\x7f\x03\x02\x01\x00\x07\x1a\x02\x0d__abi_version\x00\x00\x06_start\x00\x00\x0a\x06\x01\x04\x00\x41\x00\x0b' > "$WASM_POSIX_DEP_OUT_DIR/tinyprog.wasm""#, + &[("tinyprog", "tinyprog.wasm")], + ); + let reg = Registry { roots: vec![root] }; + let m = reg.load("tinyprog").unwrap(); + ensure_built(&m, ®, TargetArch::Wasm32, 4, &resolve_opts(&cache, None)).unwrap(); + } + + #[test] + fn build_fails_when_program_wasm_output_missing() { + let root = tempdir("prog-out-miss"); + let cache = tempdir("prog-out-miss-cache"); + write_program( + &root, + "miss", + "0.1.0", + &[], + // Build script does NOT produce miss.wasm. + r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR""#, + &[("miss", "miss.wasm")], + ); + let reg = Registry { roots: vec![root] }; + let m = reg.load("miss").unwrap(); + let err = + ensure_built(&m, ®, TargetArch::Wasm32, 4, &resolve_opts(&cache, None)).unwrap_err(); + assert!(err.contains("miss.wasm"), "got: {err}"); + } + #[test] + fn program_runtime_file_is_required_and_cached_as_a_regular_file() { + let root = tempdir("prog-runtime-file"); + let cache = tempdir("prog-runtime-file-cache"); + write_program( + &root, + "runtimeprog", + "0.1.0", + &[], + r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR" +printf '\x00asm\x01\x00\x00\x00\x01\x05\x01\x60\x00\x01\x7f\x03\x02\x01\x00\x07\x1a\x02\x0d__abi_version\x00\x00\x06_start\x00\x00\x0a\x06\x01\x04\x00\x41\x00\x0b' > "$WASM_POSIX_DEP_OUT_DIR/runtimeprog.wasm" +printf runtime-data > "$WASM_POSIX_DEP_OUT_DIR/icu.dat""#, + &[("runtimeprog", "runtimeprog.wasm")], + ); + append_program_runtime_file(&root, "runtimeprog", "icu.dat", "/usr/lib/php/icu.dat"); + let reg = Registry { roots: vec![root] }; + let m = reg.load("runtimeprog").unwrap(); + assert_eq!( + runtime_file_metadata_value(&m, "icu.dat").unwrap(), + serde_json::json!({ + "artifact": "icu.dat", + "guest_path": "/usr/lib/php/icu.dat", + "mode": 420, + "mirror_path": "runtimeprog/icu.dat", + "closure_mirror_paths": [ + "runtimeprog/runtimeprog.wasm", + "runtimeprog/icu.dat", + ], + }) + ); let path = - ensure_built(&m, ®, TEST_ARCH, TEST_ABI, &resolve_opts(&cache, None)).unwrap(); + ensure_built(&m, ®, TargetArch::Wasm32, 4, &resolve_opts(&cache, None)).unwrap(); + assert_eq!(fs::read(path.join("icu.dat")).unwrap(), b"runtime-data"); + + fs::remove_file(path.join("icu.dat")).unwrap(); + let err = validate_cache_artifacts(&m, &path).unwrap_err(); + assert!( + err.contains("runtime file") && err.contains("missing"), + "got: {err}" + ); + } + + #[test] + fn runtime_file_metadata_lists_the_complete_multi_output_closure() { + let manifest = DepsManifest::parse( + r#"kind = "program" +name = "runtimeprog" +version = "1.0" +depends_on = [] +[source] +url = "https://example.test/runtimeprog.tar.gz" +sha256 = "0000000000000000000000000000000000000000000000000000000000000000" +[license] +spdx = "MIT" +[[outputs]] +name = "runtimeprog" +wasm = "bin/runtimeprog.wasm" +[[outputs]] +name = "module" +wasm = "extensions/module.so" +[[runtime_files]] +artifact = "share/icu.dat" +guest_path = "/usr/lib/runtimeprog/icu.dat" +[[runtime_files]] +artifact = "share/timezone.dat" +guest_path = "/usr/lib/runtimeprog/timezone.dat" +"#, + PathBuf::from("/x"), + ) + .unwrap(); - assert!( - path.join("via-build").exists(), - "top-level index ABI mismatch must fall through to source build" + let metadata = runtime_file_metadata_value(&manifest, "share/icu.dat").unwrap(); + assert_eq!( + metadata["closure_mirror_paths"], + serde_json::json!([ + "runtimeprog/runtimeprog.wasm", + "runtimeprog/module.so", + "runtimeprog/share/icu.dat", + "runtimeprog/share/timezone.dat", + ]) ); - let lib = std::fs::read(path.join("lib/out.a")).unwrap(); - assert_ne!(lib, b"REMOTE", "remote bytes must not have been installed"); } - #[test] - fn binaries_dir_program_fetch_does_not_require_built_deps() { - let root = tempdir("prog-bdir-remote-first-reg"); - let cache = tempdir("prog-bdir-remote-first-cache"); - let bin_dir = tempdir("prog-bdir-remote-first-bin"); - let archive_dir = tempdir("prog-bdir-remote-first-archive"); - let index_dir = tempdir("prog-bdir-remote-first-index"); - - write_program( - &root, - "baddep", - "1.0.0", - &[], - "echo baddep source build should not run >&2; exit 42", - &[("baddep", "baddep.wasm")], - ); - write_program( - &root, - "progIdx", - "1.0.0", - &["baddep@1.0.0"], - "echo progIdx source build should not run >&2; exit 43", - &[("progIdx", "progIdx.wasm")], - ); + fn local_generation_manifest() -> DepsManifest { + DepsManifest::parse( + r#"kind = "program" +name = "local-python" +version = "1.0" +depends_on = [] +[source] +url = "https://example.test/local-python.tar.gz" +sha256 = "0000000000000000000000000000000000000000000000000000000000000000" +[license] +spdx = "MIT" +[[outputs]] +name = "python" +wasm = "bin/python.wasm" +[[runtime_files]] +artifact = "share/python-runtime.zip" +guest_path = "/usr/share/local-python/python-runtime.zip" +"#, + PathBuf::from("/local-python"), + ) + .unwrap() + } - let index_path = index_dir.join("index.toml"); - let index_url = format!("file://{}", index_path.display()); - write_program_build_toml(&root, "progIdx", &index_url); + const LOCAL_GENERATION_CACHE_KEY: &str = + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; - let reg = Registry { - roots: vec![root.clone()], - }; - let m = reg.load("progIdx").unwrap(); - let cache_key_hex = hex(&compute_sha( - &m, - ®, + #[test] + fn direct_local_generation_waits_for_complete_closure_and_never_mutates_fetched_targets() { + let root = tempdir("direct-local-generation"); + let binaries = root.join("local-binaries"); + let fetched = root.join("fetched-cache"); + let sources = root.join("build-output"); + let manifest = local_generation_manifest(); + let fetched_wasm = fetched.join("bin/python.wasm"); + let fetched_runtime = fetched.join("share/python-runtime.zip"); + fs::create_dir_all(fetched_wasm.parent().unwrap()).unwrap(); + fs::create_dir_all(fetched_runtime.parent().unwrap()).unwrap(); + let mut fetched_wasm_bytes = minimal_executable_wasm(); + fetched_wasm_bytes.extend(wasm_section(0, wasm_name("fetched-generation"))); + fs::write(&fetched_wasm, &fetched_wasm_bytes).unwrap(); + fs::write(&fetched_runtime, b"FETCHED-RUNTIME").unwrap(); + place_binaries_symlinks(&manifest, &fetched, &binaries, TEST_ARCH).unwrap(); + + let local_wasm = sources.join("python.wasm"); + let local_runtime = sources.join("python-runtime.zip"); + fs::create_dir_all(&sources).unwrap(); + let mut local_wasm_bytes = minimal_executable_wasm(); + local_wasm_bytes.extend(wasm_section(0, wasm_name("local-generation"))); + fs::write(&local_wasm, &local_wasm_bytes).unwrap(); + fs::write(&local_runtime, b"LOCAL-RUNTIME").unwrap(); + + let first = install_local_artifact( + &manifest, + LOCAL_GENERATION_CACHE_KEY, + "python.wasm", + &local_wasm, + "build-one", + &binaries, TEST_ARCH, - TEST_ABI, - &mut BTreeMap::new(), - &mut Vec::new(), ) - .unwrap()); - - let manifest_text = archived_program_manifest_text( - "progIdx", - "progIdx", - "progIdx.wasm", - "wasm32", - &[TEST_ABI], - &cache_key_hex, - ); - let prog_wasm = minimal_executable_wasm(); - let archive_bytes = crate::remote_fetch::build_test_archive( - &manifest_text, - &[("progIdx.wasm", prog_wasm.as_slice())], + .unwrap(); + let generation = binaries + .join(LOCAL_GENERATIONS_DIR) + .join("wasm32") + .join("local-python") + .join(LOCAL_GENERATION_CACHE_KEY) + .join("build-one"); + let generation = fs::canonicalize(generation.parent().unwrap()) + .unwrap() + .join(generation.file_name().unwrap()); + assert_eq!( + first, + LocalArtifactInstall::Staged { + generation: generation.clone(), + remaining: 1, + } ); - let archive_sha_hex = sha256_hex(&archive_bytes); - let archive_path = archive_dir.join("progIdx-1.0.0.tar.zst"); - std::fs::write(&archive_path, &archive_bytes).unwrap(); - let archive_url = format!("file://{}", archive_path.display()); - stage_index_toml( - &index_path, - "progIdx", - TargetArch::Wasm32, - &archive_url, - &archive_sha_hex, - &cache_key_hex, + assert_eq!( + fs::read(generation.join("bin/python.wasm")).unwrap(), + local_wasm_bytes ); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!( + fs::symlink_metadata(generation.join("bin/python.wasm")) + .unwrap() + .permissions() + .mode() + & 0o222, + 0, + "immutable generation member remained writable" + ); + } + assert!(!generation.join("share/python-runtime.zip").exists()); - let opts = ResolveOpts { - cache_root: &cache, - local_libs: None, - force_source_build: None, - fetch_only: false, - repo_root: Some(&root), - binaries_dir: Some(&bin_dir), - }; - let path = ensure_built(&m, ®, TEST_ARCH, TEST_ABI, &opts).unwrap(); - - assert_eq!(std::fs::read(path.join("progIdx.wasm")).unwrap(), prog_wasm); - let baddep_cached = std::fs::read_dir(cache.join("programs")) + let live = fs::canonicalize(binaries.join("programs/wasm32")) .unwrap() - .filter_map(Result::ok) - .any(|entry| entry.file_name().to_string_lossy().starts_with("baddep-")); - assert!( - !baddep_cached, - "binary materialization should not have source-built baddep first" + .join("local-python"); + assert_eq!( + fs::read(live.join("python.wasm")).unwrap(), + fetched_wasm_bytes ); - } - - #[test] - fn fetched_program_runtime_file_matches_source_mirror_layout() { - let root = tempdir("runtime-fetch-parity-reg"); - let remote_cache = tempdir("runtime-fetch-parity-remote-cache"); - let source_cache = tempdir("runtime-fetch-parity-source-cache"); - let remote_bin = tempdir("runtime-fetch-parity-remote-bin"); - let source_bin = tempdir("runtime-fetch-parity-source-bin"); - let archive_dir = tempdir("runtime-fetch-parity-archive"); - let index_dir = tempdir("runtime-fetch-parity-index"); - let index_path = index_dir.join("index.toml"); - let index_url = format!("file://{}", index_path.display()); - let name = "runtimeFetched"; - write_runtime_program_with_index(&root, name, &index_url); + assert_eq!( + fs::read(live.join("share/python-runtime.zip")).unwrap(), + b"FETCHED-RUNTIME" + ); + assert_eq!(fs::read(&fetched_wasm).unwrap(), fetched_wasm_bytes); + assert_eq!(fs::read(&fetched_runtime).unwrap(), b"FETCHED-RUNTIME"); - let reg = Registry { - roots: vec![root.clone()], - }; - let manifest = reg.load(name).unwrap(); - let cache_key_hex = hex(&compute_sha( + let second = install_local_artifact( &manifest, - ®, + LOCAL_GENERATION_CACHE_KEY, + "share/python-runtime.zip", + &local_runtime, + "build-one", + &binaries, TEST_ARCH, - TEST_ABI, - &mut BTreeMap::new(), - &mut Vec::new(), ) - .unwrap()); - let archived_manifest = archived_program_runtime_manifest_text( - name, - TEST_ARCH.as_str(), - &[TEST_ABI], - &cache_key_hex, + .unwrap(); + assert_eq!( + second, + LocalArtifactInstall::Published { + mirror: live.clone(), + generation: generation.clone(), + } ); - let wasm_name = format!("{name}.wasm"); - let wasm_bytes = b"\x00asm\x01\x00\x00\x00\x01\x05\x01\x60\x00\x01\x7f\x03\x02\x01\x00\x07\x1a\x02\x0d__abi_version\x00\x00\x06_start\x00\x00\x0a\x06\x01\x04\x00\x41\x00\x0b"; - let archive_bytes = crate::remote_fetch::build_test_archive( - &archived_manifest, - &[ - (wasm_name.as_str(), wasm_bytes.as_slice()), - ("icu.dat", b"RUNTIME-BYTES"), - ], + assert_eq!( + fs::read(generation.join("share/python-runtime.zip")).unwrap(), + b"LOCAL-RUNTIME" ); - let archive_path = archive_dir.join(format!("{name}-1.0.0.tar.zst")); - fs::write(&archive_path, &archive_bytes).unwrap(); - stage_index_toml( - &index_path, - name, - TEST_ARCH, - &format!("file://{}", archive_path.display()), - &sha256_hex(&archive_bytes), - &cache_key_hex, + assert_eq!( + fs::read(live.join("python.wasm")).unwrap(), + local_wasm_bytes ); - - let remote_opts = ResolveOpts { - cache_root: &remote_cache, - local_libs: None, - force_source_build: None, - fetch_only: false, - repo_root: Some(&root), - binaries_dir: Some(&remote_bin), - }; - let remote_path = ensure_built(&manifest, ®, TEST_ARCH, TEST_ABI, &remote_opts).unwrap(); - assert!(!remote_path.join("via-build").exists()); - place_binaries_symlinks(&manifest, &remote_path, &remote_bin, TEST_ARCH).unwrap(); - - let force = BTreeSet::from([name.to_string()]); - let source_opts = ResolveOpts { - cache_root: &source_cache, - local_libs: None, - force_source_build: Some(&force), - fetch_only: false, - repo_root: Some(&root), - binaries_dir: Some(&source_bin), - }; - let source_path = ensure_built(&manifest, ®, TEST_ARCH, TEST_ABI, &source_opts).unwrap(); - assert!(source_path.join("via-build").exists()); - place_binaries_symlinks(&manifest, &source_path, &source_bin, TEST_ARCH).unwrap(); - - let mirror_rel = Path::new("programs/wasm32").join(name).join("icu.dat"); assert_eq!( - fs::read(remote_bin.join(&mirror_rel)).unwrap(), - b"RUNTIME-BYTES" + fs::read(live.join("share/python-runtime.zip")).unwrap(), + b"LOCAL-RUNTIME" ); assert_eq!( - fs::read(source_bin.join(&mirror_rel)).unwrap(), - b"RUNTIME-BYTES" + fs::read_link(live.join("python.wasm")).unwrap(), + generation.canonicalize().unwrap().join("bin/python.wasm") + ); + assert_eq!( + fs::read_link(live.join("share/python-runtime.zip")).unwrap(), + generation + .canonicalize() + .unwrap() + .join("share/python-runtime.zip") + ); + assert_eq!(fs::read(&fetched_wasm).unwrap(), fetched_wasm_bytes); + assert_eq!(fs::read(&fetched_runtime).unwrap(), b"FETCHED-RUNTIME"); + assert!( + generation + .parent() + .unwrap() + .join(".build-one.publication-claimed") + .is_file() ); - } - #[test] - fn incomplete_fetched_runtime_file_falls_back_or_fails_fetch_only() { - let root = tempdir("runtime-fetch-incomplete-reg"); - let fallback_cache = tempdir("runtime-fetch-incomplete-fallback-cache"); - let fetch_only_cache = tempdir("runtime-fetch-incomplete-only-cache"); - let archive_dir = tempdir("runtime-fetch-incomplete-archive"); - let index_dir = tempdir("runtime-fetch-incomplete-index"); - let index_path = index_dir.join("index.toml"); - let index_url = format!("file://{}", index_path.display()); - let name = "runtimeIncomplete"; - write_runtime_program_with_index(&root, name, &index_url); + let second_wasm = sources.join("python-two.wasm"); + let second_runtime = sources.join("python-runtime-two.zip"); + let mut second_wasm_bytes = minimal_executable_wasm(); + second_wasm_bytes.extend(wasm_section(0, wasm_name("local-generation-two"))); + fs::write(&second_wasm, &second_wasm_bytes).unwrap(); + fs::write(&second_runtime, b"LOCAL-RUNTIME-TWO").unwrap(); + assert!(matches!( + install_local_artifact( + &manifest, + LOCAL_GENERATION_CACHE_KEY, + "python.wasm", + &second_wasm, + "build-two", + &binaries, + TEST_ARCH, + ) + .unwrap(), + LocalArtifactInstall::Staged { remaining: 1, .. } + )); + install_local_artifact( + &manifest, + LOCAL_GENERATION_CACHE_KEY, + "share/python-runtime.zip", + &second_runtime, + "build-two", + &binaries, + TEST_ARCH, + ) + .unwrap(); + assert_eq!( + fs::read(live.join("python.wasm")).unwrap(), + second_wasm_bytes + ); + assert_eq!( + fs::read(live.join("share/python-runtime.zip")).unwrap(), + b"LOCAL-RUNTIME-TWO" + ); - let reg = Registry { - roots: vec![root.clone()], - }; - let manifest = reg.load(name).unwrap(); - let cache_key_hex = hex(&compute_sha( + let stale_error = install_local_artifact( &manifest, - ®, + LOCAL_GENERATION_CACHE_KEY, + "share/python-runtime.zip", + &local_runtime, + "build-one", + &binaries, TEST_ARCH, - TEST_ABI, - &mut BTreeMap::new(), - &mut Vec::new(), ) - .unwrap()); - let archived_manifest = archived_program_runtime_manifest_text( - name, - TEST_ARCH.as_str(), - &[TEST_ABI], - &cache_key_hex, + .unwrap_err(); + assert!( + stale_error.contains("consumed its one publication attempt"), + "got: {stale_error}" + ); + + fs::write(&local_runtime, b"DIFFERENT-RUNTIME").unwrap(); + let error = install_local_artifact( + &manifest, + LOCAL_GENERATION_CACHE_KEY, + "share/python-runtime.zip", + &local_runtime, + "build-one", + &binaries, + TEST_ARCH, + ) + .unwrap_err(); + assert!( + error.contains("immutable") && error.contains("new install session"), + "got: {error}" ); - let wasm_name = format!("{name}.wasm"); - let wasm_bytes = b"\x00asm\x01\x00\x00\x00\x01\x05\x01\x60\x00\x01\x7f\x03\x02\x01\x00\x07\x1a\x02\x0d__abi_version\x00\x00\x06_start\x00\x00\x0a\x06\x01\x04\x00\x41\x00\x0b"; - let archive_bytes = crate::remote_fetch::build_test_archive( - &archived_manifest, - &[(wasm_name.as_str(), wasm_bytes.as_slice())], + assert_eq!( + fs::read(live.join("share/python-runtime.zip")).unwrap(), + b"LOCAL-RUNTIME-TWO" ); - let archive_path = archive_dir.join(format!("{name}-1.0.0.tar.zst")); - fs::write(&archive_path, &archive_bytes).unwrap(); - stage_index_toml( - &index_path, - name, + + fs::remove_dir_all(&generation).unwrap(); + let missing_claimed_error = install_local_artifact( + &manifest, + LOCAL_GENERATION_CACHE_KEY, + "share/python-runtime.zip", + &local_runtime, + "build-one", + &binaries, TEST_ARCH, - &format!("file://{}", archive_path.display()), - &sha256_hex(&archive_bytes), - &cache_key_hex, + ) + .unwrap_err(); + assert!( + missing_claimed_error.contains("claimed local package generation"), + "got: {missing_claimed_error}" ); + assert!( + !generation.exists(), + "producer recreated a publication-claimed generation pathname" + ); + } - let fallback = ensure_built( + #[test] + fn direct_single_member_install_replaces_destination_symlink_without_following_it() { + let manifest = DepsManifest::parse( + r#"kind = "program" +name = "single-local" +version = "1.0" +depends_on = [] +[source] +url = "https://example.test/single-local.tar.gz" +sha256 = "0000000000000000000000000000000000000000000000000000000000000000" +[license] +spdx = "MIT" +[[outputs]] +name = "single-local" +wasm = "single-local.wasm" +"#, + PathBuf::from("/single-local"), + ) + .unwrap(); + let root = tempdir("direct-single-no-follow"); + let binaries = root.join("local-binaries"); + let arch_root = binaries.join("programs/wasm32"); + let fetched = root.join("fetched-cache/single-local.wasm"); + let local = root.join("build-output/single-local.wasm"); + fs::create_dir_all(&arch_root).unwrap(); + fs::create_dir_all(fetched.parent().unwrap()).unwrap(); + fs::create_dir_all(local.parent().unwrap()).unwrap(); + let mut fetched_bytes = minimal_executable_wasm(); + fetched_bytes.extend(wasm_section(0, wasm_name("fetched-single"))); + let mut local_bytes = minimal_executable_wasm(); + local_bytes.extend(wasm_section(0, wasm_name("local-single"))); + fs::write(&fetched, &fetched_bytes).unwrap(); + fs::write(&local, &local_bytes).unwrap(); + let destination = arch_root.join("single-local.wasm"); + symlink_file(&fetched, &destination).unwrap(); + + let outcome = install_local_artifact( &manifest, - ®, + LOCAL_GENERATION_CACHE_KEY, + "single-local.wasm", + &local, + "ignored-for-single", + &binaries, TEST_ARCH, - TEST_ABI, - &resolve_opts(&fallback_cache, None), ) .unwrap(); - assert!(fallback.join("via-build").exists()); assert_eq!( - fs::read(fallback.join("icu.dat")).unwrap(), - b"RUNTIME-BYTES" + outcome, + LocalArtifactInstall::Replaced { + mirror: fs::canonicalize(destination.parent().unwrap()) + .unwrap() + .join(destination.file_name().unwrap()), + } ); + assert_eq!(fs::read(&fetched).unwrap(), fetched_bytes); + assert_eq!(fs::read(&destination).unwrap(), local_bytes); + assert!( + destination + .symlink_metadata() + .unwrap() + .file_type() + .is_symlink() + ); + let target = fs::read_link(&destination).unwrap(); + assert!( + target + .components() + .any(|component| component.as_os_str() == LOCAL_GENERATION_CACHE_KEY) + ); + } - let fetch_only_opts = ResolveOpts { - cache_root: &fetch_only_cache, - local_libs: None, - force_source_build: None, - fetch_only: true, - repo_root: Some(&root), - binaries_dir: None, - }; - let err = ensure_built(&manifest, ®, TEST_ARCH, TEST_ABI, &fetch_only_opts).unwrap_err(); - assert!(err.contains("fetch-only"), "got: {err}"); + fn scalar_local_transaction_manifest() -> DepsManifest { + DepsManifest::parse( + r#"kind = "program" +name = "scalar-local" +version = "1.0" +depends_on = [] +[source] +url = "https://example.test/scalar-local.tar.gz" +sha256 = "0000000000000000000000000000000000000000000000000000000000000000" +[license] +spdx = "MIT" +[[outputs]] +name = "scalar-local" +wasm = "scalar-local.wasm" +"#, + PathBuf::from("/scalar-local"), + ) + .unwrap() + } + + fn assert_no_local_file_transaction_siblings(destination: &Path) { + let parent = destination.parent().unwrap(); + let file_name = destination.file_name().unwrap().to_string_lossy(); + let prefixes = [ + format!(".{file_name}.local-transaction-"), + format!(".{file_name}.symlink-transaction-"), + ]; + let leftovers: Vec<_> = fs::read_dir(parent) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .filter(|name| { + prefixes + .iter() + .any(|prefix| name.to_string_lossy().starts_with(prefix)) + }) + .collect(); assert!( - !canonical_path( - &fetch_only_cache, - &manifest, - TEST_ARCH, - &compute_sha( - &manifest, - ®, - TEST_ARCH, - TEST_ABI, - &mut BTreeMap::new(), - &mut Vec::new(), - ) - .unwrap(), - ) - .join("via-build") - .exists() + leftovers.is_empty(), + "local file transaction left private siblings: {leftovers:?}" ); } #[test] - fn index_fetch_falls_through_on_archive_sha_mismatch() { - let root = tempdir("idx-shafail-reg"); - let cache = tempdir("idx-shafail-cache"); - let archive_dir = tempdir("idx-shafail-archive"); - let index_dir = tempdir("idx-shafail-index"); - - // Build a real archive but advertise the WRONG sha in the index. - let manifest_text = archived_manifest_text( - "libIdxSha", - "wasm32", - &[TEST_ABI], - // cache_key_sha is irrelevant: we never get past the sha - // check. Fill with a valid-shaped dummy so parse_archived - // wouldn't complain (defence in depth). - &"a".repeat(64), + fn scalar_local_transaction_detects_a_directory_swap_and_restores_it() { + let root = tempdir("scalar-local-directory-swap"); + let source = root.join("source.wasm"); + let destination = root.join("scalar-local.wasm"); + let displaced_old = root.join("displaced-old"); + let replacement = root.join("replacement"); + fs::write(&source, minimal_executable_wasm()).unwrap(); + fs::write(&destination, b"old").unwrap(); + fs::create_dir(&replacement).unwrap(); + fs::write(replacement.join("sentinel"), b"user-owned").unwrap(); + let manifest = scalar_local_transaction_manifest(); + + let mut transaction = LocalFileTransaction::prepare( + &manifest, + &source, + &destination, + ForkInstrumentationPolicy::Auto, + &EXECUTABLE_PROGRAM_REQUIRED_EXPORTS, + ) + .unwrap(); + let mut first_rename = true; + let mut swap_before_rename = |from: &Path, to: &Path| { + if first_rename { + first_rename = false; + fs::rename(from, &displaced_old)?; + fs::rename(&replacement, from)?; + } + fs::rename(from, to) + }; + let error = transaction + .move_existing_aside_with(&manifest, &mut swap_before_rename) + .unwrap_err(); + assert!( + error.contains("ownership changed during quarantine") && error.contains("restored"), + "got: {error}" ); - let archive_bytes = - crate::remote_fetch::build_test_archive(&manifest_text, &[("lib/out.a", b"REMOTE")]); - let archive_path = archive_dir.join("libIdxSha-1.0.0.tar.zst"); - std::fs::write(&archive_path, &archive_bytes).unwrap(); - let archive_url = format!("file://{}", archive_path.display()); + drop(transaction); - let bogus_sha = "0".repeat(64); - let index_path = index_dir.join("index.toml"); - stage_index_toml( - &index_path, - "libIdxSha", - TargetArch::Wasm32, - &archive_url, - &bogus_sha, - &"a".repeat(64), + assert_eq!( + fs::read(destination.join("sentinel")).unwrap(), + b"user-owned" ); - let index_url = format!("file://{}", index_path.display()); - write_lib_with_build_toml(&root, "libIdxSha", &index_url); + assert_eq!(fs::read(&displaced_old).unwrap(), b"old"); + assert_no_local_file_transaction_siblings(&destination); + } - let reg = Registry { roots: vec![root] }; - let m = reg.load("libIdxSha").unwrap(); - let path = - ensure_built(&m, ®, TEST_ARCH, TEST_ABI, &resolve_opts(&cache, None)).unwrap(); + #[test] + fn scalar_local_transaction_never_deletes_a_tampered_private_backup() { + let root = tempdir("scalar-local-tampered-backup"); + let source = root.join("source.wasm"); + let destination = root.join("scalar-local.wasm"); + let source_bytes = minimal_executable_wasm(); + fs::write(&source, &source_bytes).unwrap(); + fs::write(&destination, b"old").unwrap(); + let manifest = scalar_local_transaction_manifest(); + let mut transaction = LocalFileTransaction::prepare( + &manifest, + &source, + &destination, + ForkInstrumentationPolicy::Auto, + &EXECUTABLE_PROGRAM_REQUIRED_EXPORTS, + ) + .unwrap(); + let transaction_root = transaction.transaction_root.clone(); + let backup = transaction.backup.clone(); + let displaced_backup = root.join("displaced-backup"); + let mut rename = |from: &Path, to: &Path| fs::rename(from, to); + transaction + .move_existing_aside_with(&manifest, &mut rename) + .unwrap(); + fs::rename(&backup, &displaced_backup).unwrap(); + fs::create_dir(&backup).unwrap(); + fs::write(backup.join("sentinel"), b"do-not-delete").unwrap(); + transaction.publish_with(&manifest, &mut rename).unwrap(); + let error = transaction.finish().unwrap_err(); + assert!(error.contains("refusing to remove changed"), "got: {error}"); - // Source build ran. + assert_eq!(fs::read(&destination).unwrap(), source_bytes); + assert_eq!(fs::read(backup.join("sentinel")).unwrap(), b"do-not-delete"); + assert_eq!(fs::read(&displaced_backup).unwrap(), b"old"); + remove_owned_transaction_path(&transaction_root).unwrap(); + assert_no_local_file_transaction_siblings(&destination); + } + + #[test] + fn scalar_local_transaction_detects_same_length_backup_rewrites() { + let root = tempdir("scalar-local-rewritten-backup"); + let source = root.join("source.wasm"); + let destination = root.join("scalar-local.wasm"); + let source_bytes = minimal_executable_wasm(); + fs::write(&source, &source_bytes).unwrap(); + fs::write(&destination, b"old").unwrap(); + let manifest = scalar_local_transaction_manifest(); + let mut transaction = LocalFileTransaction::prepare( + &manifest, + &source, + &destination, + ForkInstrumentationPolicy::Auto, + &EXECUTABLE_PROGRAM_REQUIRED_EXPORTS, + ) + .unwrap(); + let transaction_root = transaction.transaction_root.clone(); + let backup = transaction.backup.clone(); + let mut rename = |from: &Path, to: &Path| fs::rename(from, to); + transaction + .move_existing_aside_with(&manifest, &mut rename) + .unwrap(); + let old_modified = fs::metadata(&backup).unwrap().modified().unwrap(); + let mut backup_file = fs::OpenOptions::new().write(true).open(&backup).unwrap(); + std::io::Write::write_all(&mut backup_file, b"new").unwrap(); + backup_file + .set_times(std::fs::FileTimes::new().set_modified(old_modified)) + .unwrap(); + drop(backup_file); + transaction.publish_with(&manifest, &mut rename).unwrap(); + let error = transaction.finish().unwrap_err(); + assert!(error.contains("refusing to remove changed"), "got: {error}"); + + assert_eq!(fs::read(&destination).unwrap(), source_bytes); + assert_eq!(fs::read(&backup).unwrap(), b"new"); + remove_owned_transaction_path(&transaction_root).unwrap(); + assert_no_local_file_transaction_siblings(&destination); + } + + #[test] + fn scalar_local_transaction_leaves_a_concurrent_winner_intact() { + let root = tempdir("scalar-local-concurrent-winner"); + let source = root.join("source.wasm"); + let destination = root.join("scalar-local.wasm"); + fs::write(&source, minimal_executable_wasm()).unwrap(); + fs::write(&destination, b"old").unwrap(); + let manifest = scalar_local_transaction_manifest(); + let mut transaction = LocalFileTransaction::prepare( + &manifest, + &source, + &destination, + ForkInstrumentationPolicy::Auto, + &EXECUTABLE_PROGRAM_REQUIRED_EXPORTS, + ) + .unwrap(); + let mut rename = |from: &Path, to: &Path| fs::rename(from, to); + transaction + .move_existing_aside_with(&manifest, &mut rename) + .unwrap(); + fs::write(&destination, b"concurrent-winner").unwrap(); + let error = transaction + .publish_with(&manifest, &mut rename) + .unwrap_err(); assert!( - path.join("via-build").exists(), - "sha mismatch must fall through to source build" + error.contains("another writer installed an entry"), + "got: {error}" ); - let lib = std::fs::read(path.join("lib/out.a")).unwrap(); - assert_ne!(lib, b"REMOTE", "remote bytes must not have been installed"); + drop(transaction); + + assert_eq!(fs::read(&destination).unwrap(), b"concurrent-winner"); + assert_no_local_file_transaction_siblings(&destination); } + #[cfg(unix)] #[test] - fn index_fetch_falls_through_on_target_arch_mismatch() { - let root = tempdir("idx-archfail-reg"); - let cache = tempdir("idx-archfail-cache"); - let archive_dir = tempdir("idx-archfail-archive"); - let index_dir = tempdir("idx-archfail-index"); - - // Archive's internal compatibility block declares wasm64 — - // resolver requests wasm32 (TEST_ARCH). The index entry - // points the wasm32 slot at this archive (an - // archive-staging bug a real CI would never produce, but - // the resolver must defend against it). - let manifest_text = - archived_manifest_text("libIdxArch", "wasm64", &[TEST_ABI], &"a".repeat(64)); - let archive_bytes = - crate::remote_fetch::build_test_archive(&manifest_text, &[("lib/out.a", b"REMOTE")]); - let archive_sha = sha256_hex(&archive_bytes); - let archive_path = archive_dir.join("libIdxArch-1.0.0.tar.zst"); - std::fs::write(&archive_path, &archive_bytes).unwrap(); - let archive_url = format!("file://{}", archive_path.display()); + fn scalar_local_transaction_reserves_its_private_parent_as_mode_0700() { + use std::os::unix::fs::PermissionsExt; - let index_path = index_dir.join("index.toml"); - stage_index_toml( - &index_path, - "libIdxArch", - TargetArch::Wasm32, - &archive_url, - &archive_sha, - &"a".repeat(64), + let root = tempdir("scalar-local-private-mode"); + let source = root.join("source.wasm"); + let destination = root.join("scalar-local.wasm"); + fs::write(&source, minimal_executable_wasm()).unwrap(); + let manifest = scalar_local_transaction_manifest(); + let transaction = LocalFileTransaction::prepare( + &manifest, + &source, + &destination, + ForkInstrumentationPolicy::Auto, + &EXECUTABLE_PROGRAM_REQUIRED_EXPORTS, + ) + .unwrap(); + assert_eq!( + fs::symlink_metadata(&transaction.transaction_root) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o700 ); - let index_url = format!("file://{}", index_path.display()); - write_lib_with_build_toml(&root, "libIdxArch", &index_url); + drop(transaction); + assert_no_local_file_transaction_siblings(&destination); + } - let reg = Registry { roots: vec![root] }; - let m = reg.load("libIdxArch").unwrap(); - let path = ensure_built( - &m, - ®, - TEST_ARCH, // wasm32 - TEST_ABI, - &resolve_opts(&cache, None), + #[test] + fn scalar_local_transaction_revalidates_policy_after_the_source_changes() { + let root = tempdir("scalar-local-source-policy-race"); + let source = root.join("source.wasm"); + let destination = root.join("scalar-local.wasm"); + fs::write(&source, minimal_executable_wasm()).unwrap(); + fs::write(&destination, b"old").unwrap(); + validate_wasm_artifact_policy( + &source, + ForkInstrumentationPolicy::Auto, + &EXECUTABLE_PROGRAM_REQUIRED_EXPORTS, ) .unwrap(); + // Model a rebuild replacing the same source pathname after the caller's + // initial validation but before the transaction copies it. + fs::write(&source, b"not wasm anymore").unwrap(); + let manifest = scalar_local_transaction_manifest(); + let error = match LocalFileTransaction::prepare( + &manifest, + &source, + &destination, + ForkInstrumentationPolicy::Auto, + &EXECUTABLE_PROGRAM_REQUIRED_EXPORTS, + ) { + Ok(_) => panic!("invalid staged Wasm unexpectedly passed policy validation"), + Err(error) => error, + }; + assert!(error.contains("is not a wasm binary"), "got: {error}"); + assert_eq!(fs::read(&destination).unwrap(), b"old"); + assert_no_local_file_transaction_siblings(&destination); + } + + #[test] + fn scalar_symlink_transaction_refuses_regular_files_and_directories() { + let root = tempdir("scalar-symlink-refuses-user-entries"); + let target = root.join("target.wasm"); + let regular = root.join("regular.wasm"); + let directory = root.join("directory.wasm"); + fs::write(&target, minimal_executable_wasm()).unwrap(); + fs::write(®ular, b"user regular file").unwrap(); + fs::create_dir(&directory).unwrap(); + fs::write(directory.join("sentinel"), b"user directory").unwrap(); + let manifest = scalar_local_transaction_manifest(); + + for destination in [®ular, &directory] { + let mut transaction = + LocalFileTransaction::prepare_symlink(&manifest, &target, destination).unwrap(); + let mut rename = |from: &Path, to: &Path| fs::rename(from, to); + let error = transaction + .move_existing_aside_with(&manifest, &mut rename) + .unwrap_err(); + assert!( + error.contains("refusing to replace regular file") + || error.contains("refusing to replace non-file"), + "got: {error}", + ); + drop(transaction); + assert_no_local_file_transaction_siblings(destination); + } + assert_eq!(fs::read(®ular).unwrap(), b"user regular file"); + assert_eq!( + fs::read(directory.join("sentinel")).unwrap(), + b"user directory", + ); + } + + #[test] + fn scalar_symlink_transaction_preserves_the_old_link_on_publish_failure() { + let root = tempdir("scalar-symlink-publish-failure"); + let old_target = root.join("old.wasm"); + let new_target = root.join("new.wasm"); + let destination = root.join("scalar-local.wasm"); + fs::write(&old_target, b"old").unwrap(); + fs::write(&new_target, b"new").unwrap(); + symlink_file(&old_target, &destination).unwrap(); + let manifest = scalar_local_transaction_manifest(); + let mut transaction = + LocalFileTransaction::prepare_symlink(&manifest, &new_target, &destination).unwrap(); + let mut rename = |from: &Path, to: &Path| fs::rename(from, to); + transaction + .move_existing_aside_with(&manifest, &mut rename) + .unwrap(); + let mut fail_publish = |_stage: &Path, _destination: &Path| { + Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "injected scalar symlink publication failure", + )) + }; + let error = transaction + .publish_with_operation(&manifest, &mut rename, &mut fail_publish) + .unwrap_err(); + assert!( + error.contains("injected scalar symlink publication failure"), + "got: {error}", + ); + drop(transaction); + + assert_eq!(fs::read_link(&destination).unwrap(), old_target); + assert_no_local_file_transaction_siblings(&destination); + } + + #[test] + fn scalar_symlink_transaction_leaves_a_concurrent_winner_intact() { + let root = tempdir("scalar-symlink-concurrent-winner"); + let old_target = root.join("old.wasm"); + let new_target = root.join("new.wasm"); + let winner = root.join("winner.wasm"); + let destination = root.join("scalar-local.wasm"); + fs::write(&old_target, b"old").unwrap(); + fs::write(&new_target, b"new").unwrap(); + fs::write(&winner, b"winner").unwrap(); + symlink_file(&old_target, &destination).unwrap(); + let manifest = scalar_local_transaction_manifest(); + let mut transaction = + LocalFileTransaction::prepare_symlink(&manifest, &new_target, &destination).unwrap(); + let mut rename = |from: &Path, to: &Path| fs::rename(from, to); + transaction + .move_existing_aside_with(&manifest, &mut rename) + .unwrap(); + symlink_file(&winner, &destination).unwrap(); + let error = transaction + .publish_with(&manifest, &mut rename) + .unwrap_err(); assert!( - path.join("via-build").exists(), - "arch mismatch must fall through to source build" + error.contains("another writer installed an entry"), + "got: {error}", ); + drop(transaction); + + assert_eq!(fs::read_link(&destination).unwrap(), winner); + assert_no_local_file_transaction_siblings(&destination); } #[test] - fn index_fetch_falls_through_on_abi_mismatch() { - let root = tempdir("idx-abifail-reg"); - let cache = tempdir("idx-abifail-cache"); - let archive_dir = tempdir("idx-abifail-archive"); - let index_dir = tempdir("idx-abifail-index"); - - // Archive supports only ABI 999 — resolver passes TEST_ABI. - let manifest_text = archived_manifest_text("libIdxAbi", "wasm32", &[999], &"a".repeat(64)); - let archive_bytes = - crate::remote_fetch::build_test_archive(&manifest_text, &[("lib/out.a", b"REMOTE")]); - let archive_sha = sha256_hex(&archive_bytes); - let archive_path = archive_dir.join("libIdxAbi-1.0.0.tar.zst"); - std::fs::write(&archive_path, &archive_bytes).unwrap(); - let archive_url = format!("file://{}", archive_path.display()); + fn scalar_symlink_explicit_rollback_cannot_overwrite_a_late_winner() { + let root = tempdir("scalar-symlink-explicit-rollback-winner"); + let old_target = root.join("old.wasm"); + let new_target = root.join("new.wasm"); + let winner = root.join("winner.wasm"); + let destination = root.join("scalar-local.wasm"); + fs::write(&old_target, b"old").unwrap(); + fs::write(&new_target, b"new").unwrap(); + fs::write(&winner, b"winner").unwrap(); + symlink_file(&old_target, &destination).unwrap(); + let manifest = scalar_local_transaction_manifest(); + let mut transaction = + LocalFileTransaction::prepare_symlink(&manifest, &new_target, &destination).unwrap(); + let mut rename = |from: &Path, to: &Path| fs::rename(from, to); + transaction + .move_existing_aside_with(&manifest, &mut rename) + .unwrap(); + let mut fail_publish = |_stage: &Path, _destination: &Path| { + Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "injected publication failure before rollback", + )) + }; + let mut restore_attempted = false; + let mut install_winner_before_restore = |from: &Path, to: &Path| { + restore_attempted = true; + symlink_file(&winner, to)?; + rename_entry_no_replace(from, to) + }; - let index_path = index_dir.join("index.toml"); - stage_index_toml( - &index_path, - "libIdxAbi", - TargetArch::Wasm32, - &archive_url, - &archive_sha, - &"a".repeat(64), + let error = transaction + .publish_with_operations( + &manifest, + &mut rename, + &mut fail_publish, + &mut install_winner_before_restore, + ) + .unwrap_err(); + assert!(restore_attempted); + assert!( + error.contains("concurrent writer won before rollback") + && error.contains("left intact"), + "got: {error}", ); - let index_url = format!("file://{}", index_path.display()); - write_lib_with_build_toml(&root, "libIdxAbi", &index_url); + drop(transaction); - let reg = Registry { roots: vec![root] }; - let m = reg.load("libIdxAbi").unwrap(); - let path = ensure_built( - &m, - ®, - TEST_ARCH, - TEST_ABI, // not in [999] - &resolve_opts(&cache, None), - ) - .unwrap(); + assert_eq!(fs::read_link(&destination).unwrap(), winner); + assert_no_local_file_transaction_siblings(&destination); + } + #[test] + fn scalar_symlink_quarantine_recovery_cannot_overwrite_a_late_winner() { + let root = tempdir("scalar-symlink-quarantine-winner"); + let old_target = root.join("old.wasm"); + let new_target = root.join("new.wasm"); + let substituted_target = root.join("substituted.wasm"); + let winner = root.join("winner.wasm"); + let destination = root.join("scalar-local.wasm"); + let displaced_old = root.join("displaced-old"); + for (path, bytes) in [ + (&old_target, b"old".as_slice()), + (&new_target, b"new".as_slice()), + (&substituted_target, b"substituted".as_slice()), + (&winner, b"winner".as_slice()), + ] { + fs::write(path, bytes).unwrap(); + } + symlink_file(&old_target, &destination).unwrap(); + let manifest = scalar_local_transaction_manifest(); + let mut transaction = + LocalFileTransaction::prepare_symlink(&manifest, &new_target, &destination).unwrap(); + let transaction_root = transaction.transaction_root.clone(); + let backup = transaction.backup.clone(); + let mut first_rename = true; + let mut substitute_before_quarantine = |from: &Path, to: &Path| { + if first_rename { + first_rename = false; + fs::rename(from, &displaced_old)?; + symlink_file(&substituted_target, from)?; + } + fs::rename(from, to) + }; + let mut install_winner_before_restore = |from: &Path, to: &Path| { + symlink_file(&winner, to)?; + rename_entry_no_replace(from, to) + }; + + let error = transaction + .move_existing_aside_with_restore( + &manifest, + &mut substitute_before_quarantine, + &mut install_winner_before_restore, + ) + .unwrap_err(); assert!( - path.join("via-build").exists(), - "abi mismatch must fall through to source build" + error.contains("ownership changed during quarantine") + && error.contains("concurrent entry") + && error.contains("left intact"), + "got: {error}", ); + drop(transaction); + + assert_eq!(fs::read_link(&destination).unwrap(), winner); + assert_eq!(fs::read_link(&backup).unwrap(), substituted_target); + assert_eq!(fs::read_link(&displaced_old).unwrap(), old_target); + remove_owned_transaction_path(&transaction_root).unwrap(); + assert_no_local_file_transaction_siblings(&destination); } #[test] - fn index_fetch_falls_through_on_cache_key_mismatch() { - let root = tempdir("idx-ckfail-reg"); - let cache = tempdir("idx-ckfail-cache"); - let archive_dir = tempdir("idx-ckfail-archive"); - let index_dir = tempdir("idx-ckfail-index"); + fn scalar_symlink_drop_recovery_cannot_overwrite_a_late_winner() { + let root = tempdir("scalar-symlink-drop-rollback-winner"); + let old_target = root.join("old.wasm"); + let new_target = root.join("new.wasm"); + let winner = root.join("winner.wasm"); + let destination = root.join("scalar-local.wasm"); + fs::write(&old_target, b"old").unwrap(); + fs::write(&new_target, b"new").unwrap(); + fs::write(&winner, b"winner").unwrap(); + symlink_file(&old_target, &destination).unwrap(); + let manifest = scalar_local_transaction_manifest(); + let mut transaction = + LocalFileTransaction::prepare_symlink(&manifest, &new_target, &destination).unwrap(); + let mut rename = |from: &Path, to: &Path| fs::rename(from, to); + transaction + .move_existing_aside_with(&manifest, &mut rename) + .unwrap(); + let mut install_winner_before_restore = |from: &Path, to: &Path| { + symlink_file(&winner, to)?; + rename_entry_no_replace(from, to) + }; - // Archive's internal compat.cache_key_sha is well-formed but - // doesn't match what compute_sha would produce for this lib. - let wrong_ck = "f".repeat(64); - let manifest_text = archived_manifest_text("libIdxCk", "wasm32", &[TEST_ABI], &wrong_ck); - let archive_bytes = - crate::remote_fetch::build_test_archive(&manifest_text, &[("lib/out.a", b"REMOTE")]); - let archive_sha = sha256_hex(&archive_bytes); - let archive_path = archive_dir.join("libIdxCk-1.0.0.tar.zst"); - std::fs::write(&archive_path, &archive_bytes).unwrap(); - let archive_url = format!("file://{}", archive_path.display()); + // Exercise the exact helper Drop uses, with the competing writer + // injected at the no-replace rename boundary. + transaction.restore_unpublished_backup_with(&mut install_winner_before_restore); + assert_eq!(fs::read_link(&destination).unwrap(), winner); + drop(transaction); - let index_path = index_dir.join("index.toml"); - stage_index_toml( - &index_path, - "libIdxCk", - TargetArch::Wasm32, - &archive_url, - &archive_sha, - &wrong_ck, - ); - let index_url = format!("file://{}", index_path.display()); - write_lib_with_build_toml(&root, "libIdxCk", &index_url); + assert_eq!(fs::read_link(&destination).unwrap(), winner); + assert_no_local_file_transaction_siblings(&destination); + } - let reg = Registry { roots: vec![root] }; - let m = reg.load("libIdxCk").unwrap(); - let path = - ensure_built(&m, ®, TEST_ARCH, TEST_ABI, &resolve_opts(&cache, None)).unwrap(); + #[test] + fn scalar_symlink_transaction_never_deletes_a_tampered_private_backup() { + let root = tempdir("scalar-symlink-tampered-backup"); + let old_target = root.join("old.wasm"); + let new_target = root.join("new.wasm"); + let foreign_target = root.join("foreign.wasm"); + let destination = root.join("scalar-local.wasm"); + fs::write(&old_target, b"old").unwrap(); + fs::write(&new_target, b"new").unwrap(); + fs::write(&foreign_target, b"foreign").unwrap(); + symlink_file(&old_target, &destination).unwrap(); + let manifest = scalar_local_transaction_manifest(); + let mut transaction = + LocalFileTransaction::prepare_symlink(&manifest, &new_target, &destination).unwrap(); + let transaction_root = transaction.transaction_root.clone(); + let backup = transaction.backup.clone(); + let displaced_backup = root.join("displaced-backup"); + let mut rename = |from: &Path, to: &Path| fs::rename(from, to); + transaction + .move_existing_aside_with(&manifest, &mut rename) + .unwrap(); + fs::rename(&backup, &displaced_backup).unwrap(); + symlink_file(&foreign_target, &backup).unwrap(); + transaction.publish_with(&manifest, &mut rename).unwrap(); + let error = transaction.finish().unwrap_err(); + assert!(error.contains("refusing to remove changed"), "got: {error}",); - assert!( - path.join("via-build").exists(), - "cache_key_sha mismatch must fall through to source build" + assert_eq!( + fs::read_link(&destination).unwrap(), + fs::canonicalize(&new_target).unwrap(), ); + assert_eq!(fs::read_link(&backup).unwrap(), foreign_target); + assert_eq!(fs::read_link(&displaced_backup).unwrap(), old_target); + remove_owned_transaction_path(&transaction_root).unwrap(); + assert_no_local_file_transaction_siblings(&destination); } + #[cfg(unix)] #[test] - fn fetch_only_rejects_missing_index_entry_without_source_build() { - let root = tempdir("fetch-only-missing-reg"); - let cache = tempdir("fetch-only-missing-cache"); - let index_dir = tempdir("fetch-only-missing-index"); + fn scalar_symlink_transaction_reserves_its_private_parent_as_mode_0700() { + use std::os::unix::fs::PermissionsExt; - let index_path = index_dir.join("index.toml"); - std::fs::write( - &index_path, - format!( - r#"abi_version = {TEST_ABI} -generated_at = "2026-06-09T00:00:00Z" -generator = "test" + let root = tempdir("scalar-symlink-private-mode"); + let target = root.join("target.wasm"); + let destination = root.join("scalar-local.wasm"); + fs::write(&target, minimal_executable_wasm()).unwrap(); + let manifest = scalar_local_transaction_manifest(); + let transaction = + LocalFileTransaction::prepare_symlink(&manifest, &target, &destination).unwrap(); + assert_eq!( + fs::symlink_metadata(&transaction.transaction_root) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o700, + ); + drop(transaction); + assert_no_local_file_transaction_siblings(&destination); + } + + fn atomic_mirror_manifest(runtime_artifact: &str) -> DepsManifest { + DepsManifest::parse( + &format!( + r#"kind = "program" +name = "atomic-shell" +version = "1.0" +depends_on = [] +[source] +url = "https://example.test/atomic-shell.tar.gz" +sha256 = "0000000000000000000000000000000000000000000000000000000000000000" +[license] +spdx = "MIT" +[[outputs]] +name = "shell" +wasm = "image/shell.vfs.zst" +[[outputs]] +name = "homebrew" +wasm = "archives/homebrew-bootstrap.zip" +[[runtime_files]] +artifact = {runtime_artifact:?} +guest_path = "/usr/share/atomic-shell/runtime/index.dat" "# ), + PathBuf::from("/atomic-shell"), ) - .unwrap(); - let index_url = format!("file://{}", index_path.display()); - write_lib_with_build_toml(&root, "libFetchOnly", &index_url); + .unwrap() + } - let reg = Registry { - roots: vec![root.clone()], - }; - let m = reg.load("libFetchOnly").unwrap(); - let sha = compute_sha( - &m, - ®, - TEST_ARCH, - TEST_ABI, - &mut BTreeMap::new(), - &mut Vec::new(), - ) - .unwrap(); - let canonical = canonical_path(&cache, &m, TEST_ARCH, &sha); - let opts = ResolveOpts { - cache_root: &cache, - local_libs: None, - force_source_build: None, - fetch_only: true, - repo_root: Some(&root), - binaries_dir: None, - }; + fn populate_atomic_mirror_identity(canonical: &Path, manifest: &DepsManifest, label: &str) { + for (artifact, suffix) in manifest + .program_outputs + .iter() + .map(|output| (output.wasm.as_str(), "output")) + .chain( + manifest + .runtime_files + .iter() + .map(|runtime_file| (runtime_file.artifact.as_str(), "runtime")), + ) + { + let artifact_path = canonical.join(artifact); + fs::create_dir_all(artifact_path.parent().unwrap()).unwrap(); + fs::write(artifact_path, format!("{label}-{suffix}-{artifact}\n")).unwrap(); + } + } - let err = ensure_built(&m, ®, TEST_ARCH, TEST_ABI, &opts).unwrap_err(); - assert!(err.contains("fetch-only resolve"), "got: {err}"); + fn write_atomic_mirror_fixture(plan: &PackageClosureMirrorPlan) { + fs::create_dir_all(&plan.package_dir).unwrap(); + for link in &plan.links { + let destination = plan.package_dir.join(&link.package_relative); + fs::create_dir_all(destination.parent().unwrap()).unwrap(); + symlink_file(&link.source, &destination).unwrap(); + } + } + + fn assert_no_atomic_mirror_transaction_siblings(package_dir: &Path) { + let parent = package_dir.parent().unwrap(); + let package_name = package_dir.file_name().unwrap().to_string_lossy(); + let transaction_prefixes = [format!(".{package_name}.transaction-")]; + let leftovers: Vec = fs::read_dir(parent) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .filter(|name| { + transaction_prefixes + .iter() + .any(|prefix| name.starts_with(prefix)) + }) + .collect(); assert!( - !canonical.join("via-build").exists(), - "fetch-only must not run the source build script" + leftovers.is_empty(), + "transaction left private siblings behind: {leftovers:?}" ); } - // --- kind = "program" resolver tests (Task B.2) --- + #[derive(Debug, Eq, PartialEq)] + enum AtomicMirrorReaderState { + CompleteOld, + Absent, + CompleteNew, + MixedOrInvalid(BTreeMap), + } - /// Create a `kind = "program"` package.toml + build-.sh pair. - /// Mirrors `write_lib` but emits `[[outputs]]` array-of-tables. - fn write_program( - root: &Path, - name: &str, - version: &str, - deps: &[&str], - build_script_body: &str, - outputs: &[(&str, &str)], - ) { - let dir = root.join(name); - fs::create_dir_all(&dir).unwrap(); - let depends_on = deps - .iter() - .map(|d| format!("\"{}\"", d)) - .collect::>() - .join(", "); - let mut outputs_toml = String::new(); - for (n, w) in outputs { - outputs_toml.push_str(&format!("[[outputs]]\nname = \"{n}\"\nwasm = \"{w}\"\n\n")); + fn atomic_mirror_reader_state( + live_dir: &Path, + old_links: &BTreeMap, + new_links: &BTreeMap, + ) -> AtomicMirrorReaderState { + if !path_entry_exists(live_dir).unwrap() { + return AtomicMirrorReaderState::Absent; } - fs::write( - dir.join("package.toml"), - format!( - r#"kind = "program" -name = "{name}" -version = "{version}" -depends_on = [{depends_on}] -[source] -url = "https://example.test/{name}.tar.gz" -sha256 = "0000000000000000000000000000000000000000000000000000000000000000" -[license] -spdx = "MIT" -{outputs_toml}"#, - ), + let links = read_package_mirror_links(live_dir).unwrap(); + if &links == old_links { + AtomicMirrorReaderState::CompleteOld + } else if &links == new_links { + AtomicMirrorReaderState::CompleteNew + } else { + AtomicMirrorReaderState::MixedOrInvalid(links) + } + } + + #[test] + fn multi_output_mirror_two_rename_boundaries_expose_only_complete_or_absent_states() { + let root = tempdir("atomic-mirror-reader-states"); + let arch_root = root.join("binaries/programs/wasm32"); + let old_canonical = root.join("cache/old-identity"); + let new_canonical = root.join("cache/new-identity"); + let manifest = atomic_mirror_manifest("share/runtime/nested/index.dat"); + populate_atomic_mirror_identity(&old_canonical, &manifest, "old"); + populate_atomic_mirror_identity(&new_canonical, &manifest, "new"); + let old_plan = + PackageClosureMirrorPlan::validate(&manifest, &old_canonical, &arch_root).unwrap(); + let new_plan = + PackageClosureMirrorPlan::validate(&manifest, &new_canonical, &arch_root).unwrap(); + let old_links = old_plan.expected_links(); + let new_links = new_plan.expected_links(); + let live_dir = new_plan.package_dir.clone(); + write_atomic_mirror_fixture(&old_plan); + + let mut transaction = PackageDirectoryTransaction::prepare(new_plan).unwrap(); + assert_eq!( + atomic_mirror_reader_state(&live_dir, &old_links, &new_links), + AtomicMirrorReaderState::CompleteOld + ); + + let mut rename = |from: &Path, to: &Path| fs::rename(from, to); + transaction.move_existing_aside_with(&mut rename).unwrap(); + assert_eq!( + atomic_mirror_reader_state(&live_dir, &old_links, &new_links), + AtomicMirrorReaderState::Absent + ); + + transaction.publish_with(&mut rename).unwrap(); + assert_eq!( + atomic_mirror_reader_state(&live_dir, &old_links, &new_links), + AtomicMirrorReaderState::CompleteNew + ); + transaction.finish().unwrap(); + assert_no_atomic_mirror_transaction_siblings(&live_dir); + } + + #[test] + fn multi_output_mirror_replaces_preexisting_mixed_and_stale_links_as_one_directory() { + let root = tempdir("atomic-mirror-replace-mixed"); + let binaries = root.join("binaries"); + let arch_root = binaries.join("programs/wasm32"); + let old_canonical = root.join("cache/old-identity"); + let new_canonical = root.join("cache/new-identity"); + let manifest = atomic_mirror_manifest("share/runtime/nested/index.dat"); + populate_atomic_mirror_identity(&old_canonical, &manifest, "old"); + populate_atomic_mirror_identity(&new_canonical, &manifest, "new"); + let old_plan = + PackageClosureMirrorPlan::validate(&manifest, &old_canonical, &arch_root).unwrap(); + let new_plan = + PackageClosureMirrorPlan::validate(&manifest, &new_canonical, &arch_root).unwrap(); + let live_dir = new_plan.package_dir.clone(); + fs::create_dir_all(live_dir.join("share/runtime/nested")).unwrap(); + symlink_file( + &old_plan.links[0].source, + &live_dir.join(&old_plan.links[0].package_relative), ) .unwrap(); - let script_path = dir.join(format!("build-{name}.sh")); - fs::write( - &script_path, - format!("#!/bin/bash\nset -e\n{build_script_body}\n"), + symlink_file( + &new_plan.links[1].source, + &live_dir.join(&new_plan.links[1].package_relative), + ) + .unwrap(); + symlink_file( + &old_plan.links[2].source, + &live_dir.join(&old_plan.links[2].package_relative), ) .unwrap(); + symlink_file(&old_plan.links[0].source, &live_dir.join("stale-extra")).unwrap(); - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let mut p = std::fs::metadata(&script_path).unwrap().permissions(); - p.set_mode(0o755); - std::fs::set_permissions(&script_path, p).unwrap(); - } + place_binaries_symlinks(&manifest, &new_canonical, &binaries, TEST_ARCH).unwrap(); + + assert_eq!( + read_package_mirror_links(&live_dir).unwrap(), + new_plan.expected_links() + ); + assert_no_atomic_mirror_transaction_siblings(&live_dir); } - fn append_program_runtime_file(root: &Path, name: &str, artifact: &str, guest_path: &str) { - let manifest_path = root.join(name).join("package.toml"); - let mut text = fs::read_to_string(&manifest_path).unwrap(); - text.push_str(&format!( - "\n[[runtime_files]]\nartifact = {artifact:?}\nguest_path = {guest_path:?}\n" - )); - fs::write(manifest_path, text).unwrap(); + #[cfg(unix)] + #[test] + fn matching_fetched_package_mirror_is_a_true_no_op() { + use std::os::unix::fs::MetadataExt; + + let root = tempdir("atomic-mirror-no-op"); + let binaries = root.join("binaries"); + let canonical = root.join("cache/identity"); + let manifest = atomic_mirror_manifest("share/runtime/nested/index.dat"); + populate_atomic_mirror_identity(&canonical, &manifest, "same"); + + place_binaries_symlinks(&manifest, &canonical, &binaries, TEST_ARCH).unwrap(); + let live = binaries.join("programs/wasm32/atomic-shell"); + let before = fs::symlink_metadata(&live).unwrap().ino(); + place_binaries_symlinks(&manifest, &canonical, &binaries, TEST_ARCH).unwrap(); + let after = fs::symlink_metadata(&live).unwrap().ino(); + + assert_eq!( + before, after, + "no-op publication replaced the live directory" + ); + assert_no_atomic_mirror_transaction_siblings(&live); } #[test] - fn canonical_path_uses_programs_subdir_for_program_kind() { - let m = DepsManifest::parse( + fn fetched_multi_to_scalar_transition_leaves_the_old_directory_inert() { + let root = tempdir("atomic-mirror-multi-to-scalar"); + let binaries = root.join("binaries"); + let old_canonical = root.join("cache/old"); + let old_manifest = atomic_mirror_manifest("share/runtime/index.dat"); + populate_atomic_mirror_identity(&old_canonical, &old_manifest, "old"); + place_binaries_symlinks(&old_manifest, &old_canonical, &binaries, TEST_ARCH).unwrap(); + let old_live = binaries.join("programs/wasm32/atomic-shell"); + assert!(old_live.is_dir()); + + let scalar_manifest = DepsManifest::parse( r#"kind = "program" -name = "vim" -version = "9.1.0900" +name = "atomic-shell" +version = "2.0" +depends_on = [] [source] -url = "https://x.test/vim.tar.gz" +url = "https://example.test/atomic-shell.tar.gz" sha256 = "0000000000000000000000000000000000000000000000000000000000000000" [license] -spdx = "Vim" +spdx = "MIT" [[outputs]] -name = "vim" -wasm = "vim.wasm" +name = "shell" +wasm = "shell.zip" "#, - PathBuf::from("/x"), + PathBuf::from("/atomic-shell"), ) .unwrap(); - let sha = [0u8; 32]; - let p = canonical_path(Path::new("/cache"), &m, TargetArch::Wasm32, &sha); - let s = p.to_string_lossy(); - assert!(s.contains("/programs/"), "got: {s}"); - assert!(s.contains("vim-9.1.0900-rev1-wasm32-"), "got: {s}"); - } - - #[test] - fn build_validates_program_wasm_outputs_present() { - let root = tempdir("prog-out-pass"); - let cache = tempdir("prog-out-pass-cache"); - write_program( - &root, - "tinyprog", - "0.1.0", - &[], - // Build script writes the declared wasm. - r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR" && printf '\x00asm\x01\x00\x00\x00\x01\x05\x01\x60\x00\x01\x7f\x03\x02\x01\x00\x07\x1a\x02\x0d__abi_version\x00\x00\x06_start\x00\x00\x0a\x06\x01\x04\x00\x41\x00\x0b' > "$WASM_POSIX_DEP_OUT_DIR/tinyprog.wasm""#, - &[("tinyprog", "tinyprog.wasm")], - ); - let reg = Registry { roots: vec![root] }; - let m = reg.load("tinyprog").unwrap(); - ensure_built(&m, ®, TargetArch::Wasm32, 4, &resolve_opts(&cache, None)).unwrap(); - } - - #[test] - fn build_fails_when_program_wasm_output_missing() { - let root = tempdir("prog-out-miss"); - let cache = tempdir("prog-out-miss-cache"); - write_program( - &root, - "miss", - "0.1.0", - &[], - // Build script does NOT produce miss.wasm. - r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR""#, - &[("miss", "miss.wasm")], - ); - let reg = Registry { roots: vec![root] }; - let m = reg.load("miss").unwrap(); - let err = - ensure_built(&m, ®, TargetArch::Wasm32, 4, &resolve_opts(&cache, None)).unwrap_err(); - assert!(err.contains("miss.wasm"), "got: {err}"); - } + let scalar_canonical = root.join("cache/scalar"); + fs::create_dir_all(&scalar_canonical).unwrap(); + fs::write(scalar_canonical.join("shell.zip"), b"scalar").unwrap(); - #[test] - fn program_runtime_file_is_required_and_cached_as_a_regular_file() { - let root = tempdir("prog-runtime-file"); - let cache = tempdir("prog-runtime-file-cache"); - write_program( - &root, - "runtimeprog", - "0.1.0", - &[], - r#"mkdir -p "$WASM_POSIX_DEP_OUT_DIR" -printf '\x00asm\x01\x00\x00\x00\x01\x05\x01\x60\x00\x01\x7f\x03\x02\x01\x00\x07\x1a\x02\x0d__abi_version\x00\x00\x06_start\x00\x00\x0a\x06\x01\x04\x00\x41\x00\x0b' > "$WASM_POSIX_DEP_OUT_DIR/runtimeprog.wasm" -printf runtime-data > "$WASM_POSIX_DEP_OUT_DIR/icu.dat""#, - &[("runtimeprog", "runtimeprog.wasm")], + place_binaries_symlinks(&scalar_manifest, &scalar_canonical, &binaries, TEST_ARCH).unwrap(); + assert!( + old_live.is_dir(), + "scalar publication must not delete a path a concurrent package publisher can own" ); - append_program_runtime_file(&root, "runtimeprog", "icu.dat", "/usr/lib/php/icu.dat"); - let reg = Registry { roots: vec![root] }; - let m = reg.load("runtimeprog").unwrap(); assert_eq!( - runtime_file_metadata_value(&m, "icu.dat").unwrap(), - serde_json::json!({ - "artifact": "icu.dat", - "guest_path": "/usr/lib/php/icu.dat", - "mode": 420, - "mirror_path": "runtimeprog/icu.dat", - "closure_mirror_paths": [ - "runtimeprog.wasm", - "runtimeprog/icu.dat", - ], - }) + fs::read(old_live.join("shell.vfs.zst")).unwrap(), + b"old-output-image/shell.vfs.zst\n" ); - let path = - ensure_built(&m, ®, TargetArch::Wasm32, 4, &resolve_opts(&cache, None)).unwrap(); - assert_eq!(fs::read(path.join("icu.dat")).unwrap(), b"runtime-data"); - - fs::remove_file(path.join("icu.dat")).unwrap(); - let err = validate_cache_artifacts(&m, &path).unwrap_err(); - assert!( - err.contains("runtime file") && err.contains("missing"), - "got: {err}" + assert_eq!( + fs::read_link(binaries.join("programs/wasm32/shell.zip")).unwrap(), + fs::canonicalize(&scalar_canonical) + .unwrap() + .join("shell.zip") ); } #[test] - fn runtime_file_metadata_lists_the_complete_multi_output_closure() { + fn scalar_publication_never_removes_a_package_named_user_directory() { + let root = tempdir("scalar-preserves-user-directory"); + let binaries = root.join("binaries"); + let arch_root = binaries.join("programs/wasm32"); + let user_directory = arch_root.join("scalar-package"); + fs::create_dir_all(&user_directory).unwrap(); + fs::write(user_directory.join("sentinel"), b"user-owned").unwrap(); + let manifest = DepsManifest::parse( r#"kind = "program" -name = "runtimeprog" +name = "scalar-package" version = "1.0" depends_on = [] [source] -url = "https://example.test/runtimeprog.tar.gz" +url = "https://example.test/scalar.tar.gz" sha256 = "0000000000000000000000000000000000000000000000000000000000000000" [license] spdx = "MIT" [[outputs]] -name = "runtimeprog" -wasm = "bin/runtimeprog.wasm" -[[outputs]] -name = "module" -wasm = "extensions/module.so" -[[runtime_files]] -artifact = "share/icu.dat" -guest_path = "/usr/lib/runtimeprog/icu.dat" -[[runtime_files]] -artifact = "share/timezone.dat" -guest_path = "/usr/lib/runtimeprog/timezone.dat" +name = "scalar" +wasm = "scalar.zip" "#, - PathBuf::from("/x"), + PathBuf::from("/scalar-package"), ) .unwrap(); + let canonical = root.join("cache/scalar"); + fs::create_dir_all(&canonical).unwrap(); + fs::write(canonical.join("scalar.zip"), b"scalar").unwrap(); - let metadata = runtime_file_metadata_value(&manifest, "share/icu.dat").unwrap(); + place_binaries_symlinks(&manifest, &canonical, &binaries, TEST_ARCH).unwrap(); assert_eq!( - metadata["closure_mirror_paths"], - serde_json::json!([ - "runtimeprog/runtimeprog.wasm", - "runtimeprog/module.so", - "runtimeprog/share/icu.dat", - "runtimeprog/share/timezone.dat", - ]) + fs::read(user_directory.join("sentinel")).unwrap(), + b"user-owned" + ); + assert_eq!( + fs::read_link(arch_root.join("scalar.zip")).unwrap(), + fs::canonicalize(&canonical).unwrap().join("scalar.zip") + ); + } + + #[cfg(unix)] + #[test] + fn fetched_publication_rejects_symlinked_root_programs_and_arch_ancestors() { + let manifest = atomic_mirror_manifest("share/runtime/nested/index.dat"); + for attacked_component in ["root", "programs", "arch"] { + let root = tempdir(&format!("atomic-mirror-symlink-{attacked_component}")); + let canonical = root.join("cache/identity"); + let outside = root.join("outside"); + fs::create_dir_all(&outside).unwrap(); + fs::write(outside.join("sentinel"), b"outside").unwrap(); + populate_atomic_mirror_identity(&canonical, &manifest, "new"); + + let real_binaries = root.join("binaries-real"); + let binaries = root.join("binaries"); + match attacked_component { + "root" => { + fs::create_dir_all(&real_binaries).unwrap(); + symlink_file(&real_binaries, &binaries).unwrap(); + } + "programs" => { + fs::create_dir_all(&binaries).unwrap(); + symlink_file(&outside, &binaries.join("programs")).unwrap(); + } + "arch" => { + fs::create_dir_all(binaries.join("programs")).unwrap(); + symlink_file(&outside, &binaries.join("programs/wasm32")).unwrap(); + } + _ => unreachable!(), + } + + let error = + place_binaries_symlinks(&manifest, &canonical, &binaries, TEST_ARCH).unwrap_err(); + assert!( + error.contains("real directory") || error.contains("file or symlink"), + "unexpected {attacked_component} rejection: {error}" + ); + assert_eq!(fs::read(outside.join("sentinel")).unwrap(), b"outside"); + assert!( + !outside.join("atomic-shell").exists(), + "publication escaped through {attacked_component}" + ); + } + } + + #[test] + fn multi_output_mirror_validates_nested_runtime_and_collisions_before_destination_mutation() { + let root = tempdir("atomic-mirror-preflight"); + let binaries = root.join("binaries"); + let arch_root = binaries.join("programs/wasm32"); + let old_canonical = root.join("cache/old-identity"); + let incomplete_canonical = root.join("cache/incomplete-identity"); + let manifest = atomic_mirror_manifest("share/runtime/nested/index.dat"); + populate_atomic_mirror_identity(&old_canonical, &manifest, "old"); + populate_atomic_mirror_identity(&incomplete_canonical, &manifest, "incomplete"); + fs::remove_file(incomplete_canonical.join("share/runtime/nested/index.dat")).unwrap(); + let old_plan = + PackageClosureMirrorPlan::validate(&manifest, &old_canonical, &arch_root).unwrap(); + let old_links = old_plan.expected_links(); + let live_dir = old_plan.package_dir.clone(); + write_atomic_mirror_fixture(&old_plan); + + let missing_error = + place_binaries_symlinks(&manifest, &incomplete_canonical, &binaries, TEST_ARCH) + .unwrap_err(); + assert!( + missing_error.contains("runtime file") + && missing_error.contains("share/runtime/nested/index.dat"), + "got: {missing_error}" + ); + assert_eq!(read_package_mirror_links(&live_dir).unwrap(), old_links); + assert_no_atomic_mirror_transaction_siblings(&live_dir); + + // Source manifests reject this collision at parse time. Mutate the + // already-validated fixture to prove the installer independently + // preserves the invariant before touching a preexisting live tree. + let mut collision_manifest = atomic_mirror_manifest("share/runtime/nested/index.dat"); + collision_manifest.runtime_files[0].artifact = "shell.vfs.zst".to_string(); + let collision_canonical = root.join("cache/collision-identity"); + populate_atomic_mirror_identity(&collision_canonical, &collision_manifest, "collision"); + let collision_error = place_binaries_symlinks( + &collision_manifest, + &collision_canonical, + &binaries, + TEST_ARCH, + ) + .unwrap_err(); + assert!( + collision_error.contains("collides"), + "got: {collision_error}" + ); + assert_eq!(read_package_mirror_links(&live_dir).unwrap(), old_links); + assert_no_atomic_mirror_transaction_siblings(&live_dir); + } + + #[test] + fn multi_output_mirror_first_rename_failure_preserves_old_and_cleans_stage() { + let root = tempdir("atomic-mirror-first-rename-failure"); + let arch_root = root.join("binaries/programs/wasm32"); + let old_canonical = root.join("cache/old-identity"); + let new_canonical = root.join("cache/new-identity"); + let manifest = atomic_mirror_manifest("share/runtime/nested/index.dat"); + populate_atomic_mirror_identity(&old_canonical, &manifest, "old"); + populate_atomic_mirror_identity(&new_canonical, &manifest, "new"); + let old_plan = + PackageClosureMirrorPlan::validate(&manifest, &old_canonical, &arch_root).unwrap(); + let new_plan = + PackageClosureMirrorPlan::validate(&manifest, &new_canonical, &arch_root).unwrap(); + let old_links = old_plan.expected_links(); + let live_dir = old_plan.package_dir.clone(); + write_atomic_mirror_fixture(&old_plan); + + let mut transaction = PackageDirectoryTransaction::prepare(new_plan).unwrap(); + let mut fail_first_rename = |_from: &Path, _to: &Path| { + Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "injected first rename failure", + )) + }; + let error = transaction + .move_existing_aside_with(&mut fail_first_rename) + .unwrap_err(); + assert!(error.contains("injected first rename failure")); + drop(transaction); + + assert_eq!(read_package_mirror_links(&live_dir).unwrap(), old_links); + assert_no_atomic_mirror_transaction_siblings(&live_dir); + } + + #[test] + fn multi_output_mirror_refuses_a_non_owned_live_directory_without_mutating_it() { + let root = tempdir("atomic-mirror-refuse-user-directory"); + let arch_root = root.join("binaries/programs/wasm32"); + let canonical = root.join("cache/new-identity"); + let manifest = atomic_mirror_manifest("share/runtime/nested/index.dat"); + populate_atomic_mirror_identity(&canonical, &manifest, "new"); + let plan = PackageClosureMirrorPlan::validate(&manifest, &canonical, &arch_root).unwrap(); + let live_dir = plan.package_dir.clone(); + fs::create_dir_all(&live_dir).unwrap(); + fs::write(live_dir.join("sentinel"), b"user-owned").unwrap(); + + let error = install_package_closure_mirror(plan).unwrap_err(); + assert!( + error.contains("without resolver ownership proof"), + "got: {error}" + ); + assert_eq!(fs::read(live_dir.join("sentinel")).unwrap(), b"user-owned"); + assert_no_atomic_mirror_transaction_siblings(&live_dir); + } + + #[cfg(unix)] + #[test] + fn multi_output_mirror_refuses_a_symlink_live_directory_without_following_it() { + let root = tempdir("atomic-mirror-refuse-live-symlink"); + let arch_root = root.join("binaries/programs/wasm32"); + let canonical = root.join("cache/new-identity"); + let outside = root.join("outside"); + let manifest = atomic_mirror_manifest("share/runtime/nested/index.dat"); + populate_atomic_mirror_identity(&canonical, &manifest, "new"); + fs::create_dir_all(&outside).unwrap(); + fs::write(outside.join("sentinel"), b"outside").unwrap(); + let plan = PackageClosureMirrorPlan::validate(&manifest, &canonical, &arch_root).unwrap(); + let live_dir = plan.package_dir.clone(); + fs::create_dir_all(&arch_root).unwrap(); + symlink_file(&outside, &live_dir).unwrap(); + + let error = install_package_closure_mirror(plan).unwrap_err(); + assert!( + error.contains("without resolver ownership proof"), + "got: {error}" + ); + assert_eq!(fs::read(outside.join("sentinel")).unwrap(), b"outside"); + assert!( + fs::symlink_metadata(&live_dir) + .unwrap() + .file_type() + .is_symlink() + ); + assert_no_atomic_mirror_transaction_siblings(&live_dir); + } + + #[test] + fn multi_output_mirror_detects_a_live_tree_swap_during_quarantine_and_restores_it() { + let root = tempdir("atomic-mirror-live-swap"); + let arch_root = root.join("binaries/programs/wasm32"); + let old_canonical = root.join("cache/old-identity"); + let new_canonical = root.join("cache/new-identity"); + let manifest = atomic_mirror_manifest("share/runtime/nested/index.dat"); + populate_atomic_mirror_identity(&old_canonical, &manifest, "old"); + populate_atomic_mirror_identity(&new_canonical, &manifest, "new"); + let old_plan = + PackageClosureMirrorPlan::validate(&manifest, &old_canonical, &arch_root).unwrap(); + let new_plan = + PackageClosureMirrorPlan::validate(&manifest, &new_canonical, &arch_root).unwrap(); + let live_dir = old_plan.package_dir.clone(); + write_atomic_mirror_fixture(&old_plan); + let displaced_old = root.join("displaced-old"); + let replacement = root.join("user-replacement"); + fs::create_dir(&replacement).unwrap(); + fs::write(replacement.join("sentinel"), b"user-owned").unwrap(); + + let mut transaction = PackageDirectoryTransaction::prepare(new_plan).unwrap(); + let mut first_rename = true; + let mut swap_before_rename = |from: &Path, to: &Path| { + if first_rename { + first_rename = false; + fs::rename(from, &displaced_old)?; + fs::rename(&replacement, from)?; + } + fs::rename(from, to) + }; + let error = transaction + .move_existing_aside_with(&mut swap_before_rename) + .unwrap_err(); + assert!( + error.contains("ownership changed during quarantine") && error.contains("restored"), + "got: {error}" + ); + drop(transaction); + + assert_eq!(fs::read(live_dir.join("sentinel")).unwrap(), b"user-owned"); + assert_eq!( + read_package_mirror_links(&displaced_old).unwrap(), + old_plan.expected_links() + ); + assert_no_atomic_mirror_transaction_siblings(&live_dir); + } + + #[test] + fn multi_output_mirror_second_rename_failure_rolls_back_complete_old_directory() { + let root = tempdir("atomic-mirror-second-rename-failure"); + let arch_root = root.join("binaries/programs/wasm32"); + let old_canonical = root.join("cache/old-identity"); + let new_canonical = root.join("cache/new-identity"); + let manifest = atomic_mirror_manifest("share/runtime/nested/index.dat"); + populate_atomic_mirror_identity(&old_canonical, &manifest, "old"); + populate_atomic_mirror_identity(&new_canonical, &manifest, "new"); + let old_plan = + PackageClosureMirrorPlan::validate(&manifest, &old_canonical, &arch_root).unwrap(); + let new_plan = + PackageClosureMirrorPlan::validate(&manifest, &new_canonical, &arch_root).unwrap(); + let old_links = old_plan.expected_links(); + let live_dir = old_plan.package_dir.clone(); + write_atomic_mirror_fixture(&old_plan); + + let mut transaction = PackageDirectoryTransaction::prepare(new_plan).unwrap(); + let mut rename = |from: &Path, to: &Path| fs::rename(from, to); + transaction.move_existing_aside_with(&mut rename).unwrap(); + let mut publish_rename_count = 0; + let mut fail_publish_then_rollback = |from: &Path, to: &Path| -> std::io::Result<()> { + publish_rename_count += 1; + if publish_rename_count == 1 { + Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "injected publish rename failure", + )) + } else { + fs::rename(from, to) + } + }; + let error = transaction + .publish_with(&mut fail_publish_then_rollback) + .unwrap_err(); + assert!(error.contains("restored the previous complete package directory")); + drop(transaction); + + assert_eq!(read_package_mirror_links(&live_dir).unwrap(), old_links); + assert_no_atomic_mirror_transaction_siblings(&live_dir); + } + + #[test] + fn multi_output_mirror_never_deletes_a_tampered_private_quarantine() { + let root = tempdir("atomic-mirror-tampered-quarantine"); + let arch_root = root.join("binaries/programs/wasm32"); + let old_canonical = root.join("cache/old-identity"); + let new_canonical = root.join("cache/new-identity"); + let manifest = atomic_mirror_manifest("share/runtime/nested/index.dat"); + populate_atomic_mirror_identity(&old_canonical, &manifest, "old"); + populate_atomic_mirror_identity(&new_canonical, &manifest, "new"); + let old_plan = + PackageClosureMirrorPlan::validate(&manifest, &old_canonical, &arch_root).unwrap(); + let new_plan = + PackageClosureMirrorPlan::validate(&manifest, &new_canonical, &arch_root).unwrap(); + let live_dir = old_plan.package_dir.clone(); + write_atomic_mirror_fixture(&old_plan); + + let mut transaction = PackageDirectoryTransaction::prepare(new_plan.clone()).unwrap(); + let transaction_root = transaction.transaction_root.clone(); + let backup_dir = transaction.backup_dir.clone(); + let mut rename = |from: &Path, to: &Path| fs::rename(from, to); + transaction.move_existing_aside_with(&mut rename).unwrap(); + fs::write(backup_dir.join("user-sentinel"), b"do-not-delete").unwrap(); + transaction.publish_with(&mut rename).unwrap(); + let error = transaction.finish().unwrap_err(); + assert!(error.contains("refusing to remove changed"), "got: {error}"); + + assert_eq!( + read_package_mirror_links(&live_dir).unwrap(), + new_plan.expected_links() ); + assert_eq!( + fs::read(backup_dir.join("user-sentinel")).unwrap(), + b"do-not-delete" + ); + assert!(transaction_root.is_dir()); + + // Test-only cleanup after proving production cleanup left the changed + // quarantine untouched. + remove_owned_transaction_path(&transaction_root).unwrap(); + assert_no_atomic_mirror_transaction_siblings(&live_dir); + } + + #[test] + fn multi_output_mirror_drop_retries_a_failed_explicit_rollback() { + let root = tempdir("atomic-mirror-drop-rollback"); + let arch_root = root.join("binaries/programs/wasm32"); + let old_canonical = root.join("cache/old-identity"); + let new_canonical = root.join("cache/new-identity"); + let manifest = atomic_mirror_manifest("share/runtime/nested/index.dat"); + populate_atomic_mirror_identity(&old_canonical, &manifest, "old"); + populate_atomic_mirror_identity(&new_canonical, &manifest, "new"); + let old_plan = + PackageClosureMirrorPlan::validate(&manifest, &old_canonical, &arch_root).unwrap(); + let new_plan = + PackageClosureMirrorPlan::validate(&manifest, &new_canonical, &arch_root).unwrap(); + let old_links = old_plan.expected_links(); + let live_dir = old_plan.package_dir.clone(); + write_atomic_mirror_fixture(&old_plan); + + let mut transaction = PackageDirectoryTransaction::prepare(new_plan).unwrap(); + let mut rename = |from: &Path, to: &Path| fs::rename(from, to); + transaction.move_existing_aside_with(&mut rename).unwrap(); + let mut fail_publish_and_rollback = |_from: &Path, _to: &Path| { + Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "injected publish and rollback failure", + )) + }; + let error = transaction + .publish_with(&mut fail_publish_and_rollback) + .unwrap_err(); + assert!(error.contains("rollback") && error.contains("also failed")); + assert!(!path_entry_exists(&live_dir).unwrap()); + + drop(transaction); + assert_eq!(read_package_mirror_links(&live_dir).unwrap(), old_links); + assert_no_atomic_mirror_transaction_siblings(&live_dir); + } + + #[test] + fn multi_output_mirror_crash_after_first_rename_leaves_absent_live_and_complete_siblings() { + let root = tempdir("atomic-mirror-interrupt-after-first"); + let arch_root = root.join("binaries/programs/wasm32"); + let old_canonical = root.join("cache/old-identity"); + let new_canonical = root.join("cache/new-identity"); + let manifest = atomic_mirror_manifest("share/runtime/nested/index.dat"); + populate_atomic_mirror_identity(&old_canonical, &manifest, "old"); + populate_atomic_mirror_identity(&new_canonical, &manifest, "new"); + let old_plan = + PackageClosureMirrorPlan::validate(&manifest, &old_canonical, &arch_root).unwrap(); + let new_plan = + PackageClosureMirrorPlan::validate(&manifest, &new_canonical, &arch_root).unwrap(); + let old_links = old_plan.expected_links(); + let new_links = new_plan.expected_links(); + let live_dir = old_plan.package_dir.clone(); + write_atomic_mirror_fixture(&old_plan); + + let mut transaction = PackageDirectoryTransaction::prepare(new_plan).unwrap(); + let transaction_root = transaction.transaction_root.clone(); + let stage_dir = transaction.stage_dir.clone(); + let backup_dir = transaction.backup_dir.clone(); + let mut rename = |from: &Path, to: &Path| fs::rename(from, to); + transaction.move_existing_aside_with(&mut rename).unwrap(); + std::mem::forget(transaction); + + assert!(!path_entry_exists(&live_dir).unwrap()); + assert_eq!(read_package_mirror_links(&backup_dir).unwrap(), old_links); + assert_eq!(read_package_mirror_links(&stage_dir).unwrap(), new_links); + + fs::rename(&backup_dir, &live_dir).unwrap(); + remove_owned_transaction_path(&stage_dir).unwrap(); + fs::remove_dir(&transaction_root).unwrap(); + assert_no_atomic_mirror_transaction_siblings(&live_dir); + } + + #[test] + fn multi_output_mirror_crash_after_second_rename_leaves_complete_new_live() { + let root = tempdir("atomic-mirror-interrupt-after-second"); + let arch_root = root.join("binaries/programs/wasm32"); + let old_canonical = root.join("cache/old-identity"); + let new_canonical = root.join("cache/new-identity"); + let manifest = atomic_mirror_manifest("share/runtime/nested/index.dat"); + populate_atomic_mirror_identity(&old_canonical, &manifest, "old"); + populate_atomic_mirror_identity(&new_canonical, &manifest, "new"); + let old_plan = + PackageClosureMirrorPlan::validate(&manifest, &old_canonical, &arch_root).unwrap(); + let new_plan = + PackageClosureMirrorPlan::validate(&manifest, &new_canonical, &arch_root).unwrap(); + let old_links = old_plan.expected_links(); + let new_links = new_plan.expected_links(); + let live_dir = old_plan.package_dir.clone(); + write_atomic_mirror_fixture(&old_plan); + + let mut transaction = PackageDirectoryTransaction::prepare(new_plan).unwrap(); + let transaction_root = transaction.transaction_root.clone(); + let stage_dir = transaction.stage_dir.clone(); + let backup_dir = transaction.backup_dir.clone(); + let mut rename = |from: &Path, to: &Path| fs::rename(from, to); + transaction.move_existing_aside_with(&mut rename).unwrap(); + transaction.publish_with(&mut rename).unwrap(); + std::mem::forget(transaction); + + assert_eq!(read_package_mirror_links(&live_dir).unwrap(), new_links); + assert!(!path_entry_exists(&stage_dir).unwrap()); + assert_eq!(read_package_mirror_links(&backup_dir).unwrap(), old_links); + + remove_owned_transaction_path(&backup_dir).unwrap(); + fs::remove_dir(&transaction_root).unwrap(); + assert_no_atomic_mirror_transaction_siblings(&live_dir); + } + + #[test] + fn multi_output_mirror_accepts_only_an_exact_complete_concurrent_winner() { + let root = tempdir("atomic-mirror-concurrent-winner"); + let arch_root = root.join("binaries/programs/wasm32"); + let old_canonical = root.join("cache/old-identity"); + let requested_canonical = root.join("cache/requested-identity"); + let different_canonical = root.join("cache/different-identity"); + let manifest = atomic_mirror_manifest("share/runtime/nested/index.dat"); + populate_atomic_mirror_identity(&old_canonical, &manifest, "old"); + populate_atomic_mirror_identity(&requested_canonical, &manifest, "requested"); + populate_atomic_mirror_identity(&different_canonical, &manifest, "different"); + let old_plan = + PackageClosureMirrorPlan::validate(&manifest, &old_canonical, &arch_root).unwrap(); + let requested_plan = + PackageClosureMirrorPlan::validate(&manifest, &requested_canonical, &arch_root) + .unwrap(); + let different_plan = + PackageClosureMirrorPlan::validate(&manifest, &different_canonical, &arch_root) + .unwrap(); + let live_dir = old_plan.package_dir.clone(); + write_atomic_mirror_fixture(&old_plan); + + let mut transaction = PackageDirectoryTransaction::prepare(requested_plan.clone()).unwrap(); + let mut rename = |from: &Path, to: &Path| fs::rename(from, to); + transaction.move_existing_aside_with(&mut rename).unwrap(); + write_atomic_mirror_fixture(&different_plan); + let error = transaction.publish_with(&mut rename).unwrap_err(); + assert!(error.contains("another writer installed a different")); + drop(transaction); + assert_eq!( + read_package_mirror_links(&live_dir).unwrap(), + different_plan.expected_links() + ); + assert_no_atomic_mirror_transaction_siblings(&live_dir); + + remove_owned_transaction_path(&live_dir).unwrap(); + write_atomic_mirror_fixture(&old_plan); + let mut extra_directory_transaction = + PackageDirectoryTransaction::prepare(requested_plan.clone()).unwrap(); + extra_directory_transaction + .move_existing_aside_with(&mut rename) + .unwrap(); + write_atomic_mirror_fixture(&requested_plan); + fs::create_dir(live_dir.join("unexpected-empty-directory")).unwrap(); + let error = extra_directory_transaction + .publish_with(&mut rename) + .unwrap_err(); + assert!(error.contains("another writer installed a different")); + drop(extra_directory_transaction); + assert!(live_dir.join("unexpected-empty-directory").is_dir()); + assert_no_atomic_mirror_transaction_siblings(&live_dir); + + remove_owned_transaction_path(&live_dir).unwrap(); + write_atomic_mirror_fixture(&old_plan); + let mut matching_transaction = + PackageDirectoryTransaction::prepare(requested_plan.clone()).unwrap(); + matching_transaction + .move_existing_aside_with(&mut rename) + .unwrap(); + write_atomic_mirror_fixture(&requested_plan); + matching_transaction.publish_with(&mut rename).unwrap(); + matching_transaction.finish().unwrap(); + assert_eq!( + read_package_mirror_links(&live_dir).unwrap(), + requested_plan.expected_links() + ); + assert_no_atomic_mirror_transaction_siblings(&live_dir); } #[test] @@ -8854,6 +15556,56 @@ fork_instrumentation = "disabled" assert_eq!(names, vec!["libL".to_string(), "progP".to_string()]); } + #[test] + fn walk_all_matches_normal_resolution_revision_and_overlay_loading() { + let root = tempdir("walk-all-loader-parity"); + write(&root, "libRev", "1.0.0", &[]); + write_build_revision(&root, "libRev", 7); + let reg = Registry { + roots: vec![root.clone()], + }; + + let normally_loaded = reg.load("libRev").unwrap(); + let (_, walked) = reg + .walk_all() + .unwrap() + .into_iter() + .find(|(name, _)| name == "libRev") + .unwrap(); + assert_eq!(walked.revision, 7); + assert_eq!( + package_context_cache_keys(&walked, ®).unwrap(), + package_context_cache_keys(&normally_loaded, ®).unwrap(), + "registry enumeration and dependency resolution must compute one cache identity", + ); + + fs::write( + root.join("libRev/package.pr.toml"), + r#" +[binary.wasm32] +archive_url = "https://example.test/pr/libRev.tar.zst" +archive_sha256 = "2222222222222222222222222222222222222222222222222222222222222222" +"#, + ) + .unwrap(); + let (_, walked_with_overlay) = reg + .walk_all() + .unwrap() + .into_iter() + .find(|(name, _)| name == "libRev") + .unwrap(); + assert_eq!(walked_with_overlay.revision, 7); + assert!( + walked_with_overlay.binary.contains_key(&TargetArch::Wasm32), + "walk_all must honor the same binary-only PR overlay as Registry::load", + ); + assert_eq!( + package_context_cache_keys(&walked_with_overlay, ®).unwrap(), + package_context_cache_keys(&normally_loaded, ®).unwrap(), + "binary fetch overlays must not change the canonical package identity", + ); + } + #[test] fn programs_by_name_filters_to_program_kind() { let root = tempdir("progs-by-name"); @@ -8922,6 +15674,25 @@ fork_instrumentation = "disabled" ); } + #[test] + fn walk_all_rejects_manifest_names_that_do_not_match_the_registry_directory() { + let root = tempdir("walk-name-directory-mismatch"); + write(&root, "directory-name", "1.0.0", &[]); + let manifest_path = root.join("directory-name/package.toml"); + let changed = fs::read_to_string(&manifest_path) + .unwrap() + .replace("name = \"directory-name\"", "name = \"different-name\""); + fs::write(&manifest_path, changed).unwrap(); + let reg = Registry { roots: vec![root] }; + + let error = reg.walk_all().unwrap_err(); + assert!( + error.contains("package name \"different-name\"") + && error.contains("registry directory \"directory-name\""), + "got: {error}", + ); + } + #[test] fn source_kind_sha_omits_arch_and_abi_inputs() { let dir = tempdir("c3a"); @@ -9935,7 +16706,15 @@ printf canonical-runtime > "$WASM_POSIX_DEP_OUT_DIR/icu.dat""#, let runtime = bin_dir.join("programs/wasm32/runtimebin/icu.dat"); assert!(runtime.symlink_metadata().unwrap().file_type().is_symlink()); assert_eq!(fs::read(runtime).unwrap(), b"canonical-runtime"); - assert!(bin_dir.join("programs/wasm32/runtimebin.wasm").exists()); + let executable = bin_dir.join("programs/wasm32/runtimebin/runtimebin.wasm"); + assert!( + executable + .symlink_metadata() + .unwrap() + .file_type() + .is_symlink() + ); + assert_eq!(fs::read(executable).unwrap(), minimal_executable_wasm()); } #[test] diff --git a/tools/xtask/src/pkg_manifest.rs b/tools/xtask/src/pkg_manifest.rs index ade4218a70..979b124336 100644 --- a/tools/xtask/src/pkg_manifest.rs +++ b/tools/xtask/src/pkg_manifest.rs @@ -1180,7 +1180,7 @@ fn validate_single_path_component(value: &str, context: &str) -> Result<(), Stri /// Both paths describe files. Equal paths and ancestor/descendant pairs are /// therefore unsatisfiable in one artifact or VFS closure. -fn file_paths_conflict(a: &str, b: &str) -> bool { +pub(crate) fn file_paths_conflict(a: &str, b: &str) -> bool { a == b || a.strip_prefix(b) .is_some_and(|suffix| suffix.starts_with('/')) @@ -1349,7 +1349,7 @@ fn default_outputs_value() -> toml::Value { fn program_output_dest_rel( package_name: &str, - output_count: usize, + closure_member_count: usize, out: &ProgramOutput, ) -> PathBuf { let basename = Path::new(&out.wasm) @@ -1361,7 +1361,7 @@ fn program_output_dest_rel( None => "", }; let dest_name = format!("{}{}", out.name, ext); - if output_count > 1 { + if closure_member_count > 1 { Path::new(package_name).join(dest_name) } else { PathBuf::from(dest_name) @@ -1369,6 +1369,32 @@ fn program_output_dest_rel( } impl DepsManifest { + /// Number of files that form this program package's resolver closure. + /// + /// Executable outputs and non-Wasm runtime files are one package identity: + /// consumers must never mix either class across builds. + pub fn program_closure_member_count(&self) -> usize { + self.program_outputs.len() + self.runtime_files.len() + } + + /// Whether the resolver mirror must be owned by one package directory. + pub fn uses_package_mirror_directory(&self) -> bool { + self.program_closure_member_count() > 1 + } + + /// Whether this package publishes its sole executable at the binary root + /// instead of under `programs//`. + /// + /// The kernel and userspace adapter are host boot artifacts, not guest + /// programs. Keep this predicate shared by publication and projection so + /// the program-package index never claims a path the publisher cannot + /// create. + pub fn uses_root_binary_mirror(&self) -> bool { + matches!(self.name.as_str(), "kernel" | "userspace") + && self.program_outputs.len() == 1 + && self.runtime_files.is_empty() + } + /// Resolver mirror path under `programs//` for a runtime file. /// Runtime files always live below the package name, independently of the /// number of executable `[[outputs]]` entries. @@ -1413,21 +1439,20 @@ impl DepsManifest { /// the consumer's expected path. /// /// Layout: - /// * 1 output: `` - /// * ≥2 outputs: `/` + /// * 1 total output/runtime member: `` + /// * ≥2 total members: `/` /// /// `` is everything from the first `.` onward in `out.wasm`'s /// basename, so `.vfs.zst`, `.tar.gz`, etc. round-trip intact. pub fn output_dest_rel_for(&self, out: &ProgramOutput) -> PathBuf { - program_output_dest_rel(&self.name, self.program_outputs.len(), out) + program_output_dest_rel(&self.name, self.program_closure_member_count(), out) } - /// Same as [`output_dest_rel_for`] but keyed by the `wasm` - /// filename instead of the output struct — used by build scripts - /// (via `xtask build-deps output-path`) which have only the file - /// they just built, not the parsed package.toml struct. - pub fn output_dest_rel(&self, wasm_basename: &str) -> Result { - let out = self.output_for_wasm_basename(wasm_basename)?; + /// Same as [`output_dest_rel_for`] but keyed by the declared `wasm` + /// artifact path. A unique basename remains accepted for existing build + /// scripts, but an exact declaration wins and ambiguous basenames fail. + pub fn output_dest_rel(&self, wasm_artifact: &str) -> Result { + let out = self.output_for_wasm_artifact(wasm_artifact)?; Ok(self.output_dest_rel_for(out)) } @@ -1435,14 +1460,14 @@ impl DepsManifest { /// output, keyed by its `wasm` basename. pub fn output_fork_instrumentation( &self, - wasm_basename: &str, + wasm_artifact: &str, ) -> Result { Ok(self - .output_for_wasm_basename(wasm_basename)? + .output_for_wasm_artifact(wasm_artifact)? .fork_instrumentation) } - fn output_for_wasm_basename(&self, wasm_basename: &str) -> Result<&ProgramOutput, String> { + pub fn output_for_wasm_artifact(&self, wasm_artifact: &str) -> Result<&ProgramOutput, String> { if self.kind != ManifestKind::Program { return Err(format!( "manifest {:?} is kind={:?}; program output lookup is program-only", @@ -1453,20 +1478,32 @@ impl DepsManifest { if outputs.is_empty() { return Err(format!("program {:?} has no [[outputs]]", self.name)); } - let out = outputs + if let Some(output) = outputs.iter().find(|output| output.wasm == wasm_artifact) { + return Ok(output); + } + let matches: Vec<_> = outputs .iter() - .find(|o| { - Path::new(&o.wasm).file_name().and_then(|s| s.to_str()) == Some(wasm_basename) + .filter(|output| { + Path::new(&output.wasm) + .file_name() + .and_then(|value| value.to_str()) + == Some(wasm_artifact) }) - .ok_or_else(|| { + .collect(); + match matches.as_slice() { + [output] => Ok(*output), + [] => { let declared: Vec<&str> = outputs.iter().map(|o| o.wasm.as_str()).collect(); - format!( - "program {:?} has no [[outputs]] entry whose wasm = {:?} \ - (declared: {:?})", - self.name, wasm_basename, declared - ) - })?; - Ok(out) + Err(format!( + "program {:?} has no [[outputs]] entry whose wasm path or unique basename is {:?} (declared: {:?})", + self.name, wasm_artifact, declared + )) + } + _ => Err(format!( + "program {:?} has multiple [[outputs]] entries with basename {:?}; pass the exact declared wasm path", + self.name, wasm_artifact + )), + } } /// Read + parse + validate a `package.toml` file. `dir` is the @@ -2067,7 +2104,11 @@ impl DepsManifest { )); } } - let mirror = program_output_dest_rel(&raw.name, program_outputs.len(), out); + let mirror = program_output_dest_rel( + &raw.name, + program_outputs.len() + runtime_files.len(), + out, + ); for prior in &output_mirrors { if file_paths_conflict(&prior.to_string_lossy(), &mirror.to_string_lossy()) { return Err(format!( @@ -3289,11 +3330,11 @@ wasm = "vim.wasm" m.runtime_file_dest_rel("icu.dat").unwrap(), PathBuf::from("vim/icu.dat") ); - // Runtime-file placement must not perturb the legacy single-output - // destination convention. + // The executable and runtime file are one closure, so both live below + // the package-owned mirror directory. assert_eq!( m.output_dest_rel("vim.wasm").unwrap(), - PathBuf::from("vim.wasm") + PathBuf::from("vim/vim.wasm") ); let nested = program_with_runtime_file("share/icu/icu.dat", "/usr/lib/php/icu.dat", None); @@ -3831,6 +3872,35 @@ spdx = "TestLicense" ); } + #[test] + fn root_binary_mirror_is_limited_to_single_output_boot_artifacts() { + for name in ["kernel", "userspace"] { + let manifest = program_manifest( + name, + &format!("[[outputs]]\nname = \"{name}\"\nwasm = \"{name}.wasm\"\n"), + ); + assert!(manifest.uses_root_binary_mirror()); + } + + let ordinary = program_manifest( + "shell", + "[[outputs]]\nname = \"shell\"\nwasm = \"shell.wasm\"\n", + ); + assert!(!ordinary.uses_root_binary_mirror()); + + let multi = program_manifest( + "kernel", + r#"[[outputs]] +name = "kernel" +wasm = "kernel.wasm" +[[runtime_files]] +artifact = "kernel-data" +guest_path = "/usr/share/kernel-data" +"#, + ); + assert!(!multi.uses_root_binary_mirror()); + } + #[test] fn output_fork_instrumentation_defaults_to_auto() { let m = program_manifest( @@ -3874,6 +3944,31 @@ spdx = "TestLicense" ); } + #[test] + fn output_dest_rel_single_output_with_runtime_file_uses_program_subdir() { + let m = program_manifest( + "cpython", + r#"[[outputs]] +name = "cpython" +wasm = "python.wasm" + +[[runtime_files]] +artifact = "python-runtime.zip" +guest_path = "/usr/share/cpython/python-runtime.zip" +"#, + ); + assert_eq!(m.program_closure_member_count(), 2); + assert!(m.uses_package_mirror_directory()); + assert_eq!( + m.output_dest_rel("python.wasm").unwrap(), + PathBuf::from("cpython/cpython.wasm") + ); + assert_eq!( + m.runtime_file_dest_rel("python-runtime.zip").unwrap(), + PathBuf::from("cpython/python-runtime.zip") + ); + } + #[test] fn output_dest_rel_multi_output_uses_program_subdir() { // ≥2 outputs: the resolver nests under /, so @@ -3901,6 +3996,33 @@ wasm = "git-remote-http.wasm" ); } + #[test] + fn exact_output_artifact_disambiguates_duplicate_basenames() { + let m = program_manifest( + "tools", + r#" +[[outputs]] +name = "first" +wasm = "a/tool.wasm" + +[[outputs]] +name = "second" +wasm = "b/tool.wasm" +"#, + ); + assert_eq!( + m.output_dest_rel("a/tool.wasm").unwrap(), + PathBuf::from("tools/first.wasm") + ); + assert_eq!( + m.output_dest_rel("b/tool.wasm").unwrap(), + PathBuf::from("tools/second.wasm") + ); + let error = m.output_dest_rel("tool.wasm").unwrap_err(); + assert!(error.contains("multiple"), "got: {error}"); + assert!(error.contains("exact declared wasm path"), "got: {error}"); + } + #[test] fn output_dest_rel_unknown_basename_errors() { // Caller passed a wasm filename not declared in [[outputs]].