🤖 tests: a digest-pinned bug-bash sandbox image, published manually from main

.github/workflows/bugbash-sandbox-image.yml

@@ -0,0 +1,88 @@
+# The bug-bash sandbox image (#5714): tests/bugbash/sandbox/Dockerfile, built by build.sh.
+#
+# Pull requests build it with a read-only token and push nothing. Publishing is manual: only a
+# workflow_dispatch run on main publishes, from that run's exact commit, with no cache and no
+# artifact from any other run. Each publish builds fresh, and the attestation names that build's
+# own output. A person then copies the digest from the job summary into image.json in a pull
+# request: no workflow writes to the repo. Nothing here deletes an image.
+#
+# Until a new image is published and its digest PR merges, runners refuse a checkout whose image
+# inputs changed (stale inputs key).
+name: Bug-bash sandbox image
+
+on:
+  pull_request:
+    paths:
+      - tests/bugbash/sandbox/Dockerfile
+      - tests/bugbash/sandbox/build.sh
+      - bun.lock
+      - Makefile
+      - .github/workflows/bugbash-sandbox-image.yml
+  workflow_dispatch: {}
+
+permissions: {}
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  # Never cancel a publish half way. Pull request runs may cancel.
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  build:
+    name: Build (no push)
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        with:
+          persist-credentials: false
+      - name: Build
+        run: |
+          id=$(make -s bugbash-sandbox-image)
+          size=$(docker image inspect --format '{{.Size}}' "$id")
+          echo "Image $id: $size bytes (inputs key $(make -s bugbash-sandbox-key))" | tee -a "$GITHUB_STEP_SUMMARY"
+
+  publish:
+    name: Publish
+    if: github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      # Push ghcr.io/coder/xum-bugbash-sandbox. Only this job, only a manual run on main.
+      packages: write
+      # Build provenance for audit (the runner trusts the digest in image.json).
+      id-token: write
+      attestations: write
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        with:
+          ref: ${{ github.sha }}
+          persist-credentials: false
+      - name: Log in to GHCR
+        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and push
+        id: push
+        run: |
+          make -s bugbash-sandbox-publish > image.json
+          # The subject of the attestation is this build's own output (image.json).
+          echo "image=$(jq -er .image image.json)" >> "$GITHUB_OUTPUT"
+          echo "digest=$(jq -er .digest image.json)" >> "$GITHUB_OUTPUT"
+          {
+            echo '### tests/bugbash/sandbox/image.json'
+            echo
+            echo '```json'
+            cat image.json
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"
+      - name: Attest
+        uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2
+        with:
+          subject-name: ${{ steps.push.outputs.image }}
+          subject-digest: ${{ steps.push.outputs.digest }}
+          push-to-registry: true

Makefile

@@ -91,7 +91,7 @@ include fmt.mk
 .PHONY: all build dev start clean help
 .PHONY: build-renderer version build-icons build-static build-docker-runtime verify-docker-runtime-artifacts
 .PHONY: lint lint-fix typecheck static-check static-check-full
-.PHONY: test-bugbash-repros test-bugbash-known-failures
+.PHONY: test-bugbash-repros test-bugbash-known-failures bugbash-sandbox-key bugbash-sandbox-image bugbash-sandbox-publish
 .PHONY: test test-unit test-unit-ci test-integration test-watch test-coverage test-e2e test-e2e-perf perf-tape-replay smoke-test
 .PHONY: dist dist-mac dist-win dist-linux install-mac-arm64 ensure-mac-sharp-runtime-deps check-appimage-icons check-mac-attach-file-runtime
 .PHONY: vscode-ext vscode-ext-install
@@ -224,6 +224,17 @@ dev-server-sandbox: ## Start an isolated dev-server instance (fresh XUM_ROOT + f
 bug-bash: ## Agent bug bash: e2e explore charters x models (paused on the host until the sandbox lands, #5714)
 	@bun tests/bugbash/run.ts $(BUGBASH_ARGS)
 
+# The bug-bash sandbox image (tests/bugbash/sandbox/build.sh, #5714). Each prints one value on
+# stdout. Publishing runs only in the manual "Bug-bash sandbox image" workflow on main.
+bugbash-sandbox-key: ## Print the inputs key of the bug-bash sandbox image for this checkout
+	@./tests/bugbash/sandbox/build.sh --key
+
+bugbash-sandbox-image: ## Build the bug-bash sandbox image locally (linux/amd64) and print its image ID
+	@./tests/bugbash/sandbox/build.sh
+
+bugbash-sandbox-publish: ## Workflow only: build and push the bug-bash sandbox image, print its image.json record
+	@./tests/bugbash/sandbox/build.sh --push
+
 # Bug-bash repro tests (tests/bugbash/repros/*.e2e.ts): exact UI steps, no model calls, against a
 # seeded `xum server` (tests/bugbash/startApp.ts). e2e needs Node.js 22.22.3+ or 24.8+ (E2E_NODE)
 # and its own Chromium: `node_modules/.bin/e2e-web install chromium`.

tests/bugbash/sandbox/Dockerfile

@@ -0,0 +1,30 @@
+# The bug-bash sandbox image (#5714): tools only. The app (dist), node_modules and the tests come
+# from the checkout at run time, read-only. Build it with build.sh only: it passes the build args,
+# an empty build context, and no cache.
+#
+# The image is published by the main branch only (.github/workflows/bugbash-sandbox-image.yml),
+# and runners pull it by the digest in image.json. The base images are pinned by digest. The
+# Debian apt mirror and the Playwright download CDN are not pinned: the build trusts them, and
+# the published digest fixes the bytes after the build.
+FROM oven/bun:1.3.12-slim@sha256:d3c7094c144dd3975d183a4dbc4ec0a764223995bff73290d983edb47043a75f AS bun
+
+FROM node:22-slim@sha256:c3de60bf2f9dd0ac6370e6117950ff62d6e339527e7472301c9c78a017978392
+# The playwright-core version of the checkout's @e2e-dev/web (build.sh reads it from bun.lock).
+ARG PLAYWRIGHT_CORE_VERSION
+# The inputs key (build.sh --key). The runner refuses an image whose label differs from the
+# key of its checkout.
+ARG INPUTS_SHA256
+LABEL org.xum.bugbash.inputs=$INPUTS_SHA256 \
+      org.xum.bugbash.playwright-core=$PLAYWRIGHT_CORE_VERSION \
+      org.opencontainers.image.source=https://github.com/coder/xum
+ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright NPM_CONFIG_UPDATE_NOTIFIER=false
+# git: startApp.ts seeds a git repo for the demo project.
+# hadolint ignore=DL3008
+RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates \
+ && rm -rf /var/lib/apt/lists/*
+# The Chromium build that this playwright-core drives. e2e looks for both the full browser and
+# the headless shell, and `install chromium` gets both.
+RUN test -n "$PLAYWRIGHT_CORE_VERSION" && test -n "$INPUTS_SHA256" \
+ && npx -y "playwright-core@${PLAYWRIGHT_CORE_VERSION}" install --with-deps chromium \
+ && rm -rf /var/lib/apt/lists/* /root/.npm /root/.cache && chmod -R a+rX /ms-playwright
+COPY --from=bun /usr/local/bin/bun /usr/local/bin/bun

tests/bugbash/sandbox/build.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+# Builds the bug-bash sandbox image (#5714, Dockerfile next to this file).
+#
+#   build.sh --key    print the inputs key of this checkout
+#   build.sh          build for linux/amd64 into the local Docker, print the image ID
+#   build.sh --push   the manual publish job only: build, push to GHCR, print the image.json record
+#
+# The inputs key is the sha256 of three lines: the sha256 of the Dockerfile, the sha256 of this
+# script, and the playwright-core version of @e2e-dev/web in bun.lock. The image carries it as
+# the label org.xum.bugbash.inputs, and the runner computes the same key for its checkout.
+# The script reads these files from the working tree and refuses when they differ from HEAD, so
+# the key always names committed source.
+#
+# The build gets an empty context, no cache and no build args from the env. --push always builds
+# fresh: it never checks the registry and never reuses an image. It pushes a new tag, so it
+# overwrites no tag, and it reports the name and digest from its own build output. Two publishes
+# of one inputs key give two digests; the reviewed digest in image.json is what runners trust.
+# Nothing here deletes an image.
+set -euo pipefail
+
+IMAGE=ghcr.io/coder/xum-bugbash-sandbox
+HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+ROOT=$(git -C "$HERE" rev-parse --show-toplevel)
+DOCKERFILE=tests/bugbash/sandbox/Dockerfile
+SCRIPT=tests/bugbash/sandbox/build.sh
+
+die() {
+  echo "build.sh: $*" >&2
+  exit 2
+}
+# sha256 of stdin. Stock macOS has shasum, not sha256sum.
+sha_stdin() {
+  if command -v sha256sum >/dev/null; then sha256sum; else shasum -a 256; fi | cut -d' ' -f1
+}
+sha() { sha_stdin <"$1"; }
+
+cd "$ROOT"
+for f in "$DOCKERFILE" "$SCRIPT" bun.lock; do
+  git cat-file -e "HEAD:$f" 2>/dev/null || die "$f is not committed: the image builds from committed source"
+done
+git diff --quiet HEAD -- "$DOCKERFILE" "$SCRIPT" bun.lock ||
+  die "$DOCKERFILE, $SCRIPT or bun.lock differs from HEAD: the image builds from committed source"
+
+# The copy that e2e drives Chromium with. The root playwright-core is a different copy.
+# No mapfile: the system bash of macOS is 3.2.
+versions=$(grep -oE '"@e2e-dev/web/playwright-core": \["playwright-core@[^"]+"' bun.lock |
+  sed -E 's/.*playwright-core@//; s/"$//') || true
+count=$(printf '%s' "$versions" | grep -c . || true)
+[ "$count" -eq 1 ] || die "expected one @e2e-dev/web/playwright-core entry in bun.lock, found $count"
+PLAYWRIGHT=$versions
+[[ $PLAYWRIGHT =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || die "unexpected playwright-core version: $PLAYWRIGHT"
+
+KEY=$(printf 'dockerfile %s\nbuild.sh %s\nplaywright-core %s\n' "$(sha "$DOCKERFILE")" "$(sha "$SCRIPT")" "$PLAYWRIGHT" |
+  sha_stdin)
+[[ $KEY =~ ^[0-9a-f]{64}$ ]] || die "bad inputs key: $KEY"
+
+mode=${1:-local}
+[ $# -le 1 ] || die "one argument at most: --key or --push"
+case "$mode" in
+  --key)
+    echo "$KEY"
+    exit 0
+    ;;
+  local | --push) ;;
+  *) die "unknown argument: $mode" ;;
+esac
+
+context=$(mktemp -d)
+trap 'rm -rf "$context" "$context.json"' EXIT
+build() {
+  docker buildx build --platform linux/amd64 --no-cache --pull --provenance=false --sbom=false \
+    --build-arg "PLAYWRIGHT_CORE_VERSION=$PLAYWRIGHT" --build-arg "INPUTS_SHA256=$KEY" \
+    --file "$ROOT/$DOCKERFILE" --metadata-file "$context.json" "$@" "$context" >&2
+}
+
+if [ "$mode" = local ]; then
+  build --load --tag "xum-bugbash-sandbox:inputs-$KEY"
+  docker image inspect --format '{{.Id}}' "xum-bugbash-sandbox:inputs-$KEY"
+  exit 0
+fi
+
+# --push. One new tag per publish (inputs key, commit, UTC time), so a publish overwrites no tag.
+build --push --tag "$IMAGE:inputs-${KEY:0:16}-$(git rev-parse --short=12 HEAD)-$(date -u +%Y%m%d%H%M%S)"
+# The name and digest come from this build's own metadata, not from a registry lookup.
+digest=$(jq -r '."containerimage.digest"' "$context.json")
+pushed=$(jq -r '."image.name"' "$context.json")
+[[ $digest =~ ^sha256:[0-9a-f]{64}$ ]] || die "bad digest in the build metadata: $digest"
+[[ $pushed == "$IMAGE:"* && $pushed != *,* ]] || die "unexpected image name in the build metadata: $pushed"
+jq -n --arg image "${pushed%:*}" --arg digest "$digest" --arg key "$KEY" --arg pw "$PLAYWRIGHT" \
+  --arg from "$(git rev-parse HEAD)" \
+  '{image: $image, digest: $digest, inputsKey: $key, playwrightCore: $pw, publishedFrom: $from}'

tests/bugbash/sandbox/build.test.ts

@@ -0,0 +1,115 @@
+import { afterEach, expect, test } from "bun:test";
+import { spawnSync } from "node:child_process";
+import * as fs from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
+
+// build.sh --key names committed source only, and the key follows each of its three inputs.
+// Each test runs the real script in a throwaway git repo with copies of its inputs.
+const SANDBOX = "tests/bugbash/sandbox";
+const lock = (version: string) =>
+  `{\n  "packages": {\n    "playwright-core": ["playwright-core@1.57.0", ""],\n` +
+  `    "@e2e-dev/web/playwright-core": ["playwright-core@${version}", ""],\n  }\n}\n`;
+
+let repo = "";
+afterEach(() => fs.rmSync(repo, { recursive: true, force: true }));
+
+function git(...args: string[]) {
+  const r = spawnSync("git", args, { cwd: repo, encoding: "utf8" });
+  expect(r.status).toBe(0);
+}
+function setup(version = "1.63.0") {
+  repo = fs.mkdtempSync(path.join(os.tmpdir(), "xbb-build-key-"));
+  fs.mkdirSync(path.join(repo, SANDBOX), { recursive: true });
+  for (const name of ["build.sh", "Dockerfile"])
+    fs.copyFileSync(path.join(import.meta.dir, name), path.join(repo, SANDBOX, name));
+  fs.writeFileSync(path.join(repo, "bun.lock"), lock(version));
+  git("init", "-q");
+  commit();
+}
+function commit() {
+  git("add", "-A");
+  git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "-qm", "x");
+}
+function key() {
+  const r = spawnSync(path.join(repo, SANDBOX, "build.sh"), ["--key"], { encoding: "utf8" });
+  return { status: r.status, out: r.stdout.trim(), err: r.stderr };
+}
+
+test("the key changes with each input and is stable otherwise", () => {
+  setup();
+  const first = key();
+  expect(first.status).toBe(0);
+  expect(first.out).toMatch(/^[0-9a-f]{64}$/);
+  expect(key().out).toBe(first.out);
+
+  const keys = new Set([first.out]);
+  fs.writeFileSync(path.join(repo, "bun.lock"), lock("1.64.0"));
+  commit();
+  keys.add(key().out);
+  fs.appendFileSync(path.join(repo, SANDBOX, "Dockerfile"), "# change\n");
+  commit();
+  keys.add(key().out);
+  fs.appendFileSync(path.join(repo, SANDBOX, "build.sh"), "# change\n");
+  commit();
+  keys.add(key().out);
+  expect(keys.size).toBe(4);
+});
+
+test("an uncommitted input refuses", () => {
+  setup();
+  fs.appendFileSync(path.join(repo, SANDBOX, "Dockerfile"), "# local edit\n");
+  expect(key()).toMatchObject({ status: 2, out: "" });
+  expect(key().err).toContain("differs from HEAD");
+});
+
+test("the version comes only from the @e2e-dev/web copy, and must be unique", () => {
+  setup();
+  // The root playwright-core entry (1.57.0) does not count.
+  fs.writeFileSync(path.join(repo, "bun.lock"), lock("1.63.0").replace("@e2e-dev/web/", "x/"));
+  commit();
+  expect(key()).toMatchObject({ status: 2, out: "" });
+  expect(key().err).toContain("found 0");
+});
+
+test("--push always builds fresh and reports the name and digest of its own build", () => {
+  setup();
+  // A fake docker: `buildx build` writes build metadata to --metadata-file, and every call is
+  // logged. A registry lookup (imagetools) would be a reuse path.
+  const bin = path.join(repo, "bin");
+  fs.mkdirSync(bin);
+  const calls = path.join(bin, "calls.log");
+  const digest = `sha256:${"ab".repeat(32)}`;
+  fs.writeFileSync(
+    path.join(bin, "docker"),
+    `#!/bin/sh
+echo "$*" >> "${calls}"
+[ "$1 $2" = "buildx build" ] || exit 9
+tag=""; meta=""
+while [ $# -gt 0 ]; do
+  case "$1" in --tag) tag=$2 ;; --metadata-file) meta=$2 ;; esac
+  shift
+done
+printf '{"containerimage.digest":"${digest}","image.name":"%s"}' "$tag" > "$meta"
+`,
+    { mode: 0o755 }
+  );
+  const push = () =>
+    spawnSync(path.join(repo, SANDBOX, "build.sh"), ["--push"], {
+      encoding: "utf8",
+      env: { ...process.env, PATH: `${bin}:${process.env.PATH ?? ""}` },
+    });
+  const r = push();
+  expect(r.status).toBe(0);
+  expect(JSON.parse(r.stdout)).toMatchObject({
+    image: "ghcr.io/coder/xum-bugbash-sandbox",
+    digest,
+    inputsKey: key().out,
+    playwrightCore: "1.63.0",
+  });
+  // A second publish of the same inputs builds again. Every call was a push build.
+  expect(push().status).toBe(0);
+  const log = fs.readFileSync(calls, "utf8").trim().split("\n");
+  expect(log).toHaveLength(2);
+  for (const line of log) expect(line).toMatch(/^buildx build .*--no-cache .*--push /);
+});