From 94ef93353addda26cd8ff058d0af779bcdc51d32 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 7 Oct 2026 16:40:17 +0000 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=A4=96=20tests:=20publish=20a=20diges?= =?UTF-8?q?t-pinned=20bug-bash=20sandbox=20image=20from=20main=20(#5714)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/bugbash-sandbox-image.yml | 88 +++++++++++++++++++++ tests/bugbash/sandbox/Dockerfile | 30 +++++++ tests/bugbash/sandbox/build.sh | 85 ++++++++++++++++++++ tests/bugbash/sandbox/build.test.ts | 73 +++++++++++++++++ 4 files changed, 276 insertions(+) create mode 100644 .github/workflows/bugbash-sandbox-image.yml create mode 100644 tests/bugbash/sandbox/Dockerfile create mode 100755 tests/bugbash/sandbox/build.sh create mode 100644 tests/bugbash/sandbox/build.test.ts diff --git a/.github/workflows/bugbash-sandbox-image.yml b/.github/workflows/bugbash-sandbox-image.yml new file mode 100644 index 00000000000..ca67e6f45d7 --- /dev/null +++ b/.github/workflows/bugbash-sandbox-image.yml @@ -0,0 +1,88 @@ +# The bug-bash sandbox image (#5714): tests/bugbash/sandbox/Dockerfile, built by build.sh. +# +# Pull requests build it with a read-only token and push nothing. Only a push to main (or a +# manual run on main) publishes, from that exact commit, with no cache and no artifact from any +# other run. A person then copies the digest from the job summary into image.json in a pull +# request: no workflow writes to the repo. Nothing here deletes an image. +name: Bug-bash sandbox image + +on: + push: + branches: [main] + paths: + - tests/bugbash/sandbox/Dockerfile + - tests/bugbash/sandbox/build.sh + - bun.lock + - .github/workflows/bugbash-sandbox-image.yml + pull_request: + paths: + - tests/bugbash/sandbox/Dockerfile + - tests/bugbash/sandbox/build.sh + - bun.lock + - .github/workflows/bugbash-sandbox-image.yml + workflow_dispatch: {} + +permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + # Never cancel a publish half way. Pull request runs may cancel. + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + build: + name: Build (no push) + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + - name: Build + run: | + id=$(./tests/bugbash/sandbox/build.sh) + size=$(docker image inspect --format '{{.Size}}' "$id") + echo "Image $id: $size bytes (inputs key $(./tests/bugbash/sandbox/build.sh --key))" | tee -a "$GITHUB_STEP_SUMMARY" + + publish: + name: Publish + if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + permissions: + contents: read + # Push ghcr.io/coder/xum-bugbash-sandbox. Only this job, only on main. + packages: write + # Build provenance for audit (the runner trusts the digest in image.json). + id-token: write + attestations: write + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Log in to GHCR + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push + id: push + run: | + ./tests/bugbash/sandbox/build.sh --push > image.json + echo "digest=$(jq -r .digest image.json)" >> "$GITHUB_OUTPUT" + { + echo '### tests/bugbash/sandbox/image.json' + echo + echo '```json' + cat image.json + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + - name: Attest + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-name: ghcr.io/coder/xum-bugbash-sandbox + subject-digest: ${{ steps.push.outputs.digest }} + push-to-registry: true diff --git a/tests/bugbash/sandbox/Dockerfile b/tests/bugbash/sandbox/Dockerfile new file mode 100644 index 00000000000..d9bbdb198b8 --- /dev/null +++ b/tests/bugbash/sandbox/Dockerfile @@ -0,0 +1,30 @@ +# The bug-bash sandbox image (#5714): tools only. The app (dist), node_modules and the tests come +# from the checkout at run time, read-only. Build it with build.sh only: it passes the build args, +# an empty build context, and no cache. +# +# The image is published by the main branch only (.github/workflows/bugbash-sandbox-image.yml), +# and runners pull it by the digest in image.json. The base images are pinned by digest. The +# Debian apt mirror and the Playwright download CDN are not pinned: the build trusts them, and +# the published digest fixes the bytes after the build. +FROM oven/bun:1.3.12-slim@sha256:d3c7094c144dd3975d183a4dbc4ec0a764223995bff73290d983edb47043a75f AS bun + +FROM node:22-slim@sha256:c3de60bf2f9dd0ac6370e6117950ff62d6e339527e7472301c9c78a017978392 +# The playwright-core version of the checkout's @e2e-dev/web (build.sh reads it from bun.lock). +ARG PLAYWRIGHT_CORE_VERSION +# The inputs key (build.sh --key). The runner refuses an image whose label differs from the +# key of its checkout. +ARG INPUTS_SHA256 +LABEL org.xum.bugbash.inputs=$INPUTS_SHA256 \ + org.xum.bugbash.playwright-core=$PLAYWRIGHT_CORE_VERSION \ + org.opencontainers.image.source=https://github.com/coder/xum +ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright NPM_CONFIG_UPDATE_NOTIFIER=false +# git: startApp.ts seeds a git repo for the demo project. +# hadolint ignore=DL3008 +RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates \ + && rm -rf /var/lib/apt/lists/* +# The Chromium build that this playwright-core drives. e2e looks for both the full browser and +# the headless shell, and `install chromium` gets both. +RUN test -n "$PLAYWRIGHT_CORE_VERSION" && test -n "$INPUTS_SHA256" \ + && npx -y "playwright-core@${PLAYWRIGHT_CORE_VERSION}" install --with-deps chromium \ + && rm -rf /var/lib/apt/lists/* /root/.npm /root/.cache && chmod -R a+rX /ms-playwright +COPY --from=bun /usr/local/bin/bun /usr/local/bin/bun diff --git a/tests/bugbash/sandbox/build.sh b/tests/bugbash/sandbox/build.sh new file mode 100755 index 00000000000..db639a8db7a --- /dev/null +++ b/tests/bugbash/sandbox/build.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# Builds the bug-bash sandbox image (#5714, Dockerfile next to this file). +# +# build.sh --key print the inputs key of this checkout +# build.sh build for linux/amd64 into the local Docker, print the image ID +# build.sh --push main publish job only: push to GHCR, print the image.json record +# +# The inputs key is the sha256 of three lines: the sha256 of the Dockerfile, the sha256 of this +# script, and the playwright-core version of @e2e-dev/web in bun.lock. The image carries it as +# the label org.xum.bugbash.inputs, and the runner computes the same key for its checkout. +# The script reads these files from the working tree and refuses when they differ from HEAD, so +# the key always names committed source. +# +# The build gets an empty context, no cache and no build args from the env. --push skips the +# build when the registry already has the tag inputs-: one key, one published digest. +# Nothing here deletes an image. +set -euo pipefail + +IMAGE=ghcr.io/coder/xum-bugbash-sandbox +HERE=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +ROOT=$(git -C "$HERE" rev-parse --show-toplevel) +DOCKERFILE=tests/bugbash/sandbox/Dockerfile +SCRIPT=tests/bugbash/sandbox/build.sh + +die() { + echo "build.sh: $*" >&2 + exit 2 +} +sha() { sha256sum "$1" | cut -d' ' -f1; } + +cd "$ROOT" +for f in "$DOCKERFILE" "$SCRIPT" bun.lock; do + git cat-file -e "HEAD:$f" 2>/dev/null || die "$f is not committed: the image builds from committed source" +done +git diff --quiet HEAD -- "$DOCKERFILE" "$SCRIPT" bun.lock || + die "$DOCKERFILE, $SCRIPT or bun.lock differs from HEAD: the image builds from committed source" + +# The copy that e2e drives Chromium with. The root playwright-core is a different copy. +mapfile -t versions < <(grep -oE '"@e2e-dev/web/playwright-core": \["playwright-core@[^"]+"' bun.lock | + sed -E 's/.*playwright-core@//; s/"$//') +[ "${#versions[@]}" -eq 1 ] || die "expected one @e2e-dev/web/playwright-core entry in bun.lock, found ${#versions[@]}" +PLAYWRIGHT=${versions[0]} +[[ $PLAYWRIGHT =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || die "unexpected playwright-core version: $PLAYWRIGHT" + +KEY=$(printf 'dockerfile %s\nbuild.sh %s\nplaywright-core %s\n' "$(sha "$DOCKERFILE")" "$(sha "$SCRIPT")" "$PLAYWRIGHT" | + sha256sum | cut -d' ' -f1) +[[ $KEY =~ ^[0-9a-f]{64}$ ]] || die "bad inputs key: $KEY" + +mode=${1:-local} +[ $# -le 1 ] || die "one argument at most: --key or --push" +case "$mode" in + --key) + echo "$KEY" + exit 0 + ;; + local | --push) ;; + *) die "unknown argument: $mode" ;; +esac + +context=$(mktemp -d) +trap 'rm -rf "$context" "$context.json"' EXIT +build() { + docker buildx build --platform linux/amd64 --no-cache --pull --provenance=false --sbom=false \ + --build-arg "PLAYWRIGHT_CORE_VERSION=$PLAYWRIGHT" --build-arg "INPUTS_SHA256=$KEY" \ + --file "$ROOT/$DOCKERFILE" --metadata-file "$context.json" "$@" "$context" >&2 +} + +if [ "$mode" = local ]; then + build --load --tag "xum-bugbash-sandbox:inputs-$KEY" + docker image inspect --format '{{.Id}}' "xum-bugbash-sandbox:inputs-$KEY" + exit 0 +fi + +# --push +TAG="$IMAGE:inputs-$KEY" +if digest=$(docker buildx imagetools inspect --format '{{.Manifest.Digest}}' "$TAG" 2>/dev/null); then + echo "build.sh: $TAG exists, no build" >&2 +else + build --push --tag "$TAG" + digest=$(jq -r '."containerimage.digest"' "$context.json") +fi +[[ $digest =~ ^sha256:[0-9a-f]{64}$ ]] || die "bad digest: $digest" +jq -n --arg image "$IMAGE" --arg digest "$digest" --arg key "$KEY" --arg pw "$PLAYWRIGHT" \ + --arg from "$(git rev-parse HEAD)" \ + '{image: $image, digest: $digest, inputsKey: $key, playwrightCore: $pw, publishedFrom: $from}' diff --git a/tests/bugbash/sandbox/build.test.ts b/tests/bugbash/sandbox/build.test.ts new file mode 100644 index 00000000000..df3baaff662 --- /dev/null +++ b/tests/bugbash/sandbox/build.test.ts @@ -0,0 +1,73 @@ +import { afterEach, expect, test } from "bun:test"; +import { spawnSync } from "node:child_process"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; + +// build.sh --key names committed source only, and the key follows each of its three inputs. +// Each test runs the real script in a throwaway git repo with copies of its inputs. +const SANDBOX = "tests/bugbash/sandbox"; +const lock = (version: string) => + `{\n "packages": {\n "playwright-core": ["playwright-core@1.57.0", ""],\n` + + ` "@e2e-dev/web/playwright-core": ["playwright-core@${version}", ""],\n }\n}\n`; + +let repo = ""; +afterEach(() => fs.rmSync(repo, { recursive: true, force: true })); + +function git(...args: string[]) { + const r = spawnSync("git", args, { cwd: repo, encoding: "utf8" }); + expect(r.status).toBe(0); +} +function setup(version = "1.63.0") { + repo = fs.mkdtempSync(path.join(os.tmpdir(), "xbb-build-key-")); + fs.mkdirSync(path.join(repo, SANDBOX), { recursive: true }); + for (const name of ["build.sh", "Dockerfile"]) + fs.copyFileSync(path.join(import.meta.dir, name), path.join(repo, SANDBOX, name)); + fs.writeFileSync(path.join(repo, "bun.lock"), lock(version)); + git("init", "-q"); + commit(); +} +function commit() { + git("add", "-A"); + git("-c", "user.name=t", "-c", "user.email=t@t", "commit", "-qm", "x"); +} +function key() { + const r = spawnSync(path.join(repo, SANDBOX, "build.sh"), ["--key"], { encoding: "utf8" }); + return { status: r.status, out: r.stdout.trim(), err: r.stderr }; +} + +test("the key changes with each input and is stable otherwise", () => { + setup(); + const first = key(); + expect(first.status).toBe(0); + expect(first.out).toMatch(/^[0-9a-f]{64}$/); + expect(key().out).toBe(first.out); + + const keys = new Set([first.out]); + fs.writeFileSync(path.join(repo, "bun.lock"), lock("1.64.0")); + commit(); + keys.add(key().out); + fs.appendFileSync(path.join(repo, SANDBOX, "Dockerfile"), "# change\n"); + commit(); + keys.add(key().out); + fs.appendFileSync(path.join(repo, SANDBOX, "build.sh"), "# change\n"); + commit(); + keys.add(key().out); + expect(keys.size).toBe(4); +}); + +test("an uncommitted input refuses", () => { + setup(); + fs.appendFileSync(path.join(repo, SANDBOX, "Dockerfile"), "# local edit\n"); + expect(key()).toMatchObject({ status: 2, out: "" }); + expect(key().err).toContain("differs from HEAD"); +}); + +test("the version comes only from the @e2e-dev/web copy, and must be unique", () => { + setup(); + // The root playwright-core entry (1.57.0) does not count. + fs.writeFileSync(path.join(repo, "bun.lock"), lock("1.63.0").replace("@e2e-dev/web/", "x/")); + commit(); + expect(key()).toMatchObject({ status: 2, out: "" }); + expect(key().err).toContain("found 0"); +}); From 14dc77b0ed2bdba01a1d48449b943716f462d4d4 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 7 Oct 2026 16:51:29 +0000 Subject: [PATCH 2/3] build.sh: hash the raw manifest for an existing tag; portable to bash 3.2 and shasum --- tests/bugbash/sandbox/build.sh | 25 +++++++++++++++++-------- tests/bugbash/sandbox/build.test.ts | 21 +++++++++++++++++++++ 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/tests/bugbash/sandbox/build.sh b/tests/bugbash/sandbox/build.sh index db639a8db7a..39e8479ade4 100755 --- a/tests/bugbash/sandbox/build.sh +++ b/tests/bugbash/sandbox/build.sh @@ -26,7 +26,11 @@ die() { echo "build.sh: $*" >&2 exit 2 } -sha() { sha256sum "$1" | cut -d' ' -f1; } +# sha256 of stdin. Stock macOS has shasum, not sha256sum. +sha_stdin() { + if command -v sha256sum >/dev/null; then sha256sum; else shasum -a 256; fi | cut -d' ' -f1 +} +sha() { sha_stdin <"$1"; } cd "$ROOT" for f in "$DOCKERFILE" "$SCRIPT" bun.lock; do @@ -36,14 +40,16 @@ git diff --quiet HEAD -- "$DOCKERFILE" "$SCRIPT" bun.lock || die "$DOCKERFILE, $SCRIPT or bun.lock differs from HEAD: the image builds from committed source" # The copy that e2e drives Chromium with. The root playwright-core is a different copy. -mapfile -t versions < <(grep -oE '"@e2e-dev/web/playwright-core": \["playwright-core@[^"]+"' bun.lock | - sed -E 's/.*playwright-core@//; s/"$//') -[ "${#versions[@]}" -eq 1 ] || die "expected one @e2e-dev/web/playwright-core entry in bun.lock, found ${#versions[@]}" -PLAYWRIGHT=${versions[0]} +# No mapfile: the system bash of macOS is 3.2. +versions=$(grep -oE '"@e2e-dev/web/playwright-core": \["playwright-core@[^"]+"' bun.lock | + sed -E 's/.*playwright-core@//; s/"$//') || true +count=$(printf '%s' "$versions" | grep -c . || true) +[ "$count" -eq 1 ] || die "expected one @e2e-dev/web/playwright-core entry in bun.lock, found $count" +PLAYWRIGHT=$versions [[ $PLAYWRIGHT =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || die "unexpected playwright-core version: $PLAYWRIGHT" KEY=$(printf 'dockerfile %s\nbuild.sh %s\nplaywright-core %s\n' "$(sha "$DOCKERFILE")" "$(sha "$SCRIPT")" "$PLAYWRIGHT" | - sha256sum | cut -d' ' -f1) + sha_stdin) [[ $KEY =~ ^[0-9a-f]{64}$ ]] || die "bad inputs key: $KEY" mode=${1:-local} @@ -58,7 +64,7 @@ case "$mode" in esac context=$(mktemp -d) -trap 'rm -rf "$context" "$context.json"' EXIT +trap 'rm -rf "$context" "$context.json" "$context.raw"' EXIT build() { docker buildx build --platform linux/amd64 --no-cache --pull --provenance=false --sbom=false \ --build-arg "PLAYWRIGHT_CORE_VERSION=$PLAYWRIGHT" --build-arg "INPUTS_SHA256=$KEY" \ @@ -73,7 +79,10 @@ fi # --push TAG="$IMAGE:inputs-$KEY" -if digest=$(docker buildx imagetools inspect --format '{{.Manifest.Digest}}' "$TAG" 2>/dev/null); then +# The digest of a manifest is the sha256 of its raw bytes. Hash the file, not a --format value: +# some buildx versions ignore --format here and print a table (docker/buildx#4006). +if docker buildx imagetools inspect --raw "$TAG" >"$context.raw" 2>/dev/null; then + digest="sha256:$(sha "$context.raw")" echo "build.sh: $TAG exists, no build" >&2 else build --push --tag "$TAG" diff --git a/tests/bugbash/sandbox/build.test.ts b/tests/bugbash/sandbox/build.test.ts index df3baaff662..c3468802025 100644 --- a/tests/bugbash/sandbox/build.test.ts +++ b/tests/bugbash/sandbox/build.test.ts @@ -71,3 +71,24 @@ test("the version comes only from the @e2e-dev/web copy, and must be unique", () expect(key()).toMatchObject({ status: 2, out: "" }); expect(key().err).toContain("found 0"); }); + +test("--push for an existing tag reports the digest of its manifest bytes, and builds nothing", () => { + setup(); + // A fake docker: `imagetools inspect --raw` prints a manifest, any other command fails. + const bin = path.join(repo, "bin"); + fs.mkdirSync(bin); + const manifest = '{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json"}\n'; + fs.writeFileSync(path.join(bin, "manifest.json"), manifest); + fs.writeFileSync( + path.join(bin, "docker"), + `#!/bin/sh\n[ "$1 $2 $3 $4" = "buildx imagetools inspect --raw" ] || exit 9\ncat "${bin}/manifest.json"\n`, + { mode: 0o755 } + ); + const r = spawnSync(path.join(repo, SANDBOX, "build.sh"), ["--push"], { + encoding: "utf8", + env: { ...process.env, PATH: `${bin}:${process.env.PATH ?? ""}` }, + }); + expect(r.status).toBe(0); + const digest = `sha256:${new Bun.CryptoHasher("sha256").update(manifest).digest("hex")}`; + expect(JSON.parse(r.stdout)).toMatchObject({ digest, playwrightCore: "1.63.0" }); +}); From 6bcad507cc83840dee1449b686413338a3c9517a Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 8 Oct 2026 05:42:53 +0000 Subject: [PATCH 3/3] Option M: manual publish, fresh build each run, Make targets, attest own output --- .github/workflows/bugbash-sandbox-image.yml | 34 ++++++++-------- Makefile | 13 ++++++- tests/bugbash/sandbox/build.sh | 31 +++++++-------- tests/bugbash/sandbox/build.test.ts | 43 +++++++++++++++------ 4 files changed, 75 insertions(+), 46 deletions(-) diff --git a/.github/workflows/bugbash-sandbox-image.yml b/.github/workflows/bugbash-sandbox-image.yml index ca67e6f45d7..ac695826b3c 100644 --- a/.github/workflows/bugbash-sandbox-image.yml +++ b/.github/workflows/bugbash-sandbox-image.yml @@ -1,24 +1,22 @@ # The bug-bash sandbox image (#5714): tests/bugbash/sandbox/Dockerfile, built by build.sh. # -# Pull requests build it with a read-only token and push nothing. Only a push to main (or a -# manual run on main) publishes, from that exact commit, with no cache and no artifact from any -# other run. A person then copies the digest from the job summary into image.json in a pull +# Pull requests build it with a read-only token and push nothing. Publishing is manual: only a +# workflow_dispatch run on main publishes, from that run's exact commit, with no cache and no +# artifact from any other run. Each publish builds fresh, and the attestation names that build's +# own output. A person then copies the digest from the job summary into image.json in a pull # request: no workflow writes to the repo. Nothing here deletes an image. +# +# Until a new image is published and its digest PR merges, runners refuse a checkout whose image +# inputs changed (stale inputs key). name: Bug-bash sandbox image on: - push: - branches: [main] - paths: - - tests/bugbash/sandbox/Dockerfile - - tests/bugbash/sandbox/build.sh - - bun.lock - - .github/workflows/bugbash-sandbox-image.yml pull_request: paths: - tests/bugbash/sandbox/Dockerfile - tests/bugbash/sandbox/build.sh - bun.lock + - Makefile - .github/workflows/bugbash-sandbox-image.yml workflow_dispatch: {} @@ -42,17 +40,17 @@ jobs: persist-credentials: false - name: Build run: | - id=$(./tests/bugbash/sandbox/build.sh) + id=$(make -s bugbash-sandbox-image) size=$(docker image inspect --format '{{.Size}}' "$id") - echo "Image $id: $size bytes (inputs key $(./tests/bugbash/sandbox/build.sh --key))" | tee -a "$GITHUB_STEP_SUMMARY" + echo "Image $id: $size bytes (inputs key $(make -s bugbash-sandbox-key))" | tee -a "$GITHUB_STEP_SUMMARY" publish: name: Publish - if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' + if: github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' runs-on: ubuntu-latest permissions: contents: read - # Push ghcr.io/coder/xum-bugbash-sandbox. Only this job, only on main. + # Push ghcr.io/coder/xum-bugbash-sandbox. Only this job, only a manual run on main. packages: write # Build provenance for audit (the runner trusts the digest in image.json). id-token: write @@ -71,8 +69,10 @@ jobs: - name: Build and push id: push run: | - ./tests/bugbash/sandbox/build.sh --push > image.json - echo "digest=$(jq -r .digest image.json)" >> "$GITHUB_OUTPUT" + make -s bugbash-sandbox-publish > image.json + # The subject of the attestation is this build's own output (image.json). + echo "image=$(jq -er .image image.json)" >> "$GITHUB_OUTPUT" + echo "digest=$(jq -er .digest image.json)" >> "$GITHUB_OUTPUT" { echo '### tests/bugbash/sandbox/image.json' echo @@ -83,6 +83,6 @@ jobs: - name: Attest uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 with: - subject-name: ghcr.io/coder/xum-bugbash-sandbox + subject-name: ${{ steps.push.outputs.image }} subject-digest: ${{ steps.push.outputs.digest }} push-to-registry: true diff --git a/Makefile b/Makefile index a65a7cf6195..fda22559426 100644 --- a/Makefile +++ b/Makefile @@ -91,7 +91,7 @@ include fmt.mk .PHONY: all build dev start clean help .PHONY: build-renderer version build-icons build-static build-docker-runtime verify-docker-runtime-artifacts .PHONY: lint lint-fix typecheck static-check static-check-full -.PHONY: test-bugbash-repros test-bugbash-known-failures +.PHONY: test-bugbash-repros test-bugbash-known-failures bugbash-sandbox-key bugbash-sandbox-image bugbash-sandbox-publish .PHONY: test test-unit test-unit-ci test-integration test-watch test-coverage test-e2e test-e2e-perf perf-tape-replay smoke-test .PHONY: dist dist-mac dist-win dist-linux install-mac-arm64 ensure-mac-sharp-runtime-deps check-appimage-icons check-mac-attach-file-runtime .PHONY: vscode-ext vscode-ext-install @@ -224,6 +224,17 @@ dev-server-sandbox: ## Start an isolated dev-server instance (fresh XUM_ROOT + f bug-bash: ## Agent bug bash: e2e explore charters x models (paused on the host until the sandbox lands, #5714) @bun tests/bugbash/run.ts $(BUGBASH_ARGS) +# The bug-bash sandbox image (tests/bugbash/sandbox/build.sh, #5714). Each prints one value on +# stdout. Publishing runs only in the manual "Bug-bash sandbox image" workflow on main. +bugbash-sandbox-key: ## Print the inputs key of the bug-bash sandbox image for this checkout + @./tests/bugbash/sandbox/build.sh --key + +bugbash-sandbox-image: ## Build the bug-bash sandbox image locally (linux/amd64) and print its image ID + @./tests/bugbash/sandbox/build.sh + +bugbash-sandbox-publish: ## Workflow only: build and push the bug-bash sandbox image, print its image.json record + @./tests/bugbash/sandbox/build.sh --push + # Bug-bash repro tests (tests/bugbash/repros/*.e2e.ts): exact UI steps, no model calls, against a # seeded `xum server` (tests/bugbash/startApp.ts). e2e needs Node.js 22.22.3+ or 24.8+ (E2E_NODE) # and its own Chromium: `node_modules/.bin/e2e-web install chromium`. diff --git a/tests/bugbash/sandbox/build.sh b/tests/bugbash/sandbox/build.sh index 39e8479ade4..1fbcf6fca25 100755 --- a/tests/bugbash/sandbox/build.sh +++ b/tests/bugbash/sandbox/build.sh @@ -3,7 +3,7 @@ # # build.sh --key print the inputs key of this checkout # build.sh build for linux/amd64 into the local Docker, print the image ID -# build.sh --push main publish job only: push to GHCR, print the image.json record +# build.sh --push the manual publish job only: build, push to GHCR, print the image.json record # # The inputs key is the sha256 of three lines: the sha256 of the Dockerfile, the sha256 of this # script, and the playwright-core version of @e2e-dev/web in bun.lock. The image carries it as @@ -11,8 +11,10 @@ # The script reads these files from the working tree and refuses when they differ from HEAD, so # the key always names committed source. # -# The build gets an empty context, no cache and no build args from the env. --push skips the -# build when the registry already has the tag inputs-: one key, one published digest. +# The build gets an empty context, no cache and no build args from the env. --push always builds +# fresh: it never checks the registry and never reuses an image. It pushes a new tag, so it +# overwrites no tag, and it reports the name and digest from its own build output. Two publishes +# of one inputs key give two digests; the reviewed digest in image.json is what runners trust. # Nothing here deletes an image. set -euo pipefail @@ -64,7 +66,7 @@ case "$mode" in esac context=$(mktemp -d) -trap 'rm -rf "$context" "$context.json" "$context.raw"' EXIT +trap 'rm -rf "$context" "$context.json"' EXIT build() { docker buildx build --platform linux/amd64 --no-cache --pull --provenance=false --sbom=false \ --build-arg "PLAYWRIGHT_CORE_VERSION=$PLAYWRIGHT" --build-arg "INPUTS_SHA256=$KEY" \ @@ -77,18 +79,13 @@ if [ "$mode" = local ]; then exit 0 fi -# --push -TAG="$IMAGE:inputs-$KEY" -# The digest of a manifest is the sha256 of its raw bytes. Hash the file, not a --format value: -# some buildx versions ignore --format here and print a table (docker/buildx#4006). -if docker buildx imagetools inspect --raw "$TAG" >"$context.raw" 2>/dev/null; then - digest="sha256:$(sha "$context.raw")" - echo "build.sh: $TAG exists, no build" >&2 -else - build --push --tag "$TAG" - digest=$(jq -r '."containerimage.digest"' "$context.json") -fi -[[ $digest =~ ^sha256:[0-9a-f]{64}$ ]] || die "bad digest: $digest" -jq -n --arg image "$IMAGE" --arg digest "$digest" --arg key "$KEY" --arg pw "$PLAYWRIGHT" \ +# --push. One new tag per publish (inputs key, commit, UTC time), so a publish overwrites no tag. +build --push --tag "$IMAGE:inputs-${KEY:0:16}-$(git rev-parse --short=12 HEAD)-$(date -u +%Y%m%d%H%M%S)" +# The name and digest come from this build's own metadata, not from a registry lookup. +digest=$(jq -r '."containerimage.digest"' "$context.json") +pushed=$(jq -r '."image.name"' "$context.json") +[[ $digest =~ ^sha256:[0-9a-f]{64}$ ]] || die "bad digest in the build metadata: $digest" +[[ $pushed == "$IMAGE:"* && $pushed != *,* ]] || die "unexpected image name in the build metadata: $pushed" +jq -n --arg image "${pushed%:*}" --arg digest "$digest" --arg key "$KEY" --arg pw "$PLAYWRIGHT" \ --arg from "$(git rev-parse HEAD)" \ '{image: $image, digest: $digest, inputsKey: $key, playwrightCore: $pw, publishedFrom: $from}' diff --git a/tests/bugbash/sandbox/build.test.ts b/tests/bugbash/sandbox/build.test.ts index c3468802025..ce497627490 100644 --- a/tests/bugbash/sandbox/build.test.ts +++ b/tests/bugbash/sandbox/build.test.ts @@ -72,23 +72,44 @@ test("the version comes only from the @e2e-dev/web copy, and must be unique", () expect(key().err).toContain("found 0"); }); -test("--push for an existing tag reports the digest of its manifest bytes, and builds nothing", () => { +test("--push always builds fresh and reports the name and digest of its own build", () => { setup(); - // A fake docker: `imagetools inspect --raw` prints a manifest, any other command fails. + // A fake docker: `buildx build` writes build metadata to --metadata-file, and every call is + // logged. A registry lookup (imagetools) would be a reuse path. const bin = path.join(repo, "bin"); fs.mkdirSync(bin); - const manifest = '{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json"}\n'; - fs.writeFileSync(path.join(bin, "manifest.json"), manifest); + const calls = path.join(bin, "calls.log"); + const digest = `sha256:${"ab".repeat(32)}`; fs.writeFileSync( path.join(bin, "docker"), - `#!/bin/sh\n[ "$1 $2 $3 $4" = "buildx imagetools inspect --raw" ] || exit 9\ncat "${bin}/manifest.json"\n`, + `#!/bin/sh +echo "$*" >> "${calls}" +[ "$1 $2" = "buildx build" ] || exit 9 +tag=""; meta="" +while [ $# -gt 0 ]; do + case "$1" in --tag) tag=$2 ;; --metadata-file) meta=$2 ;; esac + shift +done +printf '{"containerimage.digest":"${digest}","image.name":"%s"}' "$tag" > "$meta" +`, { mode: 0o755 } ); - const r = spawnSync(path.join(repo, SANDBOX, "build.sh"), ["--push"], { - encoding: "utf8", - env: { ...process.env, PATH: `${bin}:${process.env.PATH ?? ""}` }, - }); + const push = () => + spawnSync(path.join(repo, SANDBOX, "build.sh"), ["--push"], { + encoding: "utf8", + env: { ...process.env, PATH: `${bin}:${process.env.PATH ?? ""}` }, + }); + const r = push(); expect(r.status).toBe(0); - const digest = `sha256:${new Bun.CryptoHasher("sha256").update(manifest).digest("hex")}`; - expect(JSON.parse(r.stdout)).toMatchObject({ digest, playwrightCore: "1.63.0" }); + expect(JSON.parse(r.stdout)).toMatchObject({ + image: "ghcr.io/coder/xum-bugbash-sandbox", + digest, + inputsKey: key().out, + playwrightCore: "1.63.0", + }); + // A second publish of the same inputs builds again. Every call was a push build. + expect(push().status).toBe(0); + const log = fs.readFileSync(calls, "utf8").trim().split("\n"); + expect(log).toHaveLength(2); + for (const line of log) expect(line).toMatch(/^buildx build .*--no-cache .*--push /); });