diff --git a/tests/bugbash/sandbox/Dockerfile b/tests/bugbash/sandbox/Dockerfile new file mode 100644 index 0000000000..8b8a23222d --- /dev/null +++ b/tests/bugbash/sandbox/Dockerfile @@ -0,0 +1,19 @@ +# The bug-bash sandbox image (#5714): tools only. The app (dist), node_modules and the tests +# come from the checkout at run time, read-only. sandbox/launch.ts tags the image with a hash +# of this file and its build args, and builds it when that tag is missing. +ARG BUN_VERSION=1.3.12 +FROM oven/bun:${BUN_VERSION}-slim AS bun + +FROM node:22-slim +ARG PLAYWRIGHT_CORE_VERSION +ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright NPM_CONFIG_UPDATE_NOTIFIER=false +# git: startApp.ts seeds a git repo for the demo project. +# hadolint ignore=DL3008 +RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates \ + && rm -rf /var/lib/apt/lists/* +# The Chromium build that the checkout's @e2e-dev/web expects (launch.ts passes its version). +# e2e looks for both the full browser and the headless shell. +RUN test -n "$PLAYWRIGHT_CORE_VERSION" \ + && npx -y "playwright-core@${PLAYWRIGHT_CORE_VERSION}" install --with-deps chromium \ + && rm -rf /var/lib/apt/lists/* /root/.npm /root/.cache && chmod -R a+rX /ms-playwright +COPY --from=bun /usr/local/bin/bun /usr/local/bin/bun diff --git a/tests/bugbash/sandbox/entry.ts b/tests/bugbash/sandbox/entry.ts new file mode 100644 index 0000000000..f09e09c062 --- /dev/null +++ b/tests/bugbash/sandbox/entry.ts @@ -0,0 +1,90 @@ +/** + * The job process of a bug-bash sandbox container (#5714), under docker-init (`--init`). + * Usage: bun sandbox/entry.ts --export -- + * + * The job's stdout and stderr go to this process's stderr, so its stdout carries only the export + * stream (exportStream.ts). Its stdin is the lifeline: sandbox/launch.ts holds the other end, so + * EOF means that the launcher is gone, and the container stops. + */ +import { spawn } from "child_process"; +import * as fs from "fs"; +import * as os from "os"; +import { writeExport } from "./exportStream"; + +assertInSandbox(); // first: on a host, kill(-1) below hits every process of this user + +const args = process.argv.slice(2); +if (args[0] !== "--export" || args[2] !== "--" || args.length < 4) { + console.error("usage: entry.ts --export -- "); + process.exit(2); +} +const exportDir = args[1]; +const [command, ...commandArgs] = args.slice(3); + +/** Linux skips PID 1 and the caller. When this process exits, docker-init exits too. */ +function killAllOthers(): void { + try { + process.kill(-1, "SIGKILL"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; + } +} + +/** SIGKILL is not instant: wait until docker-init has reaped every other process. */ +async function othersGone(): Promise { + const others = () => + fs + .readdirSync("/proc") + .filter((name) => /^\d+$/.test(name) && name !== "1" && name !== String(process.pid)); + for (let i = 0; i < 100 && others().length > 0; i++) await Bun.sleep(50); + const left = others(); + if (left.length > 0) throw new Error(`processes ${left.join(" ")} still run after SIGKILL`); +} + +process.stdin.on("end", () => { + console.error("sandbox entry: the launcher is gone, so the job stops"); + killAllOthers(); + process.exit(137); +}); +process.stdin.resume(); + +let finished = false; +async function finish(code: number): Promise { + if (finished) return; + finished = true; + killAllOthers(); // the app and Chromium too: nothing writes to the output after this + await othersGone(); + const sent = await writeExport(process.stdout, exportDir); + if (sent.skipped.length > 0) + console.error(`sandbox entry: not exported: ${sent.skipped.join(", ")}`); + process.exit(code); +} + +// A failed export exits nonzero. The launcher then reports the evidence as incomplete. +const done = (code: number): void => { + finish(code).catch((error: unknown) => { + console.error(`sandbox entry: export failed: ${String(error)}`); + process.exit(1); + }); +}; +const job = spawn(command, commandArgs, { stdio: ["ignore", 2, 2] }); +job.on("error", (error) => { + console.error(`sandbox entry: ${command}: ${error.message}`); + done(127); +}); +// The shell convention, as the launcher's exitCode(): 128 + the signal number. +job.on("exit", (code, signal) => + done(code ?? (signal != null ? 128 + os.constants.signals[signal] : 1)) +); + +function assertInSandbox(): void { + const init = fs.readFileSync("/proc/1/cmdline", "utf8"); + const ok = + process.env.BUGBASH_CONTAINER === "1" && + init.startsWith("/sbin/docker-init\0") && + fs.readdirSync("/sys/class/net").join() === "lo"; + if (!ok) { + console.error("sandbox entry: not in the bug-bash sandbox, so it refuses to run"); + process.exit(2); + } +} diff --git a/tests/bugbash/sandbox/launch.test.ts b/tests/bugbash/sandbox/launch.test.ts new file mode 100644 index 0000000000..00565d0dc8 --- /dev/null +++ b/tests/bugbash/sandbox/launch.test.ts @@ -0,0 +1,437 @@ +import { expect, test } from "bun:test"; +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import { spawnSync } from "child_process"; +import * as crypto from "crypto"; +import { appAi, containerEnv, exactStepRefusal, exitCode, outputDir, plainFolders } from "./launch"; + +test("the container env holds the allowlisted names, the fixed values and no host secret", () => { + const host = { + BUGBASH_AI_RESOLVED: "mock", + BUGBASH_AI_REASON: "https://user:pass@proxy.example/v1 unreachable", + BUGBASH_MODEL: "anthropic:claude-sonnet-5-5", + ANTHROPIC_API_KEY: "sk-ant-real", + OPENAI_API_KEY: "sk-real", + GH_TOKEN: "ghp_real", + XUM_SERVER_AUTH_TOKEN: "real", + HOME: "/home/alice", + PATH: "/usr/bin", + TMPDIR: "/home/alice/tmp", + }; + expect(containerEnv(host, { BUGBASH_APP_LOG: ".e2e/run/app.log" })).toEqual({ + HOME: "/home/bugbash", + TMPDIR: "/tmp", + BUGBASH_CONTAINER: "1", + BUGBASH_AI_RESOLVED: "mock", + BUGBASH_MODEL: "anthropic:claude-sonnet-5-5", + BUGBASH_APP_LOG: ".e2e/run/app.log", + }); +}); + +test("a credential from a caller is refused, not passed", () => { + // prettier-ignore + for (const name of ["ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "OPENAI_BASE_URL", "GH_TOKEN", "openai_api_key"]) { + expect(() => containerEnv({}, { [name]: "x" })).toThrow("no credential enters the sandbox"); + } +}); + +test("the export comes back only to one folder under .e2e", () => { + expect(outputDir(["run", "--output", ".e2e/repros", "--grep", "x"])).toBe(".e2e/repros"); + expect(outputDir(["run", "--output=.e2e/runs/one"])).toBe(".e2e/runs/one"); + for (const args of [ + ["run"], + ["run", "--output"], + ["run", "--output", ".e2e/a", "--output", ".e2e/b"], + ["run", "--output", ".e2e"], + ["run", "--output", ".e2e/../escape"], + ["run", "--output", ".e2e/./a"], + ["run", "--output", "/tmp/escape"], + ["run", "--output", "../.e2e/escape"], + ["run", "--output", ".e2e/a b"], + ]) { + expect(() => outputDir(args)).toThrow("exactly one --output"); + } +}); + +test("the launcher reads the app AI mode the way e2e.config.ts does", () => { + expect(appAi({ BUGBASH_AI: "mock" })).toBe("mock"); + expect(appAi({ BUGBASH_AI: "mock", BUGBASH_AI_RESOLVED: "real" })).toBe("real"); + expect(appAi({ BUGBASH_AI: "auto" })).toBeUndefined(); +}); + +test("a symlinked folder cannot lead a copy or an export out of the checkout", () => { + const base = fs.mkdtempSync(path.join(os.tmpdir(), "xbb-launch-test-")); + const outside = fs.mkdtempSync(path.join(os.tmpdir(), "xbb-launch-test-")); + try { + fs.symlinkSync(outside, path.join(base, ".e2e")); + fs.mkdirSync(path.join(base, "src")); + fs.writeFileSync(path.join(base, "src/file"), ""); + expect(() => plainFolders(base, ".e2e/run", true)).toThrow("not a plain folder"); + expect(() => plainFolders(base, ".e2e", false)).toThrow("not a plain folder"); + expect(() => plainFolders(base, "src/file", true)).toThrow("not a plain folder"); + expect(fs.readdirSync(outside)).toEqual([]); + plainFolders(base, "src/new/run", true); + expect(fs.lstatSync(path.join(base, "src/new/run")).isDirectory()).toBe(true); + expect(() => plainFolders(base, "src/missing", false)).toThrow("not a plain folder"); + } finally { + fs.rmSync(base, { recursive: true, force: true }); + fs.rmSync(outside, { recursive: true, force: true }); + } +}); + +test("a job that a signal ended exits 128 + the signal number", () => { + expect(exitCode(3, null)).toBe(3); + expect(exitCode(null, "SIGKILL")).toBe(137); + expect(exitCode(null, "SIGTERM")).toBe(143); +}); + +const BUGBASH_DIR = path.resolve(import.meta.dir, ".."); +const OUT = ["--output", ".e2e/launch-test"]; +const OK = ["run", "--config", "e2e.config.ts", ...OUT]; +// Each one would let e2e load another config or tests, or let a model pick actions. +const NOT_EXACT: string[][] = [ + ["run", "--config", "e2e.config.ts", "--config", "e2e.mcpapps.config.ts", ...OUT], + ["run", "--config=e2e.mcpapps.config.ts", ...OUT], + ["run", "--config", "./e2e.config.ts", ...OUT], + ["run", "--config", "../bugbash/e2e.config.ts", ...OUT], + ["run", "--config", "./tests/../e2e.config.ts", ...OUT], + ["run", "--config", "mcpapps/../e2e.config.ts", ...OUT], + [...OK, "--", "mcpapps/mcp-apps.e2e.ts"], + [...OK, "mcpapps/mcp-apps.e2e.ts"], + ["explore", "--config", "e2e.config.ts", ...OUT], + ["run", "explore", "--config", "e2e.config.ts", ...OUT], + [...OK, "explore"], + [...OK, "--agent", "explorer"], + [...OK, "--no-cache"], + ["run", "--tag", "--config", "e2e.mcpapps.config.ts", ...OUT], + ["run", ...OUT], + // Each of these is refused by one rule only (the mutation check showed that the cases above + // also trip a second rule): two positionals, a dash-led option value, `explore` as a value. + [...OK, "mcpapps/mcp-apps.e2e.ts", "mcpapps/seed.ts"], + ["run", "--grep", "--config=e2e.mcpapps.config.ts", "--config", "e2e.config.ts", ...OUT], + [...OK, "--grep", "explore"], +]; + +test("only `e2e run --config e2e.config.ts` with selection options is an exact-step run", () => { + expect(exactStepRefusal(OK, BUGBASH_DIR)).toBeNull(); + // prettier-ignore + expect(exactStepRefusal(["run", "--config=e2e.config.ts", "--tag", "a", "--grep=b", "--pass-with-no-tests", ...OUT], BUGBASH_DIR)).toBeNull(); + for (const args of NOT_EXACT) expect(exactStepRefusal(args, BUGBASH_DIR)).toBeString(); + // e2e resolves --config from its cwd: another folder, or a symlinked config, is refused. + const other = fs.mkdtempSync(path.join(os.tmpdir(), "xbb-launch-test-")); + try { + expect(exactStepRefusal(OK, other)).toContain("the cwd must be"); + fs.symlinkSync(path.join(BUGBASH_DIR, "e2e.config.ts"), path.join(other, "e2e.config.ts")); + expect(exactStepRefusal(OK, other, other)).toContain("not a regular file"); + } finally { + fs.rmSync(other, { recursive: true, force: true }); + } +}); + +// The fake docker CLI. It logs each call: its args, its env names, DOCKER_HOST, DOCKER_CONFIG, +// the files in that folder, and whether the env or that folder holds the synthetic MARKER. +// It answers like a local Linux engine. `context` answers from DOCKER_HOST, else DOCKER_CONTEXT. +// The launcher passes the fake no FAKE_* env (that is the rule under test), so launch() writes +// the log path and the modes into the script. +const FAKE_DOCKER = String.raw` +{ + echo "CALL $*" + echo "ENV $(env | sed 's/=.*//' | grep -vxE 'PWD|OLDPWD|SHLVL|_' | sort | tr '\n' ' ')" + echo "HOST $DOCKER_HOST CONFIG $DOCKER_CONFIG" + echo "FILES $(ls -A "$DOCKER_CONFIG" 2>&1 | tr '\n' ' ')" + echo "MARKER $(env | grep -c MARKER) $(grep -rl MARKER "$DOCKER_CONFIG" 2>/dev/null | wc -l)" +} >> "$FAKE_LOG" +case "$1" in + context) + if [ "$DOCKER_CONTEXT" = missing ]; then echo 'context "missing": not found' >&2; exit 1; fi + if [ -n "$DOCKER_HOST" ]; then echo "$DOCKER_HOST" + elif [ "$DOCKER_CONTEXT" = remote ]; then echo tcp://10.0.0.1:2376 + elif [ "$DOCKER_CONTEXT" = other ]; then echo unix:///run/other.sock + else echo unix:///var/run/docker.sock; fi ;; + info) echo '{"OSType":"linux","OperatingSystem":"Ubuntu","SecurityOptions":[]}' ;; + image) [ "$FAKE_IMAGE" = present ] ;; + build) cat > "$FAKE_LOG.dockerfile" ;; + run) + src=$(printf '%s\n' "$@" | sed -n 's/^type=bind,src=\([^,]*\),dst=\/probe,readonly$/\1/p') + if [ -z "$src" ]; then printf '{"end":true}\n'; exit 0; fi + if [ "$FAKE_SLOW" = probe ]; then touch "$FAKE_LOG.ready"; sleep 3; fi + [ "$FAKE_PROBE" = fail ] && exit 1 + cat /proc/sys/kernel/random/boot_id; cat "$src/.nonce"; echo ;; + ps) [ -e "$FAKE_LOG.rm" ] || echo cid123 ;; + rm) touch "$FAKE_LOG.rm" ;; + *) exit 1 ;; +esac +`; + +interface Launch { + /** The real app AI: the launcher refuses before any docker call. */ + realAi?: boolean; + cwd?: string; + env?: Record; + /** The fake's same-host probe fails. */ + probeFails?: boolean; + /** The image is missing, so the launcher builds it. */ + build?: boolean; + /** This step waits 3 s and touches `.ready` first (git ls-files for the staging). */ + slow?: "stage" | "probe"; +} + +const REAL_GIT = Bun.which("git") ?? "git"; + +/** A bin folder with the fake docker (and, for a slow staging, a git that waits first). */ +function fakeBin(options: Launch) { + const bin = fs.mkdtempSync(path.join(os.tmpdir(), "xbb-launch-test-")); + const calls = path.join(bin, "calls.log"); + const script = FAKE_DOCKER.replaceAll("$FAKE_LOG", calls) + .replaceAll("$FAKE_PROBE", options.probeFails === true ? "fail" : "ok") + .replaceAll("$FAKE_IMAGE", options.build === true ? "missing" : "present") + .replaceAll("$FAKE_SLOW", options.slow ?? ""); + fs.writeFileSync(path.join(bin, "docker"), `#!/bin/sh${script}`, { mode: 0o755 }); + if (options.slow === "stage") + fs.writeFileSync( + path.join(bin, "git"), + `#!/bin/sh\ncase "$*" in *ls-files*) touch ${calls}.ready; sleep 3 ;; esac\nexec ${REAL_GIT} "$@"\n`, + { mode: 0o755 } + ); + return { bin, calls }; +} + +function launchEnv(bin: string, options: Launch): Record { + return { + PATH: `${bin}:${process.env.PATH ?? ""}`, + HOME: process.env.HOME ?? "", + BUGBASH_AI_RESOLVED: options.realAi === true ? "real" : "mock", + ...options.env, + }; +} + +/** Runs the launcher with the fake docker, which logs each call. */ +function launch(args: string[], options: Launch = {}) { + const { bin, calls } = fakeBin(options); + try { + const r = spawnSync( + process.execPath, + [path.join(import.meta.dir, "launch.ts"), "--", ...args], + { cwd: options.cwd ?? BUGBASH_DIR, encoding: "utf8", env: launchEnv(bin, options) } + ); + const log = fs.existsSync(calls) ? fs.readFileSync(calls, "utf8") : ""; + const dockerfile = fs.existsSync(`${calls}.dockerfile`) + ? fs.readFileSync(`${calls}.dockerfile`, "utf8") + : null; + return { status: r.status, stderr: r.stderr, log, dockerfile }; + } finally { + fs.rmSync(bin, { recursive: true, force: true }); + fs.rmSync(path.join(BUGBASH_DIR, ".e2e/launch-test"), { recursive: true, force: true }); + } +} + +test("the launcher runs exact-step repros in the sandbox, and nothing else anywhere", () => { + // Control: the exact-step run reaches the probe and the job container. + const sandboxed = launch(OK); + expect([sandboxed.status, sandboxed.log.match(/CALL run/g)?.length]).toEqual([0, 2]); + for (const args of NOT_EXACT) { + const r = launch(args); + expect({ args, status: r.status, ran: r.log.match(/CALL (run|build)/g) }).toEqual({ + args, + status: 2, + ran: null, + }); + expect(r.stderr).toContain("exact-step repros"); + } + // The same args from another cwd would load another e2e.config.ts. + const elsewhere = launch(OK, { cwd: path.resolve(BUGBASH_DIR, "../..") }); + expect([elsewhere.status, elsewhere.log.includes("CALL run")]).toEqual([2, false]); +}); + +test("without the sandbox the launcher refuses: no host fallback", () => { + // The real app AI needs the provider proxy: refused before any docker call. + const real = launch(OK, { realAi: true }); + expect([real.status, real.log]).toEqual([2, ""]); + expect(real.stderr).toContain("sandbox only"); + // No usable Docker, or a failed same-host probe: refused, and no job runs. + for (const options of [{ env: { DOCKER_CONTEXT: "missing" } }, { probeFails: true }]) { + const r = launch(OK, options); + expect([r.status, r.log.match(/CALL run/g)?.length ?? 0]).toEqual([ + 2, + options.probeFails === true ? 1 : 0, + ]); + expect(r.stderr).toContain("refused:"); + } +}); + +test("the image builds from the committed Dockerfile on stdin, with no context and one build arg", () => { + const r = launch(OK, { build: true }); + expect(r.status).toBe(0); + const build = r.log.split("\n").find((line) => line.startsWith("CALL build")); + expect(build).toMatch( + /^CALL build --build-arg PLAYWRIGHT_CORE_VERSION=\d+\.\d+\.\d+\S* -t xum-bugbash-sandbox:[0-9a-f]{12} -$/ + ); + const committed = spawnSync("git", ["show", "HEAD:tests/bugbash/sandbox/Dockerfile"], { + cwd: BUGBASH_DIR, + encoding: "utf8", + }); + expect(committed.status).toBe(0); + expect(r.dockerfile).toBe(committed.stdout); +}); + +const CHECKOUT_ID = crypto + .createHash("sha256") + .update(path.resolve(BUGBASH_DIR, "../..")) + .digest("hex") + .slice(0, 12); + +test.each([ + ["stage", "SIGINT", 130], + ["stage", "SIGTERM", 143], + ["probe", "SIGINT", 130], + ["probe", "SIGTERM", 143], +] as const)( + "a %s step stopped by %s removes this job's folder only", + async (slow, signal, code) => { + const options: Launch = { slow }; + const { bin, calls } = fakeBin(options); + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "xbb-launch-test-")); + // Another job of this checkout, and a job of another checkout: both must stay. + const jobs = path.join(tmp, "xum-bugbash-sandbox"); + for (const other of [`${CHECKOUT_ID}/xbb-other-job`, "0123456789ab/xbb-another-checkout"]) { + fs.mkdirSync(path.join(jobs, other), { recursive: true }); + fs.writeFileSync(path.join(jobs, other, "keep"), ""); + } + const listJobs = () => + fs + .readdirSync(jobs, { recursive: true }) + .map(String) + .filter((entry) => entry.split("/").length === 2) + .sort(); + try { + const child = Bun.spawn( + [process.execPath, path.join(import.meta.dir, "launch.ts"), "--", ...OK], + { + cwd: BUGBASH_DIR, + env: { ...launchEnv(bin, options), TMPDIR: tmp }, + stderr: "pipe", + } + ); + const deadline = Date.now() + 15_000; + while (!fs.existsSync(`${calls}.ready`) && Date.now() < deadline) await Bun.sleep(50); + expect(fs.existsSync(`${calls}.ready`)).toBe(true); + // The launcher's own folder exists while the step runs. + expect(listJobs().length).toBe(3); + child.kill(signal); + expect(await child.exited).toBe(code); + expect(await new Response(child.stderr).text()).toContain(`stopped by ${signal}`); + expect(listJobs()).toEqual( + [`${CHECKOUT_ID}/xbb-other-job`, "0123456789ab/xbb-another-checkout"].sort() + ); + expect(fs.readdirSync(tmp).sort()).toEqual(["xum-bugbash-sandbox"]); // no docker client folder + // The job container never started. + expect(fs.readFileSync(calls, "utf8").match(/CALL run/g)?.length ?? 0).toBe( + slow === "probe" ? 1 : 0 + ); + } finally { + fs.rmSync(bin, { recursive: true, force: true }); + fs.rmSync(tmp, { recursive: true, force: true }); + fs.rmSync(path.join(BUGBASH_DIR, ".e2e/launch-test"), { recursive: true, force: true }); + } + }, + 30_000 +); + +/** The calls in a fake docker log, one record per call. */ +function dockerCalls(log: string) { + return log + .split("CALL ") + .slice(1) + .map((block) => { + const line = (key: string) => + block + .split("\n") + .find((l) => l.startsWith(`${key} `)) + ?.slice(key.length + 1) ?? ""; + const [host, , config] = line("HOST").split(" "); + return { + command: block.split("\n")[0], + env: line("ENV").trim(), + host, + config, + files: line("FILES").trim(), + marker: line("MARKER"), + }; + }) + .filter((call) => call.command !== ""); +} + +/** A client config with an authenticated proxy: the CLI would copy it into builds and containers. */ +function syntheticDockerConfig(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "xbb-launch-test-")); + const proxy = "http://user:MARKER-secret@127.0.0.1:9"; + const config = { proxies: { default: { httpProxy: proxy, httpsProxy: proxy } }, auths: {} }; + fs.writeFileSync(path.join(dir, "config.json"), JSON.stringify(config)); + return dir; +} + +test("docker commands never see the user's client config, HOME or other DOCKER_* values", () => { + const userConfig = syntheticDockerConfig(); + try { + const r = launch(OK, { + build: true, + env: { + DOCKER_CONFIG: userConfig, + DOCKER_CERT_PATH: userConfig, + DOCKER_TLS_VERIFY: "1", + }, + }); + expect(r.status).toBe(0); + const calls = dockerCalls(r.log); + expect(calls.map((call) => call.command.split(" ")[0])).toEqual([ + "context", // the only call that reads the user's config: the context selection + "info", + "image", + "build", + "run", // the same-host probe + "run", // the job + "ps", // cleanup: found, removed, gone + "rm", + "ps", + ]); + expect(calls[0].config).toBe(userConfig); + const privateDirs = new Set(calls.slice(1).map((call) => call.config)); + expect(privateDirs.size).toBe(1); + const [privateDir] = privateDirs; + for (const call of calls.slice(1)) + expect({ ...call, command: "" }).toEqual({ + command: "", + env: "DOCKER_CONFIG DOCKER_HOST PATH", + host: "unix:///var/run/docker.sock", + config: privateDir, + files: "", + marker: "0 0", + }); + expect(privateDir.startsWith(os.tmpdir())).toBe(true); + expect(fs.existsSync(privateDir)).toBe(false); // removed when the launcher exited + expect(r.stderr).not.toContain("MARKER"); + } finally { + fs.rmSync(userConfig, { recursive: true, force: true }); + } +}); + +test("the selected context is resolved once; a remote one is refused, not swapped for the default", () => { + const other = dockerCalls(launch(OK, { env: { DOCKER_CONTEXT: "other" } }).log); + expect(other.slice(1).map((call) => [call.host, call.env])).toEqual( + other.slice(1).map(() => ["unix:///run/other.sock", "DOCKER_CONFIG DOCKER_HOST PATH"]) + ); + for (const env of [ + { DOCKER_CONTEXT: "remote" } as Record, + { DOCKER_HOST: "tcp://10.0.0.1:2376" }, + { DOCKER_HOST: "ssh://user@host" }, + { DOCKER_HOST: "unix://run/relative.sock" }, + ]) { + const refused = launch(OK, { env }); + expect([refused.status, dockerCalls(refused.log).map((c) => c.command.split(" ")[0])]).toEqual([ + 2, + ["context"], + ]); + expect(refused.stderr).toContain("not a local socket"); + } +}); diff --git a/tests/bugbash/sandbox/launch.ts b/tests/bugbash/sandbox/launch.ts new file mode 100644 index 0000000000..5c3fce983c --- /dev/null +++ b/tests/bugbash/sandbox/launch.ts @@ -0,0 +1,523 @@ +/** + * Runs one bug-bash e2e job in a disposable container: the bug-bash sandbox (#5714). + * Usage, from tests/bugbash: + * bun sandbox/launch.ts -- run --config e2e.config.ts --output .e2e/ [e2e args...] + * + * The container runs the e2e CLI, Chromium and the seeded app. It gets no network, no + * capabilities, a read-only root, copies of the git-listed inputs, and read-only dist/ and + * node_modules/. Its output comes back on its stdout as an export stream (exportStream.ts). + * + * It runs only exact-step jobs: `e2e run --config e2e.config.ts` (the repros), with the mock + * app AI. Other e2e commands and configs (`explore`, the MCP Apps suite) let a model pick the + * actions, and the real app AI calls a provider. Both need the sandbox's provider proxy (a later + * step of #5714), so the launcher refuses them for now. + * It launches containers only. It has no host fallback: without usable Docker it refuses. The + * make targets keep their own host path (no change in this step). + * Exit codes: the job's code, 2 when the launcher refuses, 4 when the evidence is incomplete, + * 130 or 143 when SIGINT or SIGTERM stopped it. + */ +import { spawn, spawnSync } from "child_process"; +import * as crypto from "crypto"; +import * as fs from "fs"; +import { createRequire } from "module"; +import * as os from "os"; +import * as path from "path"; +import { receiveExport } from "./exportStream"; + +const ROOT = path.resolve(import.meta.dir, "../../.."); +const BUGBASH_DIR = path.join(ROOT, "tests/bugbash"); +const DEADLINE_MS = 30 * 60_000; +// The container reads copies of these git-listed inputs. A symlink stops the launch. +const INPUTS = ["src", "tests/bugbash", "tsconfig.json", "package.json"]; +// The host env names that e2e.config.ts and startApp.ts read. No other host value passes. +// Not BUGBASH_AI_REASON: after a failed probe it holds the provider URL and response text. +const PASS_ENV = [ + "BUGBASH_AI", + "BUGBASH_AI_RESOLVED", + "BUGBASH_APP_MODEL", + "BUGBASH_MODEL", + "BUGBASH_EFFORT", + "BUGBASH_APP_LOG", + "BUGBASH_SCENARIO", + "E2E_TELEMETRY_DISABLED", +]; + +export class Refusal extends Error {} +/** SIGINT or SIGTERM stopped the launcher before the job's container started. */ +class Stopped extends Error { + constructor(readonly signal: NodeJS.Signals) { + super(`stopped by ${signal}`); + } +} +const log = (message: string) => console.error(`sandbox ${message}`); +const sha = (text: string) => crypto.createHash("sha256").update(text).digest("hex"); + +/** + * The env of every docker command after checkEndpoint(): PATH, the endpoint that the user + * selected, and an empty private client config. The CLI reads no user config. Its `proxies` + * entries (a proxy URL can hold a user and password) would otherwise go into every build as + * build args and into every container as env, past containerEnv(). No HOME, no other DOCKER_*. + */ +let client: Record | null = null; + +function clientEnv(): Record { + if (client == null) throw new Refusal("docker: checkEndpoint() must pass first"); + return client; +} + +/** + * The endpoint of the user's docker CLI: DOCKER_HOST, else DOCKER_CONTEXT, else the current + * context in the user's client config. This is the one docker command that reads the user's + * config. It reads the context only, and starts no build and no container. + */ +function selectedEndpoint(): { host: string } | { error: string } { + const pass = ["PATH", "HOME", "DOCKER_HOST", "DOCKER_CONTEXT", "DOCKER_CONFIG"]; + const env: Record = {}; + for (const [key, value] of Object.entries(process.env)) + if (pass.includes(key) && value != null) env[key] = value; + const r = run(env, ["context", "inspect", "--format", "{{.Endpoints.docker.Host}}"], { + timeoutMs: 10_000, + }); + return r.ok ? { host: r.stdout } : { error: r.error }; +} + +/** A fresh, empty client config folder. It lives until this launcher exits. */ +function privateClient(host: string): Record { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "xum-bugbash-docker-")); + process.on("exit", () => fs.rmSync(dir, { recursive: true, force: true })); + return { PATH: process.env.PATH ?? "", DOCKER_HOST: host, DOCKER_CONFIG: dir }; +} + +function docker(args: string[], options: { timeoutMs: number; input?: string; quiet?: boolean }) { + return run(clientEnv(), args, options); +} + +function run( + env: Record, + args: string[], + options: { timeoutMs: number; input?: string; quiet?: boolean } +) { + const r = spawnSync("docker", args, { + env, + encoding: "utf8", + timeout: options.timeoutMs, + input: options.input, + stdio: [ + options.input == null ? "ignore" : "pipe", + options.quiet === false ? 2 : "pipe", + "pipe", + ], + }); + const error = r.error?.message ?? (r.status === 0 ? "" : (r.stderr || `exit ${r.status}`).trim()); + return { ok: error === "", stdout: (r.stdout ?? "").trim(), error }; +} + +/** Why this host cannot run the sandbox, or null. It runs before any image build. */ +export function checkEndpoint(): string | null { + if (process.platform !== "linux") return `${process.platform}: the sandbox needs Linux`; + if (process.getuid?.() === 0) return "the sandbox does not run as root"; + if (client != null) return null; // resolved and checked once per launcher + const selected = selectedEndpoint(); + if ("error" in selected) return `docker: ${selected.error}`; + // Fail closed: a remote, ssh or relative endpoint is refused, never swapped for the default. + if (!/^unix:\/\/\/./.test(selected.host)) + return `docker endpoint ${JSON.stringify(selected.host)}: not a local socket`; + const candidate = privateClient(selected.host); + const info = run(candidate, ["info", "--format", "{{json .}}"], { timeoutMs: 15_000 }); + if (!info.ok) return `docker info: ${info.error}`; + const daemon = JSON.parse(info.stdout) as { + OSType?: string; + OperatingSystem?: string; + SecurityOptions?: string[]; + ServerErrors?: string[]; + }; + // `docker info` exits 0 when no daemon answers, with only the client fields. + if ((daemon.OSType ?? "") === "") + return `docker info: no daemon answered (${(daemon.ServerErrors ?? []).join("; ")})`; + if (daemon.OSType !== "linux" || /docker desktop/i.test(daemon.OperatingSystem ?? "")) + return "Docker Desktop is not supported"; + if ((daemon.SecurityOptions ?? []).some((o) => o.includes("rootless"))) + return "rootless Docker is not supported"; + client = candidate; + return null; +} + +const DOCKERFILE = "tests/bugbash/sandbox/Dockerfile"; + +/** + * The image build runs outside the job sandbox: on the daemon, with the default build network + * (apt and the Chromium download need it). So its inputs come only from reviewed harness source: + * - the Dockerfile as committed at HEAD. A work-tree change refuses: commit it first. + * - no build context: `docker build -` with the Dockerfile on stdin sends no folder, so no + * demo repo, no .e2e evidence and no file that a job wrote can reach the build. + * - one build arg, the Playwright version of the installed @e2e-dev/web (from bun.lock). No + * build arg comes from the env, and the private client config (clientEnv) adds no proxies. + */ +function buildInputs(): { dockerfile: string; playwright: string } { + const committed = spawnSync("git", ["-C", ROOT, "show", `HEAD:${DOCKERFILE}`], { + encoding: "utf8", + }); + if (committed.status !== 0) throw new Refusal(`git show HEAD:${DOCKERFILE}: ${committed.stderr}`); + const st = fs.lstatSync(path.join(ROOT, DOCKERFILE), { throwIfNoEntry: false }); + if ( + st?.isFile() !== true || + fs.readFileSync(path.join(ROOT, DOCKERFILE), "utf8") !== committed.stdout + ) + throw new Refusal( + `${DOCKERFILE} differs from HEAD: the image builds only from committed source` + ); + // The Chromium build that the checkout's @e2e-dev/web expects. + const web = createRequire(path.join(ROOT, "package.json")).resolve("@e2e-dev/web"); + const playwright = (createRequire(web)("playwright-core/package.json") as { version: string }) + .version; + if (!/^\d+\.\d+\.\d+(-[\w.]+)?$/.test(playwright)) + throw new Refusal(`playwright-core version ${JSON.stringify(playwright)}: not a plain version`); + return { dockerfile: committed.stdout, playwright }; +} + +function ensureImage(): string { + const { dockerfile, playwright } = buildInputs(); + const image = `xum-bugbash-sandbox:${sha(`${dockerfile}\0${playwright}`).slice(0, 12)}`; + if (docker(["image", "inspect", image], { timeoutMs: 15_000 }).ok) return image; + log(`building ${image}`); + const args = ["build", "--build-arg", `PLAYWRIGHT_CORE_VERSION=${playwright}`, "-t", image, "-"]; + const built = docker(args, { timeoutMs: 20 * 60_000, input: dockerfile, quiet: false }); + if (!built.ok) throw new Refusal(`image build: ${built.error}`); + return image; +} + +/** + * Checks each folder of `rel` under `base` without following a symlink; with `create`, it makes + * the missing ones. So a symlinked folder cannot lead a copy or an export out of the checkout. + */ +export function plainFolders(base: string, rel: string, create: boolean, seen = new Set()) { + let dir = base; + for (const part of rel.split("/").filter((p) => p !== "" && p !== ".")) { + dir = path.join(dir, part); + if (seen.has(dir)) continue; + const st = fs.lstatSync(dir, { throwIfNoEntry: false }); + if (st == null && create) fs.mkdirSync(dir); + else if (st?.isDirectory() !== true) + throw new Refusal(`${path.relative(base, dir)}: not a plain folder (a symlink?)`); + seen.add(dir); + } +} + +function stage(into: string): number { + // Tracked files, and new files that git does not ignore (a new repro). Ignored files stay out: + // old .e2e runs, app logs and local env files. + // prettier-ignore + const listArgs = ["ls-files", "-z", "--cached", "--others", "--exclude-standard", "--deduplicate"]; + const listed = spawnSync("git", ["-C", ROOT, ...listArgs, "--", ...INPUTS], { + encoding: "utf8", + maxBuffer: 64 << 20, + }); + if (listed.status !== 0) throw new Refusal(`git ls-files: ${listed.stderr}`); + let count = 0; + const seen = new Set(); + for (const rel of listed.stdout.split("\0").filter((name) => name !== "")) { + const st = fs.lstatSync(path.join(ROOT, rel), { throwIfNoEntry: false }); + if (st == null) continue; // deleted in the work tree + plainFolders(ROOT, path.dirname(rel), false, seen); // lstat above follows symlinked folders + if (!st.isFile()) throw new Refusal(`stage: ${rel} is not a regular file`); + fs.mkdirSync(path.join(into, path.dirname(rel)), { recursive: true }); + fs.copyFileSync(path.join(ROOT, rel), path.join(into, rel), fs.constants.COPYFILE_EXCL); + count += 1; + } + // Mount points for the read-only build outputs and the job's tmpfs. + for (const dir of ["dist", "node_modules", "tests/bugbash/.e2e"]) + fs.mkdirSync(path.join(into, dir), { recursive: true }); + return count; +} + +// --mount, not -v: a missing source fails instead of creating an empty folder. +function bind(src: string, dst: string): string[] { + if (`${src}${dst}`.includes(",")) throw new Refusal(`a comma in a mount path: ${src}`); + return ["--mount", `type=bind,src=${src},dst=${dst},readonly`]; +} + +/** Same boot id: the daemon shares this kernel. Same nonce: it sees this host's files at these paths. */ +function checkSameHost(image: string, stageDir: string): string | null { + const nonce = crypto.randomUUID(); + fs.writeFileSync(path.join(stageDir, ".nonce"), nonce, { flag: "wx" }); + // prettier-ignore + const probe = ["run", "--rm", "--network", "none", ...bind(stageDir, "/probe"), image, + "cat", "/proc/sys/kernel/random/boot_id", "/probe/.nonce"]; + const r = docker(probe, { timeoutMs: 60_000 }); + if (!r.ok) return `probe container: ${r.error}`; + const [boot, seen] = r.stdout.split("\n"); + if (boot !== fs.readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim()) + return "the daemon runs on another kernel"; + return seen === nonce ? null : "the daemon sees other files at these paths"; +} + +/** boot id : PID namespace : PID : start time of this launcher. A later sweep reads it. */ +function ownerLabel(): string { + const boot = fs.readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim(); + const pidns = fs.readlinkSync("/proc/self/ns/pid").replace(/\D/g, ""); + const stat = fs.readFileSync(`/proc/${process.pid}/stat`, "utf8"); + return `${boot}:${pidns}:${process.pid}:${stat.slice(stat.lastIndexOf(")") + 2).split(" ")[19]}`; +} + +// No credential reaches the container, also not through a caller's extra values. +const CREDENTIAL = /(_API_KEY|_AUTH_TOKEN|_TOKEN|_BASE_URL|_SECRET|_PASSWORD)$/i; + +/** The container env: fixed values, the allowlisted host names and the caller's extra values. */ +export function containerEnv( + host: NodeJS.ProcessEnv, + extra: Record +): Record { + const env: Record = { HOME: "/home/bugbash", TMPDIR: "/tmp" }; + for (const key of PASS_ENV) if (host[key] != null) env[key] = host[key]; + Object.assign(env, extra, { BUGBASH_CONTAINER: "1" }); + for (const key of Object.keys(env)) + if (CREDENTIAL.test(key)) throw new Refusal(`${key}: no credential enters the sandbox`); + return env; +} + +export interface SandboxRun { + command: string[]; + /** The job's output folder, relative to tests/bugbash. It comes back to the same host path. */ + exportDir: string; + env?: Record; +} + +export async function runInSandbox(run: SandboxRun): Promise { + const dest = path.join(BUGBASH_DIR, run.exportDir); + if (fs.existsSync(dest)) throw new Refusal(`${run.exportDir} exists: remove it first`); + const image = ensureImage(); + const checkoutId = sha(ROOT).slice(0, 12); + const name = `xbb-${checkoutId.slice(0, 6)}-${crypto.randomBytes(3).toString("hex")}`; + const jobDir = path.join(os.tmpdir(), "xum-bugbash-sandbox", checkoutId, name); + // Cleanup has an owner before the job folder exists. A signal handler stops the default exit, + // which would skip the `finally` below. Staging and the probe are synchronous, and a handler + // runs only from the event loop, so each checkpoint first yields to it (measured: without the + // yield, the signal was seen only once the container had started). Only this job's folder + // goes: the folder is created without `recursive`, so it is ours or the launch fails. + let signal: NodeJS.Signals | null = null; + let onStop: ((reason: string) => void) | null = null; + const onSignal = (received: NodeJS.Signals) => { + signal ??= received; + onStop?.(received); + }; + const checkpoint = async () => { + await new Promise((resolve) => setImmediate(resolve)); + if (signal != null) throw new Stopped(signal); + }; + process.on("SIGINT", onSignal).on("SIGTERM", onSignal); + try { + fs.mkdirSync(path.dirname(jobDir), { recursive: true, mode: 0o700 }); + fs.mkdirSync(jobDir, { mode: 0o700 }); + } catch (error) { + process.off("SIGINT", onSignal).off("SIGTERM", onSignal); + throw error; + } + try { + const stageDir = path.join(jobDir, "stage"); + const started = Date.now(); + const files = stage(stageDir); + await checkpoint(); + log(`${name} staged ${files} files in ${((Date.now() - started) / 1000).toFixed(2)} s`); + const different = checkSameHost(image, stageDir); + if (different != null) throw new Refusal(different); + const uid = process.getuid?.() ?? 1000; + const gid = process.getgid?.() ?? 1000; + fs.writeFileSync( + path.join(jobDir, "passwd"), + `root:x:0:0::/root:/usr/sbin/nologin\nbugbash:x:${uid}:${gid}::/home/bugbash:/bin/sh\n` + ); + fs.writeFileSync(path.join(jobDir, "group"), `root:x:0:\nbugbash:x:${gid}:\n`); + const env = containerEnv(process.env, run.env ?? {}); + const owner = ownerLabel(); + // prettier-ignore + const args = ["run", "--rm", "-i", "--init", "--name", name, + "--label", `xum.bugbash.checkout=${checkoutId}`, "--label", `xum.bugbash.owner=${owner}`, + "--log-driver", "none", "--network", "none", "--user", `${uid}:${gid}`, + "--cap-drop", "ALL", "--security-opt", "no-new-privileges", "--read-only", + "--pids-limit", "4096", "--memory", "4g", "--memory-swap", "4g", + "--tmpfs", "/tmp:rw,nosuid,nodev,size=4g", "--tmpfs", "/home/bugbash:rw,nosuid,nodev,size=1g", + "--tmpfs", `/repo/tests/bugbash/.e2e:rw,nosuid,nodev,size=1g,uid=${uid},gid=${gid}`, + ...bind(stageDir, "/repo"), ...bind(path.join(ROOT, "dist"), "/repo/dist"), + ...bind(path.join(ROOT, "node_modules"), "/repo/node_modules"), + ...bind(path.join(jobDir, "passwd"), "/etc/passwd"), ...bind(path.join(jobDir, "group"), "/etc/group"), + ...Object.entries(env).flatMap(([key, value]) => ["-e", `${key}=${value}`]), + "-w", "/repo/tests/bugbash", "--entrypoint", "bun", image, + "sandbox/entry.ts", "--export", run.exportDir, "--", ...run.command]; + plainFolders(BUGBASH_DIR, path.dirname(run.exportDir), true); + log(`${name} --network none, ${appAi() ?? "no"} app AI, no proxy`); + await checkpoint(); + const job = { name, owner, checkout: checkoutId, dest }; + return await runContainer(args, job, (stop) => (onStop = stop)); + } finally { + process.off("SIGINT", onSignal).off("SIGTERM", onSignal); + fs.rmSync(jobDir, { recursive: true, force: true }); + } +} + +interface Job { + name: string; + owner: string; + checkout: string; +} + +async function runContainer( + args: string[], + job: Job & { dest: string }, + onSignal: (stop: (reason: string) => void) => void +) { + // The lifeline: this process holds the container's stdin. When it dies, the pipe closes and + // entry.ts stops the job (measured: the container was gone 0.59 s after a SIGKILL). + const child = spawn("docker", args, { env: clientEnv(), stdio: ["pipe", "pipe", "inherit"] }); + child.stdin.on("error", () => undefined); + const exported = receiveExport(child.stdout, job.dest); + let stopped: string | null = null; + const stop = (reason: string) => { + if (stopped != null) return; + stopped = reason; + log(`${job.name} stopping: ${reason}`); + log(`${job.name} ${removeContainer(job)}`); + // Also when the removal failed: the closed lifeline stops the job in the container, and the + // killed client ends the wait below, so the deadline always bounds this launcher. + child.stdin.end(); + child.kill("SIGKILL"); + }; + const timer = setTimeout(() => stop("the 30 min deadline"), DEADLINE_MS); + onSignal(stop); // runInSandbox owns the signal handlers + const code = await new Promise((resolve) => { + child + .on("error", () => resolve(125)) + .on("exit", (exit, signal) => + resolve(exit == null && signal == null ? 125 : exitCode(exit, signal)) + ); + }); + clearTimeout(timer); + const result = await exported; + const size = `${result.files} files, ${(result.bytes / 1e6).toFixed(1)} MB`; + log( + `${job.name} job exit ${code}, export ${size}, ${result.complete ? "complete" : `incomplete: ${result.error}`}` + ); + log(`${job.name} ${removeContainer(job)}`); + if (stopped != null) return stopped === "SIGINT" ? 130 : 143; + return result.complete ? code : 4; +} + +/** + * Removes the job's container. It matches the name, the owner label AND this checkout's label, + * never the name alone: other checkouts on this host run their own sandboxes. + */ +export function removeContainer(job: Job): string { + // prettier-ignore + const filters = ["--filter", `name=^/${job.name}$`, "--filter", `label=xum.bugbash.owner=${job.owner}`, + "--filter", `label=xum.bugbash.checkout=${job.checkout}`]; + const find = () => docker(["ps", "-aq", "--no-trunc", ...filters], { timeoutMs: 15_000 }); + const found = find(); + if (!found.ok) return `container state unknown: ${found.error}`; + if (found.stdout === "") return "removed"; + docker(["rm", "-f", found.stdout], { timeoutMs: 30_000 }); + const after = find(); + if (!after.ok) return `container state unknown: ${after.error}`; + return after.stdout === "" ? "removed" : `still present: docker rm -f ${found.stdout}`; +} + +/** The app AI mode, read the way e2e.config.ts reads it. */ +export const appAi = (env: NodeJS.ProcessEnv = process.env) => + env.BUGBASH_AI_RESOLVED ?? (env.BUGBASH_AI === "mock" ? "mock" : undefined); + +/** Why this job cannot run in the sandbox, or null. There is no host fallback. */ +function sandboxRefusal(): string | null { + if (appAi() !== "mock") + return "the real app AI needs the sandbox's provider proxy, which is not built yet (#5714)"; + return checkEndpoint(); +} + +/** The shell convention: the exit code, or 128 + the number of the signal that ended the process. */ +export function exitCode(code: number | null, signal: NodeJS.Signals | null): number { + return code ?? (signal != null ? 128 + os.constants.signals[signal] : 1); +} + +// The `e2e run` options that a repro run may use: selection and output only. Not allowed, among +// others: a second --config, positional files, "--", --agent and the cache and trace switches. +// e2e reads no env var that picks a config or an agent (only E2E_TELEMETRY_*, the E2E_USER*, +// E2E_SECRET* and E2E_OAUTH_CREDENTIALS test values, NODE_OPTIONS and CI names; e2e 0.17). +const RUN_VALUE_OPTIONS = ["--config", "--output", "--tag", "--exclude-tag", "--tag-mode", + "--grep", "--grep-invert", "--target", "--shard", "--workers", "--retries", "--max-failures", + "--reporter"]; // prettier-ignore +const RUN_FLAGS = ["--pass-with-no-tests", "--last-failed", "--debug"]; + +/** + * Why these e2e args are not an exact-step repro run, or null. Only `e2e run` with the repro + * config passes: its tests are repros/**. A repro that took the agent fixture anyway would find + * no model: the container has no network and no provider key. e2e resolves --config from its + * cwd, so the cwd must be tests/bugbash and the config a regular file there. + */ +export function exactStepRefusal(args: string[], cwd: string, dir = BUGBASH_DIR): string | null { + if (args[0] !== "run") return `${JSON.stringify(args[0] ?? "")} is not \`e2e run\``; + if (args.includes("explore")) return "an `explore` argument"; + const configs: string[] = []; + for (let i = 1; i < args.length; i++) { + const [name, inline] = args[i].startsWith("--") ? args[i].split(/=(.*)/s, 2) : [args[i]]; + if (RUN_FLAGS.includes(name) && inline == null) continue; + if (!RUN_VALUE_OPTIONS.includes(name)) return `the argument ${JSON.stringify(args[i])}`; + const value = inline ?? args[++i]; + if (value == null || value === "" || value.startsWith("-")) + return `${name} needs a value, got ${JSON.stringify(value ?? "")}`; + if (name === "--config") configs.push(value); + } + if (configs.length !== 1 || configs[0] !== "e2e.config.ts") + return `--config must be given once as e2e.config.ts, got ${JSON.stringify(configs)}`; + if (fs.realpathSync(cwd) !== fs.realpathSync(dir)) return `the cwd must be ${dir}, got ${cwd}`; + if (fs.lstatSync(path.join(dir, "e2e.config.ts"), { throwIfNoEntry: false })?.isFile() !== true) + return `${dir}/e2e.config.ts is not a regular file (a symlink?)`; + return null; +} + +/** The one `--output .e2e/` of the e2e args. The export comes back to that folder only. */ +export function outputDir(args: string[]): string { + const values = args.flatMap((arg, i) => + arg === "--output" ? [args[i + 1] ?? ""] : arg.startsWith("--output=") ? [arg.slice(9)] : [] + ); + const [dir] = values; + if ( + values.length !== 1 || + !/^\.e2e(\/[\w.-]+)+$/.test(dir) || + dir.split("/").some((part) => /^\.+$/.test(part) && part !== ".e2e") + ) + throw new Refusal( + `the e2e args need exactly one --output .e2e/, got ${JSON.stringify(values)}` + ); + return dir; +} + +async function main(): Promise { + // bun removes the first "--" after the script; a launcher started another way keeps it. + const given = process.argv.slice(2); + const e2eArgs = given[0] === "--" ? given.slice(1) : given; + if (e2eArgs.length === 0) throw new Refusal("usage: launch.ts -- "); + const notExact = exactStepRefusal(e2eArgs, process.cwd()); + if (notExact != null) + throw new Refusal( + `${notExact}: only \`e2e run --config e2e.config.ts\` (exact-step repros) runs for now. ` + + "Model-driven runs wait for the sandbox's provider proxy (#5714)." + ); + const output = outputDir(e2eArgs); + const why = sandboxRefusal(); + if (why != null) throw new Refusal(`${why}. The launcher runs jobs in the sandbox only.`); + const command = ["node", "../../node_modules/e2e/dist/cli/bin.js", ...e2eArgs]; + // The app log goes into the output folder, so that it comes back with the report. + const env = { BUGBASH_APP_LOG: process.env.BUGBASH_APP_LOG ?? `${output}/app.log` }; + return runInSandbox({ command, exportDir: output, env }); +} + +if (import.meta.main) { + main().then( + (code) => process.exit(code), + (error: unknown) => { + if (error instanceof Stopped) { + log(error.message); + process.exit(exitCode(null, error.signal)); + } + log(`refused: ${error instanceof Error ? error.message : String(error)}`); + process.exit(error instanceof Refusal ? 2 : 1); + } + ); +}