diff --git a/Makefile b/Makefile index 8537ac476f6..6e708e37002 100644 --- a/Makefile +++ b/Makefile @@ -219,8 +219,8 @@ dev-desktop-sandbox: ## Start an isolated Electron dev instance (fresh XUM_ROOT dev-server-sandbox: ## Start an isolated dev-server instance (fresh XUM_ROOT + free ports) @bun scripts/dev-server-sandbox.ts $(DEV_SERVER_SANDBOX_ARGS) -# bug-bash and mcp-apps-e2e are paused on the host (tests/bugbash/hostPause.ts, #5714): a model -# picks their actions. Their recipes refuse at once, so they build nothing first. +# bug-bash is paused on the host (tests/bugbash/hostPause.ts, #5714): a model picks its actions. +# Its recipe refuses at once, so it builds nothing first. bug-bash: ## Agent bug bash: e2e explore charters x models (paused on the host until the sandbox lands, #5714) @bun tests/bugbash/run.ts $(BUGBASH_ARGS) @@ -261,11 +261,14 @@ test-bugbash-repros: build-main build-renderer build-static ## Bug-bash repro te test-bugbash-known-failures: build-main build-renderer build-static ## Bug-bash repros of open bugs: each fails until its issue is fixed @export BUGBASH_AI_RESOLVED=mock BUGBASH_AI_REASON="known-failure repros"; $(BUGBASH_REPRO_RUN) --tag known-failure --output .e2e/repros-known $(BUGBASH_REPRO_ARGS) -# e2e needs Node 22.22.3+/24.8+: E2E_NODE, else the first node on PATH. Its directory leads PATH so -# the app command and its children use the same node (as tests/bugbash/run.ts does). -mcp-apps-e2e: ## MCP Apps e2e suite, agent.act driven (paused on the host until the sandbox lands, #5714) - @# Its seed writes the MCP chat directly, so it runs on the mock app AI (tests/bugbash/aiMode.ts). - @export BUGBASH_AI_RESOLVED=mock BUGBASH_AI_REASON="MCP Apps suite"; node="$${E2E_NODE:-$$(command -v node)}"; cd tests/bugbash && PATH="$$(dirname "$$node"):$$PATH" E2E_TELEMETRY_DISABLED=1 "$$node" ../../node_modules/e2e/dist/cli/bin.js run --config e2e.mcpapps.config.ts $(MCP_APPS_E2E_ARGS) +# The MCP Apps suite runs only in the bug-bash sandbox (tests/bugbash/sandbox/launch.ts, #5714). +# agent.act steps reach BUGBASH_MODEL through a provider proxy for the job, so the host needs +# BUGBASH_BUDGET_USD (a list-price cap) and ANTHROPIC_API_KEY plus ANTHROPIC_BASE_URL. Its seed +# writes the MCP chat directly, so the app AI stays the mock. The container mounts dist/. +mcp-apps-e2e: build-main build-renderer build-static ## MCP Apps e2e suite in the bug-bash sandbox (needs BUGBASH_BUDGET_USD) + @# The app AI is pinned to the mock, also over an exported BUGBASH_AI_RESOLVED. The shell PID + @# keeps two runs that start in the same second apart (their output and proxy record). + @cd tests/bugbash && BUGBASH_AI=mock BUGBASH_AI_RESOLVED=mock BUGBASH_AI_REASON="MCP Apps suite" bun sandbox/launch.ts -- run --config e2e.mcpapps.config.ts --output .e2e/mcp-apps-$$(date -u +%Y%m%dT%H%M%SZ)-$$$$ $(MCP_APPS_E2E_ARGS) rlm-eval: ## Run the RLM lever eval against a running dev-server sandbox (see scripts/rlm-eval/run.ts header) @bun run scripts/rlm-eval/run.ts $(RLM_EVAL_ARGS) diff --git a/tests/bugbash/e2e.mcpapps.config.ts b/tests/bugbash/e2e.mcpapps.config.ts index 27fc4358e7e..7fdfbdc9bab 100644 --- a/tests/bugbash/e2e.mcpapps.config.ts +++ b/tests/bugbash/e2e.mcpapps.config.ts @@ -6,12 +6,29 @@ * --charters tests/bugbash/mcpapps/charters.txt"`. * * Paused on the host (hostPause.ts, #5714): the suite drives every flow with `agent.act`, so a - * model picks its actions. Every e2e command with this config, also `run`, refuses as it loads. + * model picks its actions. Every e2e command with this config, also `run`, refuses as it loads, + * except inside the bug-bash sandbox with a provider proxy for the job (`make mcp-apps-e2e`). + * There the agents get BUGBASH_MODEL through that proxy, and the app AI stays the mock. */ // First import: the pause refuses before e2e.config.ts runs (mcpapps/hostPause.ts). import "./mcpapps/hostPause"; +import { createAnthropic } from "@ai-sdk/anthropic"; import type { E2EConfig } from "e2e"; import base from "./e2e.config"; +import { PROXY_BASE_URL } from "./sandbox/inContainer"; + +/** + * The explorer model, set by the launcher. The proxy holds the provider key and allows only + * this job's models (sandbox/proxy.ts), so the key here is a placeholder the proxy drops. + */ +function explorerModel() { + const spec = process.env.BUGBASH_MODEL ?? ""; + const [provider, id] = spec.split(/:(.*)/s, 2); + if (provider !== "anthropic" || !id) + throw new Error(`BUGBASH_MODEL must be anthropic: in the sandbox, got "${spec}"`); + return createAnthropic({ baseURL: PROXY_BASE_URL, apiKey: "bugbash-sandbox-placeholder" })(id); +} +const model = explorerModel(); const mcpContext = [ "MCP Apps setup for this run: the 'Bug bash playground' chat already holds MCP tool calls from", @@ -27,12 +44,12 @@ const mcpContext = [ "Tutorial popovers can cover controls: dismiss them with Skip.", ].join(" "); -const agents = Object.fromEntries( +const agents: E2EConfig["agents"] = Object.fromEntries( Object.entries(base.agents).map(([name, agent]) => [ name, - { ...agent, context: `${agent.context} ${mcpContext}` }, + { ...agent, model, context: `${agent.context} ${mcpContext}` }, ]) -) as typeof base.agents; +); const targets = base.targets.map((target) => ({ ...target, @@ -41,6 +58,10 @@ const targets = base.targets.map((target) => ({ command: { ...target.app.command, args: ["startApp.ts", "--port", "{port}", "--mcp-apps"], + // e2e gives the app only `command.env`, and this config loads only in a model-driven + // sandbox job (mcpapps/hostPause.ts). Without this marker startApp.ts would leave agent + // tools, terminals and project automation on while the explorer drives the app (S3). + env: { ...target.app.command.env, BUGBASH_MODEL_DRIVEN: "1" }, }, }, })) as typeof base.targets; diff --git a/tests/bugbash/hostPause.test.ts b/tests/bugbash/hostPause.test.ts index 88cc742a1ca..042201324e0 100644 --- a/tests/bugbash/hostPause.test.ts +++ b/tests/bugbash/hostPause.test.ts @@ -104,6 +104,12 @@ test.each([ { BUGBASH_AI: "mock" }, ], ["e2e run, MCP Apps", e2e("run", "--config", "e2e.mcpapps.config.ts"), { BUGBASH_AI: "mock" }], + [ + // The sandbox env names alone do not pass: this host has no docker-init and no proxy socket. + "e2e run, MCP Apps, sandbox env on the host", + e2e("run", "--config", "e2e.mcpapps.config.ts"), + { BUGBASH_AI: "mock", BUGBASH_CONTAINER: "1", BUGBASH_MODEL_DRIVEN: "1" }, + ], // No app AI mode: e2e.config.ts would throw its own error first, without the pause. ["e2e list, MCP Apps, no app AI mode", e2e("list", "--config", "e2e.mcpapps.config.ts"), {}], [ diff --git a/tests/bugbash/mcpapps/hostPause.ts b/tests/bugbash/mcpapps/hostPause.ts index 752d0fdd894..d1669b9834c 100644 --- a/tests/bugbash/mcpapps/hostPause.ts +++ b/tests/bugbash/mcpapps/hostPause.ts @@ -2,8 +2,15 @@ * The MCP Apps pause (#5714, ../hostPause.ts). e2e.mcpapps.config.ts imports this module first, * so the refusal comes before e2e.config.ts runs: that config can throw first (for example on * an unresolved app AI mode), and then no message would name the pause. + * + * The suite runs only in the bug-bash sandbox with a provider proxy for the job + * (`make mcp-apps-e2e`, sandbox/launch.ts). Everywhere else it refuses as before. */ import { modelDrivenRefusal } from "../hostPause"; +import { modelDrivenSandbox } from "../sandbox/inContainer"; -const paused = modelDrivenRefusal("the MCP Apps suite (agent.act, e2e.mcpapps.config.ts)"); -if (paused != null) throw new Error(paused); +if (!modelDrivenSandbox()) { + throw new Error( + modelDrivenRefusal("the MCP Apps suite (agent.act, e2e.mcpapps.config.ts)") ?? "refused" + ); +} diff --git a/tests/bugbash/sandbox/entry.ts b/tests/bugbash/sandbox/entry.ts index 2b38d903db7..35ee583e477 100644 --- a/tests/bugbash/sandbox/entry.ts +++ b/tests/bugbash/sandbox/entry.ts @@ -8,13 +8,21 @@ * * Before the job starts, it checks that the daemon runs on the launcher's host: the same kernel * (boot id) and the same files (a nonce that the launcher wrote into the staged copy). + * + * A model-driven job (BUGBASH_MODEL_DRIVEN=1) also needs the job's provider proxy socket, and it + * gets a TCP forwarder to it on 127.0.0.1 (inContainer.ts). The app AI stays the mock. */ import { spawn } from "child_process"; import * as fs from "fs"; import * as os from "os"; import { writeExport } from "./exportStream"; +import { forwardToProxy, inSandbox, modelDrivenSandbox } from "./inContainer"; -assertInSandbox(); // first: on a host, kill(-1) below hits every process of this user +// First: on a host, kill(-1) below hits every process of this user. +if (!inSandbox()) { + console.error("sandbox entry: not in the bug-bash sandbox, so it refuses to run"); + process.exit(2); +} const args = process.argv.slice(2); if (args[0] !== "--export" || args[2] !== "--" || args.length < 4) { @@ -34,6 +42,13 @@ if (boot !== process.env.BUGBASH_HOST_BOOT || nonce !== process.env.BUGBASH_HOST process.exit(2); } const [command, ...commandArgs] = args.slice(3); +if (process.env.BUGBASH_MODEL_DRIVEN === "1") { + if (!modelDrivenSandbox()) { + console.error("sandbox entry: a model-driven job without its proxy socket does not run"); + process.exit(2); + } + await forwardToProxy(); // ends with this process, after the export +} /** Linux skips PID 1 and the caller. When this process exits, docker-init exits too. */ function killAllOthers(): void { @@ -90,15 +105,3 @@ job.on("error", (error) => { job.on("exit", (code, signal) => done(code ?? (signal != null ? 128 + os.constants.signals[signal] : 1)) ); - -function assertInSandbox(): void { - const init = fs.readFileSync("/proc/1/cmdline", "utf8"); - const ok = - process.env.BUGBASH_CONTAINER === "1" && - init.startsWith("/sbin/docker-init\0") && - fs.readdirSync("/sys/class/net").join() === "lo"; - if (!ok) { - console.error("sandbox entry: not in the bug-bash sandbox, so it refuses to run"); - process.exit(2); - } -} diff --git a/tests/bugbash/sandbox/fakeUpstream.ts b/tests/bugbash/sandbox/fakeUpstream.ts index 208ab30bcf4..29fd6b0cb49 100644 --- a/tests/bugbash/sandbox/fakeUpstream.ts +++ b/tests/bugbash/sandbox/fakeUpstream.ts @@ -9,7 +9,15 @@ * `stream: true` request gets a complete SSE stream and any other request a JSON message. * `mode` applies a misbehavior to requests without a marker (the proxy's probe sends none). * `[fake:cut]` on a JSON request sends the headers and part of the body, then drops. + * `[fake:lowered]` and `[fake:nulled]` end a stream normally, but its message_delta lowers or + * nulls a count that message_start reported. + * + * As a command (the zero-cost transport check of the sandbox, plan section 9): + * bun sandbox/fakeUpstream.ts --port 0 --record + * prints `listening on 127.0.0.1:` and appends one line per request: the model, the body + * keys, the betas and the tool and message counts, never the text. */ +import * as fs from "node:fs"; import * as http from "node:http"; import type { AddressInfo } from "node:net"; @@ -30,7 +38,7 @@ export interface FakeRequest { const sse = (type: string, data: object) => `event: ${type}\ndata: ${JSON.stringify({ type, ...data })}\n\n`; -export async function startFakeUpstream() { +export async function startFakeUpstream(port = 0, onRequest?: (request: FakeRequest) => void) { const requests: FakeRequest[] = []; const state: { mode?: string } = {}; const server = http.createServer((req, res) => { @@ -38,7 +46,9 @@ export async function startFakeUpstream() { req.on("data", (chunk: Buffer) => chunks.push(chunk)); req.on("end", () => { const body = Buffer.concat(chunks).toString(); - requests.push({ method: req.method ?? "", url: req.url ?? "", headers: req.headers, body }); + const request = { method: req.method ?? "", url: req.url ?? "", headers: req.headers, body }; + requests.push(request); + onRequest?.(request); const mode = /\[fake:(\w+)\]/.exec(body)?.[1] ?? state.mode; const parsed = JSON.parse(body || "{}") as { model?: string; stream?: boolean }; const model = parsed.model ?? "claude-haiku-4-5"; @@ -78,10 +88,10 @@ export async function startFakeUpstream() { finish(res, mode); }); }); - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - const { port } = server.address() as AddressInfo; + await new Promise((resolve) => server.listen(port, "127.0.0.1", resolve)); + const address = server.address() as AddressInfo; return { - baseUrl: `http://127.0.0.1:${port}`, + baseUrl: `http://127.0.0.1:${address.port}`, requests, setMode: (mode?: string) => (state.mode = mode), close: () => { @@ -103,10 +113,40 @@ function finish(res: http.ServerResponse, mode: string | undefined) { res.write( sse("message_delta", { delta: { stop_reason: "end_turn" }, - usage: { output_tokens: FAKE_USAGE.output_tokens }, + usage: { + output_tokens: FAKE_USAGE.output_tokens, + ...(mode === "lowered" && { input_tokens: 1 }), + // 0 in message_start, so only the null itself is wrong here. + ...(mode === "nulled" && { cache_creation_input_tokens: null }), + }, }) ); if (mode === "cut") return res.destroy(); if (mode === "nostop") return res.end(); res.end(sse("message_stop", {})); } + +if (import.meta.main) { + const flag = (name: string) => process.argv[process.argv.indexOf(name) + 1]; + const record = flag("--record"); + if (!process.argv.includes("--record") || record == null) throw new Error("--record "); + const fake = await startFakeUpstream(Number(flag("--port") ?? 0), (request) => { + let body: Record = {}; + try { + body = JSON.parse(request.body) as typeof body; + } catch { + // recorded as an empty key list + } + const line = { + url: request.url, + model: body.model, + keys: Object.keys(body).sort(), + betas: request.headers["anthropic-beta"] ?? null, + stream: body.stream === true, + tools: Array.isArray(body.tools) ? body.tools.length : 0, + messages: Array.isArray(body.messages) ? body.messages.length : 0, + }; + fs.appendFileSync(record, `${JSON.stringify(line)}\n`); + }); + console.log(`listening on ${fake.baseUrl.replace("http://", "")}`); +} diff --git a/tests/bugbash/sandbox/inContainer.test.ts b/tests/bugbash/sandbox/inContainer.test.ts new file mode 100644 index 00000000000..ad9daf50c13 --- /dev/null +++ b/tests/bugbash/sandbox/inContainer.test.ts @@ -0,0 +1,70 @@ +import { afterEach, expect, test } from "bun:test"; +import * as fs from "node:fs"; +import * as net from "node:net"; +import * as os from "node:os"; +import * as path from "node:path"; +import { forwardToProxy, inSandbox, modelDrivenSandbox, PROXY_SOCKET } from "./inContainer"; + +const cleanups: (() => unknown)[] = []; +afterEach(async () => { + for (const cleanup of cleanups.splice(0).reverse()) await cleanup(); +}); + +/** A fake filesystem root that looks like the sandbox container, with a listening proxy socket. */ +async function fakeRoot(over: { init?: string; devices?: string[]; socket?: boolean } = {}) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "xbb-incontainer-")); + cleanups.push(() => fs.rmSync(root, { recursive: true, force: true })); + fs.mkdirSync(path.join(root, "proc/1"), { recursive: true }); + fs.writeFileSync(path.join(root, "proc/1/cmdline"), over.init ?? "/sbin/docker-init\0--\0bun\0"); + for (const device of over.devices ?? ["lo"]) + fs.mkdirSync(path.join(root, "sys/class/net", device), { recursive: true }); + fs.mkdirSync(path.dirname(path.join(root, PROXY_SOCKET)), { recursive: true }); + if (over.socket !== false) { + const server = net.createServer((socket) => socket.end("proxy says hi")); + await new Promise((resolve) => server.listen(path.join(root, PROXY_SOCKET), resolve)); + cleanups.push(() => new Promise((done) => server.close(done))); + } + return root; +} + +const SANDBOX_ENV = { BUGBASH_CONTAINER: "1", BUGBASH_MODEL_DRIVEN: "1" }; + +test("model-driven jobs pass only in a sandbox container with a proxy socket", async () => { + expect(modelDrivenSandbox(SANDBOX_ENV, await fakeRoot())).toBe(true); + // This host: its PID 1 is no docker-init, and it has other network devices. + expect(inSandbox(SANDBOX_ENV)).toBe(false); + expect(modelDrivenSandbox(SANDBOX_ENV)).toBe(false); + const cases: [Record, Parameters[0]][] = [ + [{ BUGBASH_MODEL_DRIVEN: "1" }, {}], + [{ BUGBASH_CONTAINER: "1" }, {}], + [SANDBOX_ENV, { init: "/sbin/init\0" }], + [SANDBOX_ENV, { devices: ["lo", "eth0"] }], + [SANDBOX_ENV, { socket: false }], + ]; + for (const [env, over] of cases) + expect(modelDrivenSandbox(env, await fakeRoot(over))).toBe(false); +}); + +test("a regular file in place of the proxy socket does not count", async () => { + const root = await fakeRoot({ socket: false }); + fs.writeFileSync(path.join(root, PROXY_SOCKET), ""); + expect(modelDrivenSandbox(SANDBOX_ENV, root)).toBe(false); +}); + +test("the forwarder passes bytes both ways and closes with its server", async () => { + const root = await fakeRoot(); + const server = await forwardToProxy(0, path.join(root, PROXY_SOCKET)); + const { port } = server.address() as net.AddressInfo; + const reply = await new Promise((resolve) => { + let text = ""; + const socket = net.connect(port, "127.0.0.1", () => socket.write("hello")); + socket.on("data", (chunk) => (text += chunk.toString())).on("close", () => resolve(text)); + }); + expect(reply).toBe("proxy says hi"); + await new Promise((done) => server.close(done)); + const after = await new Promise((resolve) => { + const socket = net.connect(port, "127.0.0.1", () => resolve("connected")); + socket.on("error", () => resolve("refused")); + }); + expect(after).toBe("refused"); +}); diff --git a/tests/bugbash/sandbox/inContainer.ts b/tests/bugbash/sandbox/inContainer.ts new file mode 100644 index 00000000000..147eb4cf7a5 --- /dev/null +++ b/tests/bugbash/sandbox/inContainer.ts @@ -0,0 +1,74 @@ +/** + * Where a bug-bash process runs, and its way to the provider proxy (#5714, plan PR B1). + * + * Model-driven jobs (the MCP Apps suite, later `e2e explore`) may run only inside the sandbox + * container, and only with the proxy: hostPause.ts refuses them everywhere else. The launcher + * (launch.ts) mounts the folder of the job's proxy socket read-only at PROXY_DIR and sets + * BUGBASH_MODEL_DRIVEN=1. entry.ts then forwards 127.0.0.1:PROXY_PORT to that socket, because + * the AI SDK speaks TCP. The container has no network, so the forwarder is its only way out. + * + * Every check takes its filesystem root as a parameter, so tests can build a fake sandbox. + */ +import * as fs from "node:fs"; +import * as net from "node:net"; +import * as path from "node:path"; + +export const PROXY_DIR = "/repo/.sandbox-proxy"; +export const PROXY_SOCKET = `${PROXY_DIR}/sock`; +export const PROXY_PORT = 4141; +/** The base URL for `createAnthropic()` in the container: the forwarder, then the proxy route. */ +export const PROXY_BASE_URL = `http://127.0.0.1:${PROXY_PORT}/anthropic/v1`; + +const read = (file: string) => { + try { + return fs.readFileSync(file, "utf8"); + } catch { + return null; + } +}; + +/** + * True only in a bug-bash sandbox container: the launcher's marker, docker-init as PID 1 + * (`docker run --init`), and no network device but loopback (`--network none`). A host process + * that sets BUGBASH_CONTAINER=1 still fails the other two. + */ +export function inSandbox(env: NodeJS.ProcessEnv = process.env, root = "/"): boolean { + const init = read(path.join(root, "proc/1/cmdline")); + let devices: string[] = []; + try { + devices = fs.readdirSync(path.join(root, "sys/class/net")); + } catch { + return false; + } + return ( + env.BUGBASH_CONTAINER === "1" && + init?.startsWith("/sbin/docker-init\0") === true && + devices.join() === "lo" + ); +} + +/** True in a sandbox container whose launcher started a provider proxy for this job. */ +export function modelDrivenSandbox(env: NodeJS.ProcessEnv = process.env, root = "/"): boolean { + if (!inSandbox(env, root) || env.BUGBASH_MODEL_DRIVEN !== "1") return false; + const socket = fs.lstatSync(path.join(root, PROXY_SOCKET), { throwIfNoEntry: false }); + return socket?.isSocket() === true; +} + +/** Forwards each TCP connection on 127.0.0.1:`port` to the unix socket. Port 0 picks one. */ +export async function forwardToProxy(port = PROXY_PORT, socketPath = PROXY_SOCKET) { + const server = net.createServer((inner) => { + const outer = net.connect(socketPath); + const end = () => { + inner.destroy(); + outer.destroy(); + }; + for (const side of [inner, outer]) side.on("error", end).on("close", end); + inner.pipe(outer); + outer.pipe(inner); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(port, "127.0.0.1", resolve); + }); + return server; +} diff --git a/tests/bugbash/sandbox/inputs.ts b/tests/bugbash/sandbox/inputs.ts index 2e8cb6247e8..311e7f58e37 100644 --- a/tests/bugbash/sandbox/inputs.ts +++ b/tests/bugbash/sandbox/inputs.ts @@ -19,8 +19,9 @@ import { Refusal } from "./runner"; export const INPUTS = ["src", "tests/bugbash", "tsconfig.json", "package.json"]; /** The host env names that e2e.config.ts and startApp.ts read. No other host value passes. */ // Not BUGBASH_AI_REASON: after a failed probe it holds the provider URL and response text. -const PASS_ENV = ["BUGBASH_AI", "BUGBASH_AI_RESOLVED", "BUGBASH_APP_MODEL", "BUGBASH_MODEL", - "BUGBASH_EFFORT", "BUGBASH_SCENARIO", "E2E_TELEMETRY_DISABLED"]; // prettier-ignore +// Not BUGBASH_MODEL: the launcher sets it per job kind, and only for a model-driven job. +const PASS_ENV = ["BUGBASH_AI", "BUGBASH_AI_RESOLVED", "BUGBASH_APP_MODEL", "BUGBASH_EFFORT", + "BUGBASH_SCENARIO", "E2E_TELEMETRY_DISABLED"]; // prettier-ignore // Any name that looks like a credential, anywhere in it (AWS_SECRET_ACCESS_KEY, GH_TOKEN), or a // PAT: a backstop behind the allowlist, so it may also refuse a harmless name. const CREDENTIAL = /SECRET|PASSWORD|TOKEN|KEY|CREDENTIAL|BASE_URL|(^|_)PAT$/i; @@ -120,11 +121,17 @@ const RUN_VALUE_OPTIONS = ["--config", "--output", "--tag", "--exclude-tag", "-- const RUN_FLAGS = ["--pass-with-no-tests", "--last-failed", "--debug"]; /** - * Why these e2e args are not an exact-step repro run, or null. Only `e2e run` with the repro - * config passes: its tests are repros/**, and its agents hold no model (hostPause.ts). e2e + * Why these e2e args are not an `e2e run` with `config`, or null. With the default repro config + * its tests are repros/**, and its agents hold no model (hostPause.ts). The launcher also + * accepts the MCP Apps config, whose agents get a model only through the job's proxy. e2e * resolves --config from its cwd, so the cwd must be tests/bugbash and the config a regular file. */ -export function exactStepRefusal(args: string[], cwd: string, dir: string): string | null { +export function exactStepRefusal( + args: string[], + cwd: string, + dir: string, + config = "e2e.config.ts" +): string | null { if (args[0] !== "run") return `${JSON.stringify(args[0] ?? "")} is not \`e2e run\``; const configs: string[] = []; for (let i = 1; i < args.length; i++) { @@ -136,11 +143,11 @@ export function exactStepRefusal(args: string[], cwd: string, dir: string): stri return `${name} needs a value, got ${JSON.stringify(value ?? "")}`; if (name === "--config") configs.push(value); } - if (configs.length !== 1 || configs[0] !== "e2e.config.ts") - return `--config must be given once as e2e.config.ts, got ${JSON.stringify(configs)}`; + if (configs.length !== 1 || configs[0] !== config) + return `--config must be given once as ${config}, got ${JSON.stringify(configs)}`; if (fs.realpathSync(cwd) !== fs.realpathSync(dir)) return `the cwd must be ${dir}, got ${cwd}`; - if (fs.lstatSync(path.join(dir, "e2e.config.ts"), { throwIfNoEntry: false })?.isFile() !== true) - return `${dir}/e2e.config.ts is not a regular file (a symlink?)`; + if (fs.lstatSync(path.join(dir, config), { throwIfNoEntry: false })?.isFile() !== true) + return `${dir}/${config} is not a regular file (a symlink?)`; return null; } diff --git a/tests/bugbash/sandbox/launch.ts b/tests/bugbash/sandbox/launch.ts index 247683f07a2..dd971d7df46 100644 --- a/tests/bugbash/sandbox/launch.ts +++ b/tests/bugbash/sandbox/launch.ts @@ -1,7 +1,8 @@ /** - * Runs one exact-step bug-bash e2e job in a disposable container: the bug-bash sandbox (#5714). + * Runs one bug-bash e2e job in a disposable container: the bug-bash sandbox (#5714). * Usage, from tests/bugbash, with BUGBASH_AI=mock: * bun sandbox/launch.ts -- run --config e2e.config.ts --output .e2e/ [e2e args...] + * bun sandbox/launch.ts -- run --config e2e.mcpapps.config.ts --output .e2e/ [...] * bun sandbox/launch.ts --recover (make bug-bash-sandbox-recover; see recover()) * * The container runs the e2e CLI, Chromium and the seeded app with the pinned image (runner.ts). @@ -9,11 +10,18 @@ * read-only dist/ and node_modules/ (inputs.ts). Its output folder, the app log included, comes * back on its stdout as an export stream (exportStream.ts). * - * Only exact-step repro runs with the mock app AI pass. `explore`, the MCP Apps suite and the - * real app AI need the sandbox's provider proxy (a later step of #5714), so the launcher refuses - * them, and the #5815 host pause still refuses them on the host. There is no host fallback. + * Two job kinds run, both with the mock app AI. An exact-step repro run gets no model. The MCP + * Apps suite is model-driven: its explorer reaches BUGBASH_MODEL (default Sonnet 5.5) only + * through a provider proxy (proxy.ts) that this process runs for the job, on a unix socket that + * the container gets read-only. The provider key stays here, and BUGBASH_BUDGET_USD caps the + * spend at list price (not a billing cap: the upstream key's own limit is the backstop). The + * proxy writes one record per call to `.proxy.jsonl` on the host only. `e2e explore` and + * the real app AI come in a later step of #5714, and the host pause still refuses every + * model-driven run on the host. There is no host fallback. * Exit codes: the job's code, 2 refused, 3 the container state is unknown after cleanup, 4 the - * evidence is incomplete, 130 or 143 when SIGINT or SIGTERM stopped it. + * evidence is incomplete, 5 the proxy reported a fault (a call cost more than its bound, or a + * call outlived close()), 130 or 143 when SIGINT or SIGTERM stopped it. Exit 3 outranks a stop, + * and a stop outranks 5. */ import * as crypto from "node:crypto"; import * as fs from "node:fs"; @@ -21,6 +29,9 @@ import * as os from "node:os"; import * as path from "node:path"; import type { Readable } from "node:stream"; import { receiveExport } from "./exportStream"; +import { PROXY_DIR } from "./inContainer"; +import { startProxy } from "./proxy"; +import { Ledger, priced } from "./proxyPolicy"; // prettier-ignore import { checkMountSource, containerEnv, exactStepRefusal, jobEnv, outputDir, plainFolders, stage } from "./inputs"; import { Refusal, Session, Stopped } from "./runner"; @@ -58,6 +69,46 @@ function mockOnly(env: NodeJS.ProcessEnv): boolean { } /** The container always gets the mock mode, whatever else the host env holds. */ const MOCK = { BUGBASH_AI: "mock", BUGBASH_AI_RESOLVED: "mock" }; +const MCP_APPS_CONFIG = "e2e.mcpapps.config.ts"; +const MCP_APPS_MODEL = "anthropic:claude-sonnet-5-5"; + +/** The explorer model, the spend cap and the upstream of a model-driven job. */ +interface ModelJob { + model: string; + budgetUsd: number; + upstream: { baseUrl: string; apiKey: string }; +} + +/** + * Reads a model-driven job from the host env, before any docker command. Refusal texts never + * hold the key or the base URL. + */ +function modelJob(env: NodeJS.ProcessEnv): ModelJob { + const spec = env.BUGBASH_MODEL ?? MCP_APPS_MODEL; + const [provider, model] = spec.split(/:(.*)/s, 2); + if (provider !== "anthropic" || !model) + throw new Refusal(`BUGBASH_MODEL ${JSON.stringify(spec)}: only anthropic: runs here`); + // The proxy refuses every call of a model it cannot price, so the job could only fail. + if (!priced(model)) + throw new Refusal(`BUGBASH_MODEL ${JSON.stringify(spec)}: the proxy has no price for it`); + const budget = env.BUGBASH_BUDGET_USD ?? ""; + if (!/^\d+(\.\d+)?$/.test(budget) || !(Number(budget) > 0)) + throw new Refusal("set BUGBASH_BUDGET_USD: the most this run may spend at list price, in $"); + // bash-ai-proxy needs project automation, which a model-driven job turns off (startApp.ts). + if ((env.BUGBASH_SCENARIO ?? "") !== "") + throw new Refusal("BUGBASH_SCENARIO does not run in a model-driven job"); + const apiKey = env.ANTHROPIC_API_KEY ?? ""; + const base = env.ANTHROPIC_BASE_URL ?? ""; + if (apiKey === "" || base === "") + throw new Refusal("a model-driven job needs ANTHROPIC_API_KEY and ANTHROPIC_BASE_URL"); + if (!URL.canParse(base) || !/^https?:$/.test(new URL(base).protocol)) + throw new Refusal("ANTHROPIC_BASE_URL is not an http(s) URL"); + // Like the app's settings, the base URL may end in /v1; the proxy adds /v1/messages. + const baseUrl = base.replace(/\/+$/, "").replace(/\/v1$/, ""); + return { model, budgetUsd: Number(budget), upstream: { baseUrl, apiKey } }; +} + +const usd = (nanoUsd: number) => `$${(nanoUsd / 1e9).toFixed(4)}`; // --mount, not -v: a missing source fails instead of creating an empty folder. function bind(src: string, dst: string): string[] { @@ -144,18 +195,20 @@ export interface LaunchOptions { /** Runs the job and returns the launcher's exit code. Refusal and Stopped are thrown. */ export async function launch(args: string[], o: LaunchOptions): Promise { const dir = path.join(o.root, "tests/bugbash"); - const notExact = exactStepRefusal(args, o.cwd, dir); + const mcpApps = exactStepRefusal(args, o.cwd, dir, MCP_APPS_CONFIG) == null; + const notExact = mcpApps ? null : exactStepRefusal(args, o.cwd, dir); if (notExact != null) - throw new Refusal(`${notExact}: only exact-step repros run until the provider proxy (#5714)`); + throw new Refusal(`${notExact}: only exact-step repros and the MCP Apps suite run (#5714)`); // First, before any docker command and before the container env exists. if (!mockOnly(o.env)) - throw new Refusal( - "only the mock app AI runs until the provider proxy (#5714): set BUGBASH_AI=mock" - ); + throw new Refusal("only the mock app AI runs in the sandbox (#5714): set BUGBASH_AI=mock"); + const driven = mcpApps ? modelJob(o.env) : undefined; const output = outputDir(args); const dest = path.join(dir, output); - if (fs.lstatSync(dest, { throwIfNoEntry: false }) != null) - throw new Refusal(`${output} exists: remove it first`); + const records = `${dest}.proxy.jsonl`; + for (const file of [dest, records]) + if (fs.lstatSync(file, { throwIfNoEntry: false }) != null) + throw new Refusal(`${path.relative(dir, file)} exists: remove it first`); const mounts = () => (["dist", "node_modules"] as const).flatMap((rel) => bind(checkMountSource(o.root, rel), `/repo/${rel}`) @@ -166,6 +219,29 @@ export async function launch(args: string[], o: LaunchOptions): Promise const name = `xbb-${checkout.slice(0, 6)}-${crypto.randomBytes(3).toString("hex")}`; const jobDir = path.join(os.tmpdir(), "xum-bugbash-sandbox", checkout, name); let made = false; + let proxy: Awaited> | undefined; + let ledger: Ledger | undefined; + // The stop hook and the normal end share one close. Its rejection is a bound miss. + let closing: Promise | undefined; + // A call record that could not be written (a full disk): the proxy record is incomplete, so + // the job must not look successful. The errno code only: the message can hold the path. + let recordFault: string | undefined; + const record = (entry: object) => { + try { + fs.appendFileSync(records, `${JSON.stringify(entry)}\n`, { mode: 0o600 }); + } catch (error) { + recordFault ??= `proxy: a call record was not written (${(error as NodeJS.ErrnoException).code ?? "error"})`; + } + }; + // Nothing is cached before the proxy exists, so a stop during the staging cannot skip a close. + const closeProxy = () => + proxy == null + ? Promise.resolve(undefined) + : (closing ??= proxy.close().then( + () => recordFault, + (error: Error) => error.message + )); + session.onStop(closeProxy); const run = async (): Promise => { // Leftovers of crashed launches (#5882) are only listed here, first, so a launch that fails @@ -186,24 +262,57 @@ export async function launch(args: string[], o: LaunchOptions): Promise fs.writeFileSync(path.join(jobDir, "passwd"), passwd); fs.writeFileSync(path.join(jobDir, "group"), `root:x:0:\nbugbash:x:${gid}:\n`); const host = { BUGBASH_HOST_BOOT: bootId(), BUGBASH_HOST_NONCE: nonce }; - const env = containerEnv(o.env, { ...jobEnv(output), ...host, ...MOCK }); + const drivenEnv = driven && { + BUGBASH_MODEL: `anthropic:${driven.model}`, + BUGBASH_MODEL_DRIVEN: "1", + }; + const env = containerEnv(o.env, { ...jobEnv(output), ...host, ...MOCK, ...drivenEnv }); plainFolders(dir, path.dirname(output), true); + const proxyMount: string[] = []; + if (driven) { + // The socket's folder, read-only: connect() needs no write access to the mount (measured + // on Docker 27.5.1), and the container can add nothing to a host folder. + const proxyDir = path.join(jobDir, "proxy"); + fs.mkdirSync(proxyDir, { mode: 0o700 }); + fs.mkdirSync(path.join(staged, path.relative("/repo", PROXY_DIR))); // its mount point + ledger = new Ledger(driven.budgetUsd); + proxy = await startProxy({ + socketPath: path.join(proxyDir, "sock"), + upstream: driven.upstream, + job: { models: [driven.model] }, // the app AI is the mock: only the explorer calls + ledger, + log: record, + }); + proxyMount.push(...bind(proxyDir, PROXY_DIR)); + } // prettier-ignore const flags = ["--rm", "--interactive", "--init", "--log-driver", "none", "--network", "none", "--user", `${uid}:${gid}`, "--cap-drop", "ALL", "--security-opt", "no-new-privileges", "--read-only", "--pids-limit", "4096", "--memory", "4g", "--memory-swap", "4g", "--tmpfs", "/tmp:rw,nosuid,nodev,size=4g", "--tmpfs", "/home/bugbash:rw,nosuid,nodev,size=1g", "--tmpfs", `/repo/tests/bugbash/.e2e:rw,nosuid,nodev,size=1g,uid=${uid},gid=${gid}`, - ...bind(staged, "/repo"), ...mounts(), + ...bind(staged, "/repo"), ...mounts(), ...proxyMount, ...bind(path.join(jobDir, "passwd"), "/etc/passwd"), ...bind(path.join(jobDir, "group"), "/etc/group"), ...Object.entries(env).flatMap(([key, value]) => ["--env", `${key}=${value}`]), - "--workdir", "/repo/tests/bugbash", "--entrypoint", "bun", image, "sandbox/entry.ts", - "--export", output, "--", "node", "../../node_modules/e2e/dist/cli/bin.js", ...args]; + "--workdir", "/repo/tests/bugbash", "--entrypoint", "bun"]; + const command: [string, ...string[]] = [ + image, + "sandbox/entry.ts", + "--export", + output, + "--", + "node", + "../../node_modules/e2e/dist/cli/bin.js", + ...args, + ]; // A signal during the synchronous staging runs its handler only at the next turn of the // event loop. This yield lets it run, so own() throws and no container starts (measured). await new Promise((resolve) => setImmediate(resolve)); session.own({ name, owner: ownerLabel(), checkout }); - log(`${name} starts: --network none, mock app AI`); + const via = driven + ? `proxy for anthropic:${driven.model} (budget $${driven.budgetUsd.toFixed(2)}), ` + : ""; + log(`${name} starts: --network none, ${via}mock app AI, launcher pid ${process.pid}`); // A refused frame ends the job at once: the container would otherwise block on a full pipe // until the deadline (#5930 item 4). const receive = async (out: Readable) => { @@ -211,7 +320,7 @@ export async function launch(args: string[], o: LaunchOptions): Promise if (!result.complete) session.stop(`export: ${result.error}`); return result; }; - const job = await session.runJob(flags, receive, DEADLINE_MS); + const job = await session.runJob(flags, command, receive, DEADLINE_MS); const { complete, files, error } = job.received; log( `${name} exit ${job.code}, ${files} files, ${complete ? "complete" : `incomplete: ${error}`}` @@ -227,12 +336,23 @@ export async function launch(args: string[], o: LaunchOptions): Promise } catch (error) { result = { error }; } + const missed = await closeProxy(); + if (proxy && ledger) { + const { settled, kept, refused, refusedBy } = proxy.stats(); + const { spentNanoUsd, capNanoUsd } = ledger.totals(); + log( + `${name} proxy: ${settled} settled, ${kept} kept in full, ${refused} refused ` + + `${JSON.stringify(refusedBy)}, ${usd(spentNanoUsd)} of ${usd(capNanoUsd)} at list price` + ); + } const state = await session.cleanup(); if (made) fs.rmSync(jobDir, { recursive: true, force: true }); log(`${name} cleanup: ${state}`); + if (missed != null) log(`${name} ${missed}`); if (state.startsWith("unknown")) return 3; - // A stop seen before cleanup ended outranks the job's result and an earlier error. + // A stop seen before cleanup ended outranks a proxy fault, the job's result and an error. if (o.stop.aborted) throw new Stopped(String(o.stop.reason)); + if (missed != null) return 5; if (typeof result !== "number") throw result.error; return result; } diff --git a/tests/bugbash/sandbox/proxy.test.ts b/tests/bugbash/sandbox/proxy.test.ts index 0be141a0636..d751a9ea1e9 100644 --- a/tests/bugbash/sandbox/proxy.test.ts +++ b/tests/bugbash/sandbox/proxy.test.ts @@ -181,6 +181,8 @@ test.each([ ["the proxy closes mid-call", "[fake:hang]", "close"], ["the upstream answers 529", "[fake:529]", ""], ["the response exceeds 32 MiB", "[fake:huge]", ""], + ["a delta lowers a count of message_start", "[fake:lowered]", ""], + ["a delta nulls a count", "[fake:nulled]", ""], ])("keeps the full reservation when %s (P7)", async (_name, marker, action) => { // Only the stall case may reach the deadline: the others must end without it. const deadlineMs = marker === "[fake:stall]" ? 300 : 60_000; @@ -326,10 +328,53 @@ test("refuses a fifth concurrent call, and a call that does not fit the budget, expect(poor.ledger.totals()).toMatchObject({ refused: 1, spentNanoUsd: 0 }); }); +test("close() settles at once with calls stuck at every stage, and keeps their reservations", async () => { + const { socketPath, fake, ledger, proxy, recorded } = await setup(); + // Upstream silent before its headers, a stream that never ends, and a client that never + // reads a big answer (the proxy waits on drain). Plus a connection that never sends headers. + const stream = { stream: true }; + const stuck = ["[fake:stall]", "[fake:hang]"].map((text) => send(socketPath, body(text, stream))); + const unread = new Promise((resolve) => { + const req = http.request( + { socketPath, method: "POST", path: PROXY_PATH, headers: HEADERS }, + (res) => { + res.pause(); + res.on("close", resolve); + } + ); + req.on("error", () => resolve()); + req.end(body("[fake:huge]", stream)); + }); + const idle = net.connect(socketPath); + idle.on("error", () => undefined); + while (fake.requests.length < 3 || proxy.stats().inFlight < 3) await Bun.sleep(5); + const started = Date.now(); + await proxy.close(); + expect(Date.now() - started).toBeLessThan(2_000); // CLOSE_MS (5 s) is the hard bound + const records = await recorded(3); + expect(records.map((record) => record.outcome)).toEqual(["kept", "kept", "kept"]); + const { reservedNanoUsd, spentNanoUsd } = ledger.totals(); + expect(reservedNanoUsd).toBe(0); + expect(spentNanoUsd).toBe(records.reduce((sum, record) => sum + record.reservedNanoUsd!, 0)); + await Promise.allSettled([...stuck, unread]); + idle.destroy(); +}); + +test("refusals beyond the first 20 are counted per category, not logged", async () => { + const { socketPath, records, proxy } = await setup(); + for (let i = 0; i < 30; i++) await send(socketPath, body("hi"), { path: `/x${i}` }); + for (let i = 0; i < 5; i++) await send(socketPath, body("hi", { [`key${i}`]: 1 })); + expect(records).toHaveLength(20); + // The quoted key names are the container's: they never become categories. + expect(proxy.stats().refusedBy).toEqual({ "route: not allowed": 30, "body: key": 5 }); + expect(proxy.stats().refused).toBe(35); +}); + test("after close() the socket refuses connections and the upstream sees nothing new (P8)", async () => { const { socketPath, fake, proxy } = await setup(); expect((await send(socketPath, body("hi"))).status).toBe(200); await proxy.close(); + expect(fs.existsSync(socketPath)).toBe(false); // no socket file left in the job folder const after = await send(socketPath, body("hi")).then( () => "answered", () => "refused" diff --git a/tests/bugbash/sandbox/proxy.ts b/tests/bugbash/sandbox/proxy.ts index 97d0a46c39e..12bf5674c59 100644 --- a/tests/bugbash/sandbox/proxy.ts +++ b/tests/bugbash/sandbox/proxy.ts @@ -33,6 +33,17 @@ const MAX_IN_FLIGHT = 4; const MAX_CONNECTIONS = 16; const DEADLINE_MS = 10 * 60_000; const MAX_RESPONSE_BYTES = 32 * 1024 * 1024; +/** + * Refusals cost the container nothing, so it can send them without end. The first ones are + * logged in full; after that only their count per category grows (stats().refusedBy). + */ +const LOGGED_REFUSALS = 20; +/** + * How long close() waits for the aborted calls and the listeners. The launcher's stop hook gets + * 10 s (runner.ts), so close() must settle before that. A call still open then is a proxy bug: + * close() rejects, and its reservation stays counted (Ledger: reserved). + */ +const CLOSE_MS = 5_000; const RESPONSE_HEADERS = /^(content-type|request-id|retry-after|anthropic-ratelimit-[a-z0-9-]+)$/; /** One line per call for the launcher log. */ @@ -63,6 +74,8 @@ export interface ProxyOptions { class SseUsage { usage: Record = {}; stopped = false; + /** A delta lowered or nulled a count of message_start: the shape changed, so do not settle. */ + lowered = false; #decoder = new TextDecoder(); #line = ""; @@ -78,7 +91,15 @@ class SseUsage { continue; // an unreadable event cannot settle the call: no message_stop follows it } if (event.type === "message_start") this.usage = { ...event.message?.usage }; - if (event.type === "message_delta") this.usage = { ...this.usage, ...event.usage }; + if (event.type === "message_delta") { + // Delta counts are cumulative, so one below its message_start value is a shape change. + for (const [key, value] of Object.entries(event.usage ?? {})) { + const before = this.usage[key]; + if (value == null || (typeof before === "number" && Number(value) < before)) + this.lowered = true; + } + this.usage = { ...this.usage, ...event.usage }; + } // An error after message_stop is impossible; one before it leaves `stopped` false. if (event.type === "message_stop") this.stopped = true; } @@ -93,7 +114,22 @@ export async function startProxy(options: ProxyOptions) { const deadlineMs = options.deadlineMs ?? DEADLINE_MS; const calls = new Set<{ abort: AbortController; done: Promise }>(); const counts = { refused: 0, settled: 0, kept: 0, boundExceeded: 0 }; + /** + * Refusals per category: the reason up to its first quoted (container-chosen) name, so the + * keys come from the fixed reason texts of this file and proxyPolicy.ts and stay few. + */ + const refusedBy: Record = {}; + const record = (entry: CallRecord) => { + counts[entry.outcome] += 1; + if (entry.outcome === "refused") { + const category = (entry.reason ?? "").split('"')[0].trim(); + refusedBy[category] = (refusedBy[category] ?? 0) + 1; + if (counts.refused > LOGGED_REFUSALS) return; + } + options.log(entry); + }; let closed = false; + let listening: Promise = Promise.resolve(); async function handle( req: http.IncomingMessage, @@ -101,10 +137,7 @@ export async function startProxy(options: ProxyOptions) { abort: AbortController ) { const started = Date.now(); - const log = (record: Omit) => { - counts[record.outcome] += 1; - options.log({ ...record, ms: Date.now() - started }); - }; + const log = (entry: Omit) => record({ ...entry, ms: Date.now() - started }); const reply = (status: number, reason: string) => replyError(res, status, reason); const refuse = (status: number, reason: string, model?: string) => { if (!res.headersSent && !res.destroyed) reply(status, reason); @@ -177,7 +210,7 @@ export async function startProxy(options: ProxyOptions) { // end() to a client that went away. res.end(); if (status !== 200 || abort.signal.aborted) return; - if (sse) usage = parser.stopped ? parser.usage : undefined; + if (sse) usage = parser.stopped && !parser.lowered ? parser.usage : undefined; else usage = (JSON.parse(Buffer.concat(json).toString()) as { usage?: unknown }).usage; } catch { // Fixed texts only: fetch errors can name the upstream host and other details. @@ -204,8 +237,7 @@ export async function startProxy(options: ProxyOptions) { if (closed) return res.destroy(); // a kept-alive connection after close() const refuse = (status: number, reason: string) => { replyError(res, status, reason); - counts.refused += 1; - options.log({ outcome: "refused", status, reason, ms: 0 }); + record({ outcome: "refused", status, reason, ms: 0 }); }; // P1: an exact match, so a query string or any other path refuses. if (req.method !== "POST" || req.url !== PROXY_PATH) return refuse(404, "route: not allowed"); @@ -281,13 +313,25 @@ export async function startProxy(options: ProxyOptions) { async close() { if (!closed) { closed = true; - front.close(); - server.close(); - fs.rmSync(inner, { recursive: true, force: true }); + // The close callbacks run once the socket files are gone (Bun and Node remove them). + listening = Promise.all( + [front, server].map((listener) => new Promise((done) => listener.close(done))) + ); } for (const call of calls) call.abort.abort(); for (const outer of pipes) outer.emit("close"); - await Promise.allSettled([...calls].map((call) => call.done)); + let timer: ReturnType | undefined; + const late = new Promise<"late">((resolve) => { + timer = setTimeout(() => resolve("late"), CLOSE_MS); + }); + const ended = Promise.all([Promise.allSettled([...calls].map((c) => c.done)), listening]); + const outcome = await Promise.race([ended, late]).finally(() => clearTimeout(timer)); + fs.rmSync(inner, { recursive: true, force: true }); + if (outcome === "late") { + throw new Error( + `proxy: ${calls.size} call(s) still open ${CLOSE_MS} ms after close(); their reservations stay counted` + ); + } if (counts.boundExceeded > 0) { throw new Error( `proxy: ${counts.boundExceeded} call(s) cost more than their reserved bound; fix maxCostNanoUsd in proxyPolicy.ts` @@ -321,7 +365,7 @@ export async function startProxy(options: ProxyOptions) { req.end(JSON.stringify({ model, max_tokens: 1, messages })); }); }, - stats: () => ({ ...counts, inFlight: calls.size }), + stats: () => ({ ...counts, inFlight: calls.size, refusedBy: { ...refusedBy } }), }; } diff --git a/tests/bugbash/sandbox/proxyPolicy.ts b/tests/bugbash/sandbox/proxyPolicy.ts index 4f97960cdd7..3e398f8018b 100644 --- a/tests/bugbash/sandbox/proxyPolicy.ts +++ b/tests/bugbash/sandbox/proxyPolicy.ts @@ -51,6 +51,9 @@ const PRICES = new Map([ ]); const VISUAL_TOKENS = 4784; +/** Whether the proxy can price (and so bound) a call of this model. */ +export const priced = (model: string) => PRICES.has(model); + export interface JobPolicy { /** Model IDs this job may call: its explorer model and the app model (P3). */ models: readonly string[]; diff --git a/tests/bugbash/sandbox/runner.test.ts b/tests/bugbash/sandbox/runner.test.ts index 06aa4d16316..f072d935f2d 100644 --- a/tests/bugbash/sandbox/runner.test.ts +++ b/tests/bugbash/sandbox/runner.test.ts @@ -4,6 +4,7 @@ import * as crypto from "node:crypto"; import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; +import { startFakeUpstream } from "./fakeUpstream"; import { launch, ownerState, recover } from "./launch"; import { groupState, readImageLock, Refusal, Session, Stopped } from "./runner"; @@ -54,6 +55,12 @@ case "$1" in slowcreate|swap) cat >/dev/null; exit 0 ;; hang) cat >/dev/null; echo "stdin closed" >> "$bin/calls.log"; unregister "$id"; exit 0 ;; badframe) printf 'not a frame\n'; cat >/dev/null; unregister "$id"; exit 0 ;; + # One model call through the job's proxy socket, like the container's forwarder makes. + proxycall) for a in "$@"; do case "$a" in *dst=/repo/.sandbox-proxy,readonly) psrc=\${a#type=bind,src=}; psrc=\${psrc%%,*} ;; esac; done + echo "proxy dir mode $(stat -c %a "$psrc")" >> "$bin/calls.log" + curl -s --unix-socket "$psrc/sock" -H 'content-type: application/json' -H 'anthropic-version: 2023-06-01' \ + -d '{"model":"claude-sonnet-5-5","max_tokens":10,"messages":[{"role":"user","content":"'"$CALL"'"}]}' \\ + http://proxy/anthropic/v1/messages >> "$bin/calls.log"; echo >> "$bin/calls.log" ;; esac printf '{"p":"app.log","n":2}\nok'; [ "$RUN" = cut ] || printf '{"end":true}\n' [ "$RUN" = linger ] || unregister "$id"; exit 7 ;; @@ -132,6 +139,7 @@ function fake(over: Record = {}) { PS_RC: "0", INSPECT_RC: "0", RUN: "ok", + CALL: "hi", ...over, }; fs.writeFileSync( @@ -353,7 +361,7 @@ async function jobSession(over: Record, graceMs?: number) { const s = session(stop, graceMs); await s.ensureImage(); s.own(JOB); - const job = s.runJob(["--rm", "img"], drain, 60_000); + const job = s.runJob(["--rm"], ["img"], drain, 60_000); job.catch(() => undefined); while (!calls().includes("run ")) await Bun.sleep(20); return { s, stop, job }; @@ -384,12 +392,17 @@ test.each([ ["no container", "", "unknown"], ["a container with our name and labels", "c1 xbb-1 boot:pid abc\n", "removed"], ["a foreign container with our name", "c2 xbb-1 other:pid abc\n", "unknown"], + [ + "two containers with our name and labels", + "c1 xbb-1 boot:pid abc\nc3 xbb-1 boot:pid abc\n", + "unknown", + ], ])("C3: `docker run` dies before the cidfile, %s", async (_name, present, state) => { fs.writeFileSync(path.join(bin, "containers"), present); const { s, job } = await jobSession({ RUN: "die" }); expect((await job).code).toBe(125); expect(await s.cleanup()).toStartWith(state); - expect(registry()).toBe(present.startsWith("c1") ? "" : present); + expect(registry()).toBe(state === "removed" ? "" : present); }); test.each([ @@ -500,10 +513,19 @@ test.each([ const s = session(); await s.ensureImage(); s.own(JOB); - expect(await failure(s.runJob([name, value, "img"], drain, 1_000))).toThrow(Refusal); + expect(await failure(s.runJob([name, value], ["img"], drain, 1_000))).toThrow(Refusal); expect(calls()).not.toContain("run "); }); +test("C9: a short option after the image is the job's own argument and passes", async () => { + fake({ IMAGES: "sha256:abc" }); + const s = session(); + await s.ensureImage(); + s.own(JOB); + await s.runJob(["--rm"], ["img", "run", "-g", "x"], drain, 60_000).catch(() => undefined); + expect(calls()).toMatch(/^run .* --rm img run -g x \[/m); +}); + test("C11: the stop hooks finish before the lifeline closes", async () => { const { s, stop, job } = await jobSession({ RUN: "hang" }); s.onStop(async () => { @@ -526,7 +548,7 @@ test("C11: a stop hook that never settles still lets the lifeline close and clea sessions.push(s); await s.ensureImage(); s.own(JOB); - const job = s.runJob(["--rm", "img"], drain, 60_000); + const job = s.runJob(["--rm"], ["img"], drain, 60_000); job.catch(() => undefined); while (!calls().includes("run ")) await Bun.sleep(20); s.onStop(() => new Promise(() => undefined)); @@ -546,21 +568,26 @@ function launchIn( env: Record = HOST_ENV, leftovers: string[] = [] ) { + const cwd = prepare(over, leftovers); + return launch(ARGS, { root: fs.realpathSync(root), cwd, env, stop: stop.signal }); +} +/** The checkout with both configs and both mount sources, and the fake's answers. */ +function prepare(over: Record = {}, leftovers: string[] = []) { const real = fs.realpathSync(root); fs.mkdirSync(path.join(real, "tests/bugbash"), { recursive: true }); fs.writeFileSync(path.join(real, "tests/bugbash/e2e.config.ts"), "export default {}"); for (const dir of ["dist", "node_modules"]) fs.mkdirSync(path.join(real, dir), { recursive: true }); fs.writeFileSync(path.join(bin, "containers"), [...FOREIGN, ...leftovers].join("\n") + "\n"); + fs.writeFileSync(path.join(real, "tests/bugbash/e2e.mcpapps.config.ts"), "export default {}"); fake({ IMAGES: "sha256:abc", SWAP: path.join(real, "node_modules"), ...over }); - const cwd = path.join(real, "tests/bugbash"); - return launch(ARGS, { root: real, cwd, env, stop: stop.signal }); + return path.join(real, "tests/bugbash"); } /** Nothing of a job stays: no container, no job folder, no private client folder, no image rm. */ function expectNothingLeft() { expect(fs.readFileSync(path.join(bin, "containers"), "utf8").trim().split("\n")).toEqual(FOREIGN); const left = fs.readdirSync(tmp, { recursive: true }).map(String); - expect(left.filter((p) => /xbb-|xum-bugbash-docker/.test(p))).toEqual([]); + expect(left.filter((p) => /xbb-|xum-bugbash-(docker|proxy)/.test(p))).toEqual([]); expect(calls()).not.toMatch(/^(rmi|image rm|image prune|system prune)/m); } @@ -854,6 +881,151 @@ test("C2: recover reports an unknown cleanup state (exit 3)", async () => { } }); +// The MCP Apps suite (plan PR B1): model-driven, through a proxy for the job. +const MCP_ARGS = ["run", "--config", "e2e.mcpapps.config.ts", "--output", ".e2e/m"]; +const UPSTREAM_KEY = "sk-host-upstream-key-for-the-launch-test"; +async function mcpLaunch( + over: Record, + env: Record = {}, + stop = new AbortController() +) { + const upstream = await startFakeUpstream(); + try { + const host = { + ...HOST_ENV, + BUGBASH_BUDGET_USD: "1", + ANTHROPIC_API_KEY: UPSTREAM_KEY, + ANTHROPIC_BASE_URL: `${upstream.baseUrl}/v1`, + ...env, + }; + const logged: string[] = []; + const error = spyOn(console, "error").mockImplementation( + (line: string) => void logged.push(line) + ); + try { + const cwd = prepare(over); + const options = { root: fs.realpathSync(root), cwd, env: host }; + const code = await launch(MCP_ARGS, { ...options, stop: stop.signal }).catch( + (e: unknown) => e + ); + return { + code, + logged, + requests: upstream.requests, + records: path.join(cwd, ".e2e/m.proxy.jsonl"), + }; + } finally { + error.mockRestore(); + } + } finally { + await upstream.close(); + } +} + +test.each([ + ["no budget", { BUGBASH_BUDGET_USD: "" }, /BUGBASH_BUDGET_USD/], + ["a budget that is not a number", { BUGBASH_BUDGET_USD: "1e9" }, /BUGBASH_BUDGET_USD/], + ["no key", { ANTHROPIC_API_KEY: "" }, /ANTHROPIC_API_KEY/], + ["no base URL", { ANTHROPIC_BASE_URL: "" }, /ANTHROPIC_BASE_URL/], + ["an OpenAI explorer model", { BUGBASH_MODEL: "openai:gpt-6.1-sol" }, /only anthropic/], + [ + "an Anthropic model the proxy cannot price", + { BUGBASH_MODEL: "anthropic:claude-x-9" }, + /no price/, + ], + ["the bash-ai-proxy scenario", { BUGBASH_SCENARIO: "bash-ai-proxy" }, /BUGBASH_SCENARIO/], + ["the real app AI", { BUGBASH_AI: "real" }, /only the mock app AI/], +])("B1: the MCP Apps suite refuses %s before any docker command", async (_name, env, message) => { + const { code } = await mcpLaunch({}, env); + expect(code).toBeInstanceOf(Refusal); + expect((code as Error).message).toMatch(message); + expect((code as Error).message).not.toContain(UPSTREAM_KEY); + expect(calls()).toBe(""); +}); + +test("B1: an MCP Apps job reaches the model only through its proxy, with the host key", async () => { + const { code, logged, requests, records } = await mcpLaunch({ RUN: "proxycall" }); + expect(code).toBe(7); // the fake job's own exit code + const run = calls() + .split("\n") + .find((line) => line.startsWith("run "))!; + expect(run).toMatch(/--mount type=bind,src=\S+\/proxy,dst=\/repo\/\.sandbox-proxy,readonly/); + expect(run).toContain("--env BUGBASH_MODEL=anthropic:claude-sonnet-5-5"); + expect(run).toContain("--env BUGBASH_MODEL_DRIVEN=1"); + expect(run).not.toContain(UPSTREAM_KEY); + expect(run).not.toContain("ANTHROPIC"); + expect(calls()).toContain("proxy dir mode 700"); + expect(calls()).toContain('"usage":'); + expect(requests).toHaveLength(1); + expect(requests[0].headers["x-api-key"]).toBe(UPSTREAM_KEY); + const lines = fs.readFileSync(records, "utf8").trim().split("\n"); + expect(lines.map((line) => (JSON.parse(line) as { outcome: string }).outcome)).toEqual([ + "settled", + ]); + expect(fs.statSync(records).mode & 0o777).toBe(0o600); + expect(logged.join("\n")).toMatch(/proxy for anthropic:claude-sonnet-5-5 \(budget \$1\.00\)/); + expect(logged.join("\n")).toMatch( + /proxy: 1 settled, 0 kept in full, 0 refused \{\}, \$0\.\d+ of \$1\.0000/ + ); + expect(logged.join("\n")).not.toContain(UPSTREAM_KEY); + expectNothingLeft(); // the job folder with the proxy socket is gone too +}); + +test("B1: a proxied call that costs more than its bound fails the job (exit 5)", async () => { + const { code, logged } = await mcpLaunch({ RUN: "proxycall", CALL: "[fake:overbill]" }); + expect(code).toBe(5); + expect(logged.join("\n")).toContain("cost more than their reserved bound"); + expectNothingLeft(); +}); + +test("B1: a call record that cannot be written fails the job (exit 5)", async () => { + const append = spyOn(fs, "appendFileSync").mockImplementation((file) => { + if (String(file).endsWith(".proxy.jsonl")) + throw Object.assign(new Error(`ENOSPC: ${String(file)}`), { code: "ENOSPC" }); + }); + try { + const { code, logged } = await mcpLaunch({ RUN: "proxycall" }); + expect(code).toBe(5); + expect(logged.join("\n")).toContain("a call record was not written (ENOSPC)"); + } finally { + append.mockRestore(); + } + expectNothingLeft(); +}); + +test("B1: a stop after a bound miss still ends with the stop, not exit 5", async () => { + const stop = new AbortController(); + // The signal lands after the job and the proxy close, while cleanup runs. + const cleanup = spyOn(Session.prototype, "cleanup"); + cleanup.mockImplementation(function (this: Session) { + stop.abort("SIGINT"); + cleanup.mockRestore(); // the real cleanup from here on + return this.cleanup(); + }); + try { + const { code, logged } = await mcpLaunch( + { RUN: "proxycall", CALL: "[fake:overbill]" }, + {}, + stop + ); + expect(code).toBeInstanceOf(Stopped); + expect(logged.join("\n")).toContain("cost more than their reserved bound"); + } finally { + cleanup.mockRestore(); + } + expectNothingLeft(); +}); + +test("B1: a repro job gets no proxy and no explorer model, whatever the host sets", async () => { + expect( + await launchIn({}, new AbortController(), { ...HOST_ENV, BUGBASH_MODEL: "anthropic:x" }) + ).toBe(7); + const run = calls() + .split("\n") + .find((line) => line.startsWith("run "))!; + expect(run).not.toMatch(/sandbox-proxy|BUGBASH_MODEL/); +}); + test("a signal during the synchronous staging starts no container", async () => { const stop = new AbortController(); // containerEnv reads BUGBASH_EFFORT right after staging, still synchronously. A real signal diff --git a/tests/bugbash/sandbox/runner.ts b/tests/bugbash/sandbox/runner.ts index 5fa279ad4f9..6eae34e58db 100644 --- a/tests/bugbash/sandbox/runner.ts +++ b/tests/bugbash/sandbox/runner.ts @@ -22,7 +22,12 @@ const KILL_AFTER_MS = 5_000; const GRACE_MS = 60_000; /** How long one stop hook gets before the lifeline closes anyway: a hung hook must not block it. */ const HOOK_MS = 10_000; -/** A short option, or a flag that names, labels or tracks the container: runJob() sets those. */ +/** + * A short option, or a flag that names, labels or tracks the container: runJob() sets those. It + * applies to the docker-run flags only, so the job's own arguments (e2e's `-g`) pass. `--detach` + * and `--rm=false` pass too: harmless while launch.ts is the only caller, since cleanup removes + * the container by ID. Refuse them if runJob() ever takes flags from a wider caller. + */ const CALLER_FLAG = /^(-[^-]|--(name|label|label-file|cidfile)(=|$))/; export class Refusal extends Error {} @@ -210,21 +215,26 @@ export class Session { } /** - * Runs the owned job's container: `docker run --name --label … --cidfile …` and then - * `args`. Its stdin is the lifeline: entry.ts stops the job on EOF, and the container also + * Runs the owned job's container: `docker run --name --label … --cidfile …`, then + * `flags`, then `command` (the image and its arguments). Its stdin is the lifeline: entry.ts stops the job on EOF, and the container also * gets EOF when this process dies. Its stdout goes to `receive`, its stderr to ours. A stop or * the timeout closes the lifeline; the client is killed only if it outlives the grace period. * cleanup() then removes the container by the ID that the CLI wrote to the cidfile. */ - async runJob(args: string[], receive: (out: Readable) => Promise, timeoutMs: number) { + async runJob( + flags: string[], + command: [image: string, ...args: string[]], + receive: (out: Readable) => Promise, + timeoutMs: number + ) { const job = this.#owned; if (job == null) throw new Error("runJob() needs own() first"); if (this.#stopped != null) throw new Stopped(this.#stopped); - const flag = args.find((arg) => CALLER_FLAG.test(arg)); + const flag = flags.find((arg) => CALLER_FLAG.test(arg)); if (flag != null) throw new Refusal(`runJob() sets ${flag} itself; use long options`); // prettier-ignore const named = ["run", "--name", job.name, "--label", `xum.bugbash.owner=${job.owner}`, - "--label", `xum.bugbash.checkout=${job.checkout}`, "--cidfile", this.#cidFile(), ...args]; + "--label", `xum.bugbash.checkout=${job.checkout}`, "--cidfile", this.#cidFile(), ...flags, ...command]; const received: Promise[] = []; const r = await this.#spawn("docker", named, this.#client!, timeoutMs, (child) => { this.#runClient = child; @@ -284,8 +294,10 @@ export class Session { const ps = ["ps", "-aq", "--no-trunc", ...filters]; const found = await this.#spawn("docker", ps, this.#client!, 15_000); if (!found.ok) return `unknown: ${found.error}`; - if (found.stdout !== "") - return this.#removeById(found.stdout.split("\n")[0], job, "name and labels"); + const ids = found.stdout.split("\n").filter((line) => line !== ""); + // Docker names are unique, so two matches mean the lookup is not what this code assumes. + if (ids.length > 1) return `unknown: ${ids.length} containers match the name and labels`; + if (ids.length === 1) return this.#removeById(ids[0], job, "name and labels"); if (this.#runClient != null) return "unknown: `docker run` left no container ID"; return "removed"; // no `docker run` ran, so no container of this job can exist } diff --git a/tests/bugbash/startApp.test.ts b/tests/bugbash/startApp.test.ts index fe3907f4c59..0996cdfce45 100644 --- a/tests/bugbash/startApp.test.ts +++ b/tests/bugbash/startApp.test.ts @@ -1,7 +1,7 @@ import { afterEach, expect, test } from "bun:test"; import { spawn, type ChildProcess } from "child_process"; import * as net from "net"; -import { waitForHealth } from "./startApp"; +import { appSwitches, waitForHealth } from "./startApp"; let server: net.Server | undefined; let child: ChildProcess | undefined; @@ -43,3 +43,20 @@ test("a hung /health request does not hide a seed server that exited", async () expect((error as Error).message).toContain("seed server exited early"); expect(Date.now() - started).toBeLessThan(10_000); }, 15_000); + +test("a model-driven job gets every kill switch, also with the mock app AI", () => { + const all = [ + "XUM_DISABLE_AGENT_TOOLS", + "XUM_DISABLE_PROJECT_AUTOMATION", + "XUM_DISABLE_TERMINALS", + ]; + const keys = (env: Record) => Object.keys(env).sort(); + expect(keys(appSwitches("mock", false, true))).toEqual([...all, "XUM_MOCK_AI"]); + expect(keys(appSwitches("real", false, false))).toEqual(all); + // Exact-step repros keep today's app env. + expect(appSwitches("mock", false, false)).toEqual({ XUM_MOCK_AI: "1" }); + expect(keys(appSwitches("mock", true, false))).toEqual([ + "XUM_DISABLE_AGENT_TOOLS", + "XUM_DISABLE_TERMINALS", + ]); +}); diff --git a/tests/bugbash/startApp.ts b/tests/bugbash/startApp.ts index 997475a0947..43bdcc1c94d 100644 --- a/tests/bugbash/startApp.ts +++ b/tests/bugbash/startApp.ts @@ -137,6 +137,25 @@ function failIfExited(child: ChildProcess): void { } } +/** The mode and kill-switch env of the app server. */ +export function appSwitches( + mode: "mock" | "real", + fakeProvider: boolean, + modelDriven: boolean +): Record { + // No agent tools and no terminal (AGENTS.md: not until the app runs in a sandbox). Project + // automation stays on: its kill switch also makes the bash AI proxy refuse every call, the + // fake's replies are fixed text, and the seeded demo repo has no hooks. + if (fakeProvider) return { XUM_DISABLE_AGENT_TOOLS: "1", XUM_DISABLE_TERMINALS: "1" }; + const all = { + XUM_DISABLE_AGENT_TOOLS: "1", + XUM_DISABLE_TERMINALS: "1", + XUM_DISABLE_PROJECT_AUTOMATION: "1", + }; + if (mode === "real") return all; + return { XUM_MOCK_AI: "1", ...(modelDriven && all) }; +} + export async function waitForHealth(base: string, child: ChildProcess): Promise { const deadline = Date.now() + 60_000; while (Date.now() < deadline) { @@ -346,6 +365,12 @@ async function main(): Promise { if (scenario !== "" && ai.mode !== "mock") { fail("BUGBASH_SCENARIO=bash-ai-proxy replaces the mock: run it with BUGBASH_AI=mock"); } + // A model-driven job (the sandbox launcher sets this): a model reads every reply and page, so + // the app gets all three switches in both AI modes. bash-ai-proxy needs project automation. + const modelDriven = process.env.BUGBASH_MODEL_DRIVEN === "1"; + if (modelDriven && scenario !== "") { + fail("BUGBASH_SCENARIO=bash-ai-proxy needs project automation: not in a model-driven job"); + } const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "xum-bugbash-")); const xumRoot = path.join(tempRoot, "xum"); const fakeProvider = scenario === "bash-ai-proxy" ? await startFakeProvider(xumRoot) : undefined; @@ -359,18 +384,7 @@ async function main(): Promise { // Real mode: a real model answers in the app on this host. It gets no tools, and the explorer // gets no terminal or project init hooks, so injected text in a reply that the explorer // follows cannot run host commands through them (see the header). - ...(fakeProvider - ? // No agent tools and no terminal (AGENTS.md: not until the app runs in a sandbox). - // Project automation stays on: its kill switch also makes the proxy refuse every call, - // the fake's replies are fixed text, and the seeded demo repo has no hooks. - { XUM_DISABLE_AGENT_TOOLS: "1", XUM_DISABLE_TERMINALS: "1" } - : ai.mode === "mock" - ? { XUM_MOCK_AI: "1" } - : { - XUM_DISABLE_AGENT_TOOLS: "1", - XUM_DISABLE_TERMINALS: "1", - XUM_DISABLE_PROJECT_AUTOMATION: "1", - }), + ...appSwitches(ai.mode, fakeProvider != null, modelDriven), // Bug-bash clicks are not product usage. XUM_DISABLE_TELEMETRY: "1", // Git inside the app must not read or write the user's real ~/.gitconfig.