🤖 tests: run the MCP Apps suite in the bug-bash sandbox through the provider proxy

Makefile

@@ -219,8 +219,8 @@ dev-desktop-sandbox: ## Start an isolated Electron dev instance (fresh XUM_ROOT
 dev-server-sandbox: ## Start an isolated dev-server instance (fresh XUM_ROOT + free ports)
 	@bun scripts/dev-server-sandbox.ts $(DEV_SERVER_SANDBOX_ARGS)
 
-# bug-bash and mcp-apps-e2e are paused on the host (tests/bugbash/hostPause.ts, #5714): a model
-# picks their actions. Their recipes refuse at once, so they build nothing first.
+# bug-bash is paused on the host (tests/bugbash/hostPause.ts, #5714): a model picks its actions.
+# Its recipe refuses at once, so it builds nothing first.
 bug-bash: ## Agent bug bash: e2e explore charters x models (paused on the host until the sandbox lands, #5714)
 	@bun tests/bugbash/run.ts $(BUGBASH_ARGS)
 
@@ -261,11 +261,14 @@ test-bugbash-repros: build-main build-renderer build-static ## Bug-bash repro te
 test-bugbash-known-failures: build-main build-renderer build-static ## Bug-bash repros of open bugs: each fails until its issue is fixed
 	@export BUGBASH_AI_RESOLVED=mock BUGBASH_AI_REASON="known-failure repros"; $(BUGBASH_REPRO_RUN) --tag known-failure --output .e2e/repros-known $(BUGBASH_REPRO_ARGS)
 
-# e2e needs Node 22.22.3+/24.8+: E2E_NODE, else the first node on PATH. Its directory leads PATH so
-# the app command and its children use the same node (as tests/bugbash/run.ts does).
-mcp-apps-e2e: ## MCP Apps e2e suite, agent.act driven (paused on the host until the sandbox lands, #5714)
-	@# Its seed writes the MCP chat directly, so it runs on the mock app AI (tests/bugbash/aiMode.ts).
-	@export BUGBASH_AI_RESOLVED=mock BUGBASH_AI_REASON="MCP Apps suite"; node="$${E2E_NODE:-$$(command -v node)}"; cd tests/bugbash && PATH="$$(dirname "$$node"):$$PATH" E2E_TELEMETRY_DISABLED=1 "$$node" ../../node_modules/e2e/dist/cli/bin.js run --config e2e.mcpapps.config.ts $(MCP_APPS_E2E_ARGS)
+# The MCP Apps suite runs only in the bug-bash sandbox (tests/bugbash/sandbox/launch.ts, #5714).
+# agent.act steps reach BUGBASH_MODEL through a provider proxy for the job, so the host needs
+# BUGBASH_BUDGET_USD (a list-price cap) and ANTHROPIC_API_KEY plus ANTHROPIC_BASE_URL. Its seed
+# writes the MCP chat directly, so the app AI stays the mock. The container mounts dist/.
+mcp-apps-e2e: build-main build-renderer build-static ## MCP Apps e2e suite in the bug-bash sandbox (needs BUGBASH_BUDGET_USD)
+	@# The app AI is pinned to the mock, also over an exported BUGBASH_AI_RESOLVED. The shell PID
+	@# keeps two runs that start in the same second apart (their output and proxy record).
+	@cd tests/bugbash && BUGBASH_AI=mock BUGBASH_AI_RESOLVED=mock BUGBASH_AI_REASON="MCP Apps suite" bun sandbox/launch.ts -- run --config e2e.mcpapps.config.ts --output .e2e/mcp-apps-$$(date -u +%Y%m%dT%H%M%SZ)-$$$$ $(MCP_APPS_E2E_ARGS)
 
 rlm-eval: ## Run the RLM lever eval against a running dev-server sandbox (see scripts/rlm-eval/run.ts header)
 	@bun run scripts/rlm-eval/run.ts $(RLM_EVAL_ARGS)

tests/bugbash/e2e.mcpapps.config.ts

@@ -6,12 +6,29 @@
  * --charters tests/bugbash/mcpapps/charters.txt"`.
  *
  * Paused on the host (hostPause.ts, #5714): the suite drives every flow with `agent.act`, so a
- * model picks its actions. Every e2e command with this config, also `run`, refuses as it loads.
+ * model picks its actions. Every e2e command with this config, also `run`, refuses as it loads,
+ * except inside the bug-bash sandbox with a provider proxy for the job (`make mcp-apps-e2e`).
+ * There the agents get BUGBASH_MODEL through that proxy, and the app AI stays the mock.
  */
 // First import: the pause refuses before e2e.config.ts runs (mcpapps/hostPause.ts).
 import "./mcpapps/hostPause";
+import { createAnthropic } from "@ai-sdk/anthropic";
 import type { E2EConfig } from "e2e";
 import base from "./e2e.config";
+import { PROXY_BASE_URL } from "./sandbox/inContainer";
+
+/**
+ * The explorer model, set by the launcher. The proxy holds the provider key and allows only
+ * this job's models (sandbox/proxy.ts), so the key here is a placeholder the proxy drops.
+ */
+function explorerModel() {
+  const spec = process.env.BUGBASH_MODEL ?? "";
+  const [provider, id] = spec.split(/:(.*)/s, 2);
+  if (provider !== "anthropic" || !id)
+    throw new Error(`BUGBASH_MODEL must be anthropic:<model> in the sandbox, got "${spec}"`);
+  return createAnthropic({ baseURL: PROXY_BASE_URL, apiKey: "bugbash-sandbox-placeholder" })(id);
+}
+const model = explorerModel();
 
 const mcpContext = [
   "MCP Apps setup for this run: the 'Bug bash playground' chat already holds MCP tool calls from",
@@ -27,12 +44,12 @@ const mcpContext = [
   "Tutorial popovers can cover controls: dismiss them with Skip.",
 ].join(" ");
 
-const agents = Object.fromEntries(
+const agents: E2EConfig["agents"] = Object.fromEntries(
   Object.entries(base.agents).map(([name, agent]) => [
     name,
-    { ...agent, context: `${agent.context} ${mcpContext}` },
+    { ...agent, model, context: `${agent.context} ${mcpContext}` },
   ])
-) as typeof base.agents;
+);
 
 const targets = base.targets.map((target) => ({
   ...target,
@@ -41,6 +58,10 @@ const targets = base.targets.map((target) => ({
     command: {
       ...target.app.command,
       args: ["startApp.ts", "--port", "{port}", "--mcp-apps"],
+      // e2e gives the app only `command.env`, and this config loads only in a model-driven
+      // sandbox job (mcpapps/hostPause.ts). Without this marker startApp.ts would leave agent
+      // tools, terminals and project automation on while the explorer drives the app (S3).
+      env: { ...target.app.command.env, BUGBASH_MODEL_DRIVEN: "1" },
     },
   },
 })) as typeof base.targets;

tests/bugbash/hostPause.test.ts

@@ -104,6 +104,12 @@ test.each([
     { BUGBASH_AI: "mock" },
   ],
   ["e2e run, MCP Apps", e2e("run", "--config", "e2e.mcpapps.config.ts"), { BUGBASH_AI: "mock" }],
+  [
+    // The sandbox env names alone do not pass: this host has no docker-init and no proxy socket.
+    "e2e run, MCP Apps, sandbox env on the host",
+    e2e("run", "--config", "e2e.mcpapps.config.ts"),
+    { BUGBASH_AI: "mock", BUGBASH_CONTAINER: "1", BUGBASH_MODEL_DRIVEN: "1" },
+  ],
   // No app AI mode: e2e.config.ts would throw its own error first, without the pause.
   ["e2e list, MCP Apps, no app AI mode", e2e("list", "--config", "e2e.mcpapps.config.ts"), {}],
   [

tests/bugbash/mcpapps/hostPause.ts

@@ -2,8 +2,15 @@
  * The MCP Apps pause (#5714, ../hostPause.ts). e2e.mcpapps.config.ts imports this module first,
  * so the refusal comes before e2e.config.ts runs: that config can throw first (for example on
  * an unresolved app AI mode), and then no message would name the pause.
+ *
+ * The suite runs only in the bug-bash sandbox with a provider proxy for the job
+ * (`make mcp-apps-e2e`, sandbox/launch.ts). Everywhere else it refuses as before.
  */
 import { modelDrivenRefusal } from "../hostPause";
+import { modelDrivenSandbox } from "../sandbox/inContainer";
 
-const paused = modelDrivenRefusal("the MCP Apps suite (agent.act, e2e.mcpapps.config.ts)");
-if (paused != null) throw new Error(paused);
+if (!modelDrivenSandbox()) {
+  throw new Error(
+    modelDrivenRefusal("the MCP Apps suite (agent.act, e2e.mcpapps.config.ts)") ?? "refused"
+  );
+}

tests/bugbash/sandbox/entry.ts

@@ -8,13 +8,21 @@
  *
  * Before the job starts, it checks that the daemon runs on the launcher's host: the same kernel
  * (boot id) and the same files (a nonce that the launcher wrote into the staged copy).
+ *
+ * A model-driven job (BUGBASH_MODEL_DRIVEN=1) also needs the job's provider proxy socket, and it
+ * gets a TCP forwarder to it on 127.0.0.1 (inContainer.ts). The app AI stays the mock.
  */
 import { spawn } from "child_process";
 import * as fs from "fs";
 import * as os from "os";
 import { writeExport } from "./exportStream";
+import { forwardToProxy, inSandbox, modelDrivenSandbox } from "./inContainer";
 
-assertInSandbox(); // first: on a host, kill(-1) below hits every process of this user
+// First: on a host, kill(-1) below hits every process of this user.
+if (!inSandbox()) {
+  console.error("sandbox entry: not in the bug-bash sandbox, so it refuses to run");
+  process.exit(2);
+}
 
 const args = process.argv.slice(2);
 if (args[0] !== "--export" || args[2] !== "--" || args.length < 4) {
@@ -34,6 +42,13 @@ if (boot !== process.env.BUGBASH_HOST_BOOT || nonce !== process.env.BUGBASH_HOST
   process.exit(2);
 }
 const [command, ...commandArgs] = args.slice(3);
+if (process.env.BUGBASH_MODEL_DRIVEN === "1") {
+  if (!modelDrivenSandbox()) {
+    console.error("sandbox entry: a model-driven job without its proxy socket does not run");
+    process.exit(2);
+  }
+  await forwardToProxy(); // ends with this process, after the export
+}
 
 /** Linux skips PID 1 and the caller. When this process exits, docker-init exits too. */
 function killAllOthers(): void {
@@ -90,15 +105,3 @@ job.on("error", (error) => {
 job.on("exit", (code, signal) =>
   done(code ?? (signal != null ? 128 + os.constants.signals[signal] : 1))
 );
-
-function assertInSandbox(): void {
-  const init = fs.readFileSync("/proc/1/cmdline", "utf8");
-  const ok =
-    process.env.BUGBASH_CONTAINER === "1" &&
-    init.startsWith("/sbin/docker-init\0") &&
-    fs.readdirSync("/sys/class/net").join() === "lo";
-  if (!ok) {
-    console.error("sandbox entry: not in the bug-bash sandbox, so it refuses to run");
-    process.exit(2);
-  }
-}

tests/bugbash/sandbox/fakeUpstream.ts

@@ -9,7 +9,15 @@
  * `stream: true` request gets a complete SSE stream and any other request a JSON message.
  * `mode` applies a misbehavior to requests without a marker (the proxy's probe sends none).
  * `[fake:cut]` on a JSON request sends the headers and part of the body, then drops.
+ * `[fake:lowered]` and `[fake:nulled]` end a stream normally, but its message_delta lowers or
+ * nulls a count that message_start reported.
+ *
+ * As a command (the zero-cost transport check of the sandbox, plan section 9):
+ *   bun sandbox/fakeUpstream.ts --port 0 --record <calls.jsonl>
+ * prints `listening on 127.0.0.1:<port>` and appends one line per request: the model, the body
+ * keys, the betas and the tool and message counts, never the text.
  */
+import * as fs from "node:fs";
 import * as http from "node:http";
 import type { AddressInfo } from "node:net";
 
@@ -30,15 +38,17 @@ export interface FakeRequest {
 const sse = (type: string, data: object) =>
   `event: ${type}\ndata: ${JSON.stringify({ type, ...data })}\n\n`;
 
-export async function startFakeUpstream() {
+export async function startFakeUpstream(port = 0, onRequest?: (request: FakeRequest) => void) {
   const requests: FakeRequest[] = [];
   const state: { mode?: string } = {};
   const server = http.createServer((req, res) => {
     const chunks: Buffer[] = [];
     req.on("data", (chunk: Buffer) => chunks.push(chunk));
     req.on("end", () => {
       const body = Buffer.concat(chunks).toString();
-      requests.push({ method: req.method ?? "", url: req.url ?? "", headers: req.headers, body });
+      const request = { method: req.method ?? "", url: req.url ?? "", headers: req.headers, body };
+      requests.push(request);
+      onRequest?.(request);
       const mode = /\[fake:(\w+)\]/.exec(body)?.[1] ?? state.mode;
       const parsed = JSON.parse(body || "{}") as { model?: string; stream?: boolean };
       const model = parsed.model ?? "claude-haiku-4-5";
@@ -78,10 +88,10 @@ export async function startFakeUpstream() {
       finish(res, mode);
     });
   });
-  await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
-  const { port } = server.address() as AddressInfo;
+  await new Promise<void>((resolve) => server.listen(port, "127.0.0.1", resolve));
+  const address = server.address() as AddressInfo;
   return {
-    baseUrl: `http://127.0.0.1:${port}`,
+    baseUrl: `http://127.0.0.1:${address.port}`,
     requests,
     setMode: (mode?: string) => (state.mode = mode),
     close: () => {
@@ -103,10 +113,40 @@ function finish(res: http.ServerResponse, mode: string | undefined) {
   res.write(
     sse("message_delta", {
       delta: { stop_reason: "end_turn" },
-      usage: { output_tokens: FAKE_USAGE.output_tokens },
+      usage: {
+        output_tokens: FAKE_USAGE.output_tokens,
+        ...(mode === "lowered" && { input_tokens: 1 }),
+        // 0 in message_start, so only the null itself is wrong here.
+        ...(mode === "nulled" && { cache_creation_input_tokens: null }),
+      },
     })
   );
   if (mode === "cut") return res.destroy();
   if (mode === "nostop") return res.end();
   res.end(sse("message_stop", {}));
 }
+
+if (import.meta.main) {
+  const flag = (name: string) => process.argv[process.argv.indexOf(name) + 1];
+  const record = flag("--record");
+  if (!process.argv.includes("--record") || record == null) throw new Error("--record <file>");
+  const fake = await startFakeUpstream(Number(flag("--port") ?? 0), (request) => {
+    let body: Record<string, unknown> = {};
+    try {
+      body = JSON.parse(request.body) as typeof body;
+    } catch {
+      // recorded as an empty key list
+    }
+    const line = {
+      url: request.url,
+      model: body.model,
+      keys: Object.keys(body).sort(),
+      betas: request.headers["anthropic-beta"] ?? null,
+      stream: body.stream === true,
+      tools: Array.isArray(body.tools) ? body.tools.length : 0,
+      messages: Array.isArray(body.messages) ? body.messages.length : 0,
+    };
+    fs.appendFileSync(record, `${JSON.stringify(line)}\n`);
+  });
+  console.log(`listening on ${fake.baseUrl.replace("http://", "")}`);
+}

tests/bugbash/sandbox/inContainer.test.ts

@@ -0,0 +1,70 @@
+import { afterEach, expect, test } from "bun:test";
+import * as fs from "node:fs";
+import * as net from "node:net";
+import * as os from "node:os";
+import * as path from "node:path";
+import { forwardToProxy, inSandbox, modelDrivenSandbox, PROXY_SOCKET } from "./inContainer";
+
+const cleanups: (() => unknown)[] = [];
+afterEach(async () => {
+  for (const cleanup of cleanups.splice(0).reverse()) await cleanup();
+});
+
+/** A fake filesystem root that looks like the sandbox container, with a listening proxy socket. */
+async function fakeRoot(over: { init?: string; devices?: string[]; socket?: boolean } = {}) {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), "xbb-incontainer-"));
+  cleanups.push(() => fs.rmSync(root, { recursive: true, force: true }));
+  fs.mkdirSync(path.join(root, "proc/1"), { recursive: true });
+  fs.writeFileSync(path.join(root, "proc/1/cmdline"), over.init ?? "/sbin/docker-init\0--\0bun\0");
+  for (const device of over.devices ?? ["lo"])
+    fs.mkdirSync(path.join(root, "sys/class/net", device), { recursive: true });
+  fs.mkdirSync(path.dirname(path.join(root, PROXY_SOCKET)), { recursive: true });
+  if (over.socket !== false) {
+    const server = net.createServer((socket) => socket.end("proxy says hi"));
+    await new Promise<void>((resolve) => server.listen(path.join(root, PROXY_SOCKET), resolve));
+    cleanups.push(() => new Promise((done) => server.close(done)));
+  }
+  return root;
+}
+
+const SANDBOX_ENV = { BUGBASH_CONTAINER: "1", BUGBASH_MODEL_DRIVEN: "1" };
+
+test("model-driven jobs pass only in a sandbox container with a proxy socket", async () => {
+  expect(modelDrivenSandbox(SANDBOX_ENV, await fakeRoot())).toBe(true);
+  // This host: its PID 1 is no docker-init, and it has other network devices.
+  expect(inSandbox(SANDBOX_ENV)).toBe(false);
+  expect(modelDrivenSandbox(SANDBOX_ENV)).toBe(false);
+  const cases: [Record<string, string>, Parameters<typeof fakeRoot>[0]][] = [
+    [{ BUGBASH_MODEL_DRIVEN: "1" }, {}],
+    [{ BUGBASH_CONTAINER: "1" }, {}],
+    [SANDBOX_ENV, { init: "/sbin/init\0" }],
+    [SANDBOX_ENV, { devices: ["lo", "eth0"] }],
+    [SANDBOX_ENV, { socket: false }],
+  ];
+  for (const [env, over] of cases)
+    expect(modelDrivenSandbox(env, await fakeRoot(over))).toBe(false);
+});
+
+test("a regular file in place of the proxy socket does not count", async () => {
+  const root = await fakeRoot({ socket: false });
+  fs.writeFileSync(path.join(root, PROXY_SOCKET), "");
+  expect(modelDrivenSandbox(SANDBOX_ENV, root)).toBe(false);
+});
+
+test("the forwarder passes bytes both ways and closes with its server", async () => {
+  const root = await fakeRoot();
+  const server = await forwardToProxy(0, path.join(root, PROXY_SOCKET));
+  const { port } = server.address() as net.AddressInfo;
+  const reply = await new Promise<string>((resolve) => {
+    let text = "";
+    const socket = net.connect(port, "127.0.0.1", () => socket.write("hello"));
+    socket.on("data", (chunk) => (text += chunk.toString())).on("close", () => resolve(text));
+  });
+  expect(reply).toBe("proxy says hi");
+  await new Promise((done) => server.close(done));
+  const after = await new Promise<string>((resolve) => {
+    const socket = net.connect(port, "127.0.0.1", () => resolve("connected"));
+    socket.on("error", () => resolve("refused"));
+  });
+  expect(after).toBe("refused");
+});