🤖 tests: run make bug-bash in the bug-bash sandbox, with the mock app AI

Makefile

@@ -219,9 +219,12 @@ dev-desktop-sandbox: ## Start an isolated Electron dev instance (fresh XUM_ROOT
 dev-server-sandbox: ## Start an isolated dev-server instance (fresh XUM_ROOT + free ports)
 	@bun scripts/dev-server-sandbox.ts $(DEV_SERVER_SANDBOX_ARGS)
 
-# bug-bash is paused on the host (tests/bugbash/hostPause.ts, #5714): a model picks its actions.
-# Its recipe refuses at once, so it builds nothing first.
-bug-bash: ## Agent bug bash: e2e explore charters x models (paused on the host until the sandbox lands, #5714)
+# bug-bash runs each e2e explore charter in its own bug-bash sandbox container (tests/bugbash/run.ts,
+# #5714): a model picks its actions, so it never runs on the host. The explorers reach their models
+# through a provider proxy per job, so the host needs BUGBASH_BUDGET_USD (one list-price cap for
+# the run) and ANTHROPIC_API_KEY plus ANTHROPIC_BASE_URL. The app AI is the mock (run.ts refuses
+# another BUGBASH_AI). The containers mount dist/.
+bug-bash: build-main build-renderer build-static ## Agent bug bash in the bug-bash sandbox: e2e explore charters x models (needs BUGBASH_BUDGET_USD)
 	@bun tests/bugbash/run.ts $(BUGBASH_ARGS)
 
 # The bug-bash sandbox image (tests/bugbash/sandbox/build.sh, #5714). Each prints one value on

docs/AGENTS.md

@@ -247,8 +247,8 @@ Freely make breaking changes, and reorganize / cleanup IPC as needed.
 
 ## Bug Bash and Repro Tests (tests/bugbash)
 
-- Host pause (#5714): model-driven runs are paused on this host until they run in the bug-bash sandbox. `make bug-bash`, `make mcp-apps-e2e`, `tests/bugbash/run.ts`, any `e2e explore` and every e2e command with `e2e.mcpapps.config.ts` refuse with exit 2 before any app, browser or model starts (`tests/bugbash/hostPause.ts`). `e2e.config.ts` loads only for `e2e run` and `e2e list`, and its agents hold no model, so an `agent.*` step in a repro fails with `MODEL_UNAVAILABLE` and makes no model request. No override exists. Exact-step repros run as before.
-- `make bug-bash` runs AI explorers (`e2e explore` charters) against seeded `xum server`s (paused, see above). This section is the repo's rule set: `.xum/skills/bug-bash/` is vendored, so do not edit it; where the two differ (repros are tracked), follow this section.
+- Host pause (#5714): model-driven runs (`make bug-bash`, `make mcp-apps-e2e`, any `e2e explore` and every e2e command with `e2e.mcpapps.config.ts`) run only inside the bug-bash sandbox (`tests/bugbash/sandbox/launch.ts`, `tests/bugbash/run.ts`). On the host they still refuse with exit 2 before any app, browser or model starts (`tests/bugbash/hostPause.ts`). They need `BUGBASH_BUDGET_USD` plus `ANTHROPIC_API_KEY` and `ANTHROPIC_BASE_URL` on the host, and their app runs with agent tools, terminals and project automation off. `make bug-bash` runs the mock app AI (`BUGBASH_AI` unset or `mock`). Outside a sandboxed `e2e explore`, the agents of `e2e.config.ts` hold no model, so an `agent.*` step in a repro fails with `MODEL_UNAVAILABLE` and makes no model request. No override exists. Exact-step repros run as before.
+- `make bug-bash` runs AI explorers (`e2e explore` charters) against seeded `xum server`s, each in its own sandbox container (see above). This section is the repo's rule set: `.xum/skills/bug-bash/` is vendored, so do not edit it; where the two differ (repros are tracked), follow this section.
 - A finding is a bug only once a repro fails for the reported reason. Put it in `tests/bugbash/repros/<slug>.e2e.ts` (tracked) with exact `screen`/`browser`/`expect` steps, never `agent.*`, so it needs no model key and gives the same result every run. `tests/bugbash/reproRules.test.ts` fails on a repro that takes the `agent` fixture.
 - Prove every repro: restore the bug in a scratch edit, rebuild, see `ASSERTION_FAILED` on the bug's own assertion, then restore and see it pass.
 - Tags: `bugbash` plus the bug ID. Add `mock-only` when the test needs a `[mock:...]` prompt or a terminal (real mode turns terminals off). Add `known-failure` plus the issue number for an open bug, and remove that tag in the PR that fixes it.
@@ -260,8 +260,8 @@ Freely make breaking changes, and reorganize / cleanup IPC as needed.
 
 ## Agent E2E (tests/bugbash)
 
-- Paused on the host (#5714, see the host pause above): the targets below refuse until the bug-bash sandbox lands.
-- `make bug-bash` runs `e2e explore` charters against seeded servers (app AI mode: see the section above). `BUGBASH_ARGS="--config <file>"` picks another e2e config.
+- Sandbox only (#5714, see the host pause above): the targets below run in the bug-bash sandbox and refuse on the host.
+- `make bug-bash` runs `e2e explore` charters against seeded servers (app AI mode: see the section above). `BUGBASH_ARGS="--config tests/bugbash/e2e.mcpapps.config.ts"` picks the MCP Apps config, the only other config that runs.
 - MCP Apps: `make mcp-apps-e2e` runs `tests/bugbash/mcpapps/mcp-apps.e2e.ts` through `e2e.mcpapps.config.ts`; explore it with `BUGBASH_ARGS="--config tests/bugbash/e2e.mcpapps.config.ts --charters tests/bugbash/mcpapps/charters.txt"`. Its seed (`startApp.ts --mcp-apps`, `mcpapps/seed.ts`) writes the chat directly because mock AI cannot call MCP tools, so `make mcp-apps-e2e` pins the mock app AI.
 - Both need Node 22.22.3+ or 24.8+ (`E2E_NODE` or PATH) and an explorer model key (`tests/bugbash/e2e.config.ts`). They make paid model calls; no CI job runs them.
 - Suite tests drive flows with `agent.act` in on-screen words and pin each outcome with an exact `expect`. Use `agent.assert(..., { vision: "only" })` for "is it on screen": the accessibility tree also lists off-screen nodes. Prove a new test fails on the old `dist/` before the fix lands.

src/node/services/agentSkills/builtInSkillContent.generated.ts

@@ -2300,8 +2300,8 @@ export const BUILTIN_SKILL_FILES: Record<string, Record<string, string>> = {
       "",
       "## Bug Bash and Repro Tests (tests/bugbash)",
       "",
-      "- Host pause (#5714): model-driven runs are paused on this host until they run in the bug-bash sandbox. `make bug-bash`, `make mcp-apps-e2e`, `tests/bugbash/run.ts`, any `e2e explore` and every e2e command with `e2e.mcpapps.config.ts` refuse with exit 2 before any app, browser or model starts (`tests/bugbash/hostPause.ts`). `e2e.config.ts` loads only for `e2e run` and `e2e list`, and its agents hold no model, so an `agent.*` step in a repro fails with `MODEL_UNAVAILABLE` and makes no model request. No override exists. Exact-step repros run as before.",
-      "- `make bug-bash` runs AI explorers (`e2e explore` charters) against seeded `xum server`s (paused, see above). This section is the repo's rule set: `.xum/skills/bug-bash/` is vendored, so do not edit it; where the two differ (repros are tracked), follow this section.",
+      "- Host pause (#5714): model-driven runs (`make bug-bash`, `make mcp-apps-e2e`, any `e2e explore` and every e2e command with `e2e.mcpapps.config.ts`) run only inside the bug-bash sandbox (`tests/bugbash/sandbox/launch.ts`, `tests/bugbash/run.ts`). On the host they still refuse with exit 2 before any app, browser or model starts (`tests/bugbash/hostPause.ts`). They need `BUGBASH_BUDGET_USD` plus `ANTHROPIC_API_KEY` and `ANTHROPIC_BASE_URL` on the host, and their app runs with agent tools, terminals and project automation off. `make bug-bash` runs the mock app AI (`BUGBASH_AI` unset or `mock`). Outside a sandboxed `e2e explore`, the agents of `e2e.config.ts` hold no model, so an `agent.*` step in a repro fails with `MODEL_UNAVAILABLE` and makes no model request. No override exists. Exact-step repros run as before.",
+      "- `make bug-bash` runs AI explorers (`e2e explore` charters) against seeded `xum server`s, each in its own sandbox container (see above). This section is the repo's rule set: `.xum/skills/bug-bash/` is vendored, so do not edit it; where the two differ (repros are tracked), follow this section.",
       "- A finding is a bug only once a repro fails for the reported reason. Put it in `tests/bugbash/repros/<slug>.e2e.ts` (tracked) with exact `screen`/`browser`/`expect` steps, never `agent.*`, so it needs no model key and gives the same result every run. `tests/bugbash/reproRules.test.ts` fails on a repro that takes the `agent` fixture.",
       "- Prove every repro: restore the bug in a scratch edit, rebuild, see `ASSERTION_FAILED` on the bug's own assertion, then restore and see it pass.",
       "- Tags: `bugbash` plus the bug ID. Add `mock-only` when the test needs a `[mock:...]` prompt or a terminal (real mode turns terminals off). Add `known-failure` plus the issue number for an open bug, and remove that tag in the PR that fixes it.",
@@ -2313,8 +2313,8 @@ export const BUILTIN_SKILL_FILES: Record<string, Record<string, string>> = {
       "",
       "## Agent E2E (tests/bugbash)",
       "",
-      "- Paused on the host (#5714, see the host pause above): the targets below refuse until the bug-bash sandbox lands.",
-      '- `make bug-bash` runs `e2e explore` charters against seeded servers (app AI mode: see the section above). `BUGBASH_ARGS="--config <file>"` picks another e2e config.',
+      "- Sandbox only (#5714, see the host pause above): the targets below run in the bug-bash sandbox and refuse on the host.",
+      '- `make bug-bash` runs `e2e explore` charters against seeded servers (app AI mode: see the section above). `BUGBASH_ARGS="--config tests/bugbash/e2e.mcpapps.config.ts"` picks the MCP Apps config, the only other config that runs.',
       '- MCP Apps: `make mcp-apps-e2e` runs `tests/bugbash/mcpapps/mcp-apps.e2e.ts` through `e2e.mcpapps.config.ts`; explore it with `BUGBASH_ARGS="--config tests/bugbash/e2e.mcpapps.config.ts --charters tests/bugbash/mcpapps/charters.txt"`. Its seed (`startApp.ts --mcp-apps`, `mcpapps/seed.ts`) writes the chat directly because mock AI cannot call MCP tools, so `make mcp-apps-e2e` pins the mock app AI.',
       "- Both need Node 22.22.3+ or 24.8+ (`E2E_NODE` or PATH) and an explorer model key (`tests/bugbash/e2e.config.ts`). They make paid model calls; no CI job runs them.",
       '- Suite tests drive flows with `agent.act` in on-screen words and pin each outcome with an exact `expect`. Use `agent.assert(..., { vision: "only" })` for "is it on screen": the accessibility tree also lists off-screen nodes. Prove a new test fails on the old `dist/` before the fix lands.',

tests/bugbash/e2e.config.ts

@@ -7,13 +7,16 @@
  *
  * `e2e explore` charters (`make bug-bash`) are paused on the host (hostPause.ts, #5714): this
  * config refuses every e2e command except `run` and `list` as it loads, before any app, browser
- * or model starts. Its agents keep their personas but hold no model, so an `agent.*` step in a
- * repro fails with MODEL_UNAVAILABLE before any model request. The sandbox work (#5714) brings
- * the explorer model (`BUGBASH_MODEL`, `BUGBASH_EFFORT`) back.
+ * or model starts. Only inside the bug-bash sandbox with a provider proxy for the job does
+ * `explore` pass, and only there do the agents get the explorer model (`BUGBASH_MODEL` through
+ * the proxy, sandbox/explorerModel.ts; reasoning `BUGBASH_EFFORT`). Everywhere else they keep
+ * their personas but hold no model, so an `agent.*` step in a repro fails with
+ * MODEL_UNAVAILABLE before any model request.
  */
 import type { E2EConfig } from "e2e";
 import { web } from "@e2e-dev/web";
 import { e2eCommandRefusal } from "./hostPause";
+import { explorerModel } from "./sandbox/explorerModel";
 
 // First, before anything else in this config runs.
 const paused = e2eCommandRefusal();
@@ -149,9 +152,11 @@ const context = scenario === "bash-ai-proxy" ? bashAiProxyContext : mockOrRealCo
 // Exploration steps need large per-step budgets (`e2e guide bug-bash`, step 1).
 // Only the active provider reads its key, so both can be set at once.
 const effort = explorerEffort();
-// No `model` during the host pause (hostPause.ts): explore is refused above, and an agent.* step
-// in a repro must find no model to call.
+// A model only for `e2e explore` in a model-driven sandbox job: an agent.* step in a repro must
+// find no model to call (hostPause.ts).
+const model = explorerModel();
 const persona = {
+  ...(model && { model }),
   maxSteps: 40,
   maxModelCalls: 40,
   context,
@@ -166,6 +171,8 @@ const APP_ENV_VARS = [
   "BUGBASH_AI_REASON",
   "BUGBASH_APP_MODEL",
   "BUGBASH_SCENARIO",
+  // A model-driven sandbox job: startApp.ts turns off agent tools, terminals and automation.
+  "BUGBASH_MODEL_DRIVEN",
   "ANTHROPIC_API_KEY",
   "ANTHROPIC_BASE_URL",
   "OPENAI_API_KEY",

tests/bugbash/e2e.mcpapps.config.ts

@@ -12,23 +12,12 @@
  */
 // First import: the pause refuses before e2e.config.ts runs (mcpapps/hostPause.ts).
 import "./mcpapps/hostPause";
-import { createAnthropic } from "@ai-sdk/anthropic";
 import type { E2EConfig } from "e2e";
 import base from "./e2e.config";
-import { PROXY_BASE_URL } from "./sandbox/inContainer";
+import { proxyModel } from "./sandbox/explorerModel";
 
-/**
- * The explorer model, set by the launcher. The proxy holds the provider key and allows only
- * this job's models (sandbox/proxy.ts), so the key here is a placeholder the proxy drops.
- */
-function explorerModel() {
-  const spec = process.env.BUGBASH_MODEL ?? "";
-  const [provider, id] = spec.split(/:(.*)/s, 2);
-  if (provider !== "anthropic" || !id)
-    throw new Error(`BUGBASH_MODEL must be anthropic:<model> in the sandbox, got "${spec}"`);
-  return createAnthropic({ baseURL: PROXY_BASE_URL, apiKey: "bugbash-sandbox-placeholder" })(id);
-}
-const model = explorerModel();
+// Every command with this config is model-driven (`run` too), so it always gets the model.
+const model = proxyModel();
 
 const mcpContext = [
   "MCP Apps setup for this run: the 'Bug bash playground' chat already holds MCP tool calls from",

tests/bugbash/hostPause.test.ts

@@ -117,12 +117,6 @@ test.each([
     e2e("explore", "--config", "e2e.config.ts", "x"),
     { BUGBASH_AI_RESOLVED: "real" },
   ],
-  [
-    // Unguarded, run.ts probes the app model first (BUGBASH_AI=auto) and then starts e2e-node.
-    "run.ts (make bug-bash)",
-    [process.execPath, "run.ts", "--only", "composer"],
-    {},
-  ],
 ] as const)(
   "%s refuses before any app, e2e or model starts",
   async (_name, command, env) => {
@@ -147,3 +141,17 @@ test("control: exact-step repros still load and list", async () => {
   expect(r.output).toMatch(/^repros\/\S+\.e2e\.ts › /m);
   expect(r.output).not.toContain("#5714");
 });
+
+// run.ts runs no e2e on the host any more: each charter is a sandbox job (sandbox/launch.ts).
+// Without a budget it refuses before any job, e2e process or model request.
+test("run.ts (make bug-bash) without a budget refuses before any app, e2e or model starts", async () => {
+  const r = await run([process.execPath, "run.ts", "--only", "composer"], {
+    E2E_NODE: path.join(bin, "e2e-node"),
+  });
+  expect({ status: r.status, started: r.started, modelRequests }).toEqual({
+    status: 2,
+    started: "",
+    modelRequests: 0,
+  });
+  expect(r.output).toContain("BUGBASH_BUDGET_USD");
+}, 60_000);

tests/bugbash/hostPause.ts

@@ -10,11 +10,14 @@
  * repro fails with MODEL_UNAVAILABLE before any model request (e2e has no default model and no
  * env fallback).
  *
- * The pause has no override: no env var or flag turns it off. It ends when the sandbox lands.
+ * The pause has no override: no env var or flag turns it off. Model-driven runs pass only
+ * inside the bug-bash sandbox with a provider proxy for the job (sandbox/inContainer.ts):
+ * `e2e explore` there (`make bug-bash`, run.ts), and the MCP Apps suite (mcpapps/hostPause.ts).
  */
 import * as fs from "node:fs";
 import * as path from "node:path";
 import { fileURLToPath } from "node:url";
+import { modelDrivenSandbox } from "./sandbox/inContainer";
 
 /** Why `what` must not run, or null when it may run. During the pause it never may. */
 export function modelDrivenRefusal(what: string): string | null {
@@ -42,19 +45,31 @@ const real = (file: string | undefined) => {
   }
 };
 
+/** The e2e command (`run`, `explore`, ...) of an e2e CLI process, or undefined. */
+export function e2eCommand(argv: readonly string[] = process.argv): string | undefined {
+  const main = real(argv[1]);
+  if (main == null || main !== real(E2E_CLI)) return undefined;
+  return argv.slice(2).find((arg) => !arg.startsWith("-"));
+}
+
 /**
  * Why the e2e process that loads a bug-bash config must stop, or null. Every e2e command loads
  * the config in its own process before it starts an app, a browser or a model, so this check
  * runs first. Fail closed: a config loaded by anything other than the e2e CLI or its worker
- * (another script, `import()` from a tool) is refused too.
+ * (another script, `import()` from a tool) is refused too. `env` and `root` are for tests.
  */
-export function e2eCommandRefusal(argv: readonly string[] = process.argv): string | null {
+export function e2eCommandRefusal(
+  argv: readonly string[] = process.argv,
+  env: NodeJS.ProcessEnv = process.env,
+  root = "/"
+): string | null {
   const main = real(argv[1]);
   // A worker gets no args. Only a CLI run that passed this check forks one.
   if (main != null && main === real(E2E_WORKER)) return null;
   if (main == null || main !== real(E2E_CLI))
     return modelDrivenRefusal(`a bug-bash e2e config loaded by ${argv[1] ?? "an unknown script"}`);
-  const command = argv.slice(2).find((arg) => !arg.startsWith("-"));
+  const command = e2eCommand(argv);
   if (command != null && EXACT_STEP_COMMANDS.includes(command)) return null;
+  if (command === "explore" && modelDrivenSandbox(env, root)) return null;
   return modelDrivenRefusal(`e2e ${command ?? "(no command)"}`);
 }