🤖 tests: the bug bash defaults to Sonnet alone and refuses a budget below its estimate

tests/bugbash/run.test.ts

@@ -3,7 +3,7 @@ import * as childProcess from "node:child_process";
 import * as fs from "node:fs";
 import * as os from "node:os";
 import * as path from "node:path";
-import { type RunDeps, runBugBash } from "./run";
+import { estimateRun, type RunDeps, runBugBash } from "./run";
 import type { JobOutcome, LaunchOptions } from "./sandbox/launch";
 import type { Ledger } from "./sandbox/proxyPolicy";
 import { Refusal, Stopped } from "./sandbox/runner";
@@ -12,7 +12,8 @@ import { Refusal, Stopped } from "./sandbox/runner";
 // (sandbox/runner.test.ts) cover the job itself.
 const ENV = {
   BUGBASH_MODELS: "anthropic:claude-sonnet-5-5",
-  BUGBASH_BUDGET_USD: "2",
+  // Above the recommended budget of every run here (estimateRun): up to 4 charters at once.
+  BUGBASH_BUDGET_USD: "10",
   ANTHROPIC_API_KEY: "sk-run-test",
   ANTHROPIC_BASE_URL: "http://127.0.0.1:9/v1",
 };
@@ -93,7 +94,7 @@ test("B2: every charter runs through the launcher with one shared ledger, none o
     expect(spawn).not.toHaveBeenCalled();
     const findings = lines.find((line) => line.startsWith("Findings: "))!.slice(10);
     expect(fs.readFileSync(findings, "utf8")).toMatch(
-      /Cost: 0 proxied calls, 0 refused by the budget, \$0\.0000 of \$2\.0000 at list price/
+      /Cost: 0 proxied calls, 0 refused by the budget, \$0\.0000 of \$10\.0000 at list price/
     );
   } finally {
     spawn.mockRestore();
@@ -316,7 +317,10 @@ test("U1: Docker is checked once, before the probe and any folder; --preflight s
   expect(checks).toBe(1);
   expect(dead.probes).toHaveLength(0);
   expect(dead.calls).toHaveLength(0);
-  expect(dead.lines).toEqual(["make bug-bash: docker info: no daemon answered"]);
+  // The cost estimate prints first (U4); the only refusal is Docker's.
+  expect(dead.lines.filter((line) => line.startsWith("make bug-bash:"))).toEqual([
+    "make bug-bash: docker info: no daemon answered",
+  ]);
 
   const live = fakeLaunch(() => ({ code: 0, cleanup: "removed" }));
   let liveChecks = 0;
@@ -341,7 +345,7 @@ test("U1: a probe the budget refused (HTTP 402) blames the budget, not the key",
   const stop = new AbortController().signal;
   expect(await runBugBash([...charters("a")], ENV, stop, deps)).toBe(2);
   expect(calls).toHaveLength(0);
-  expect(lines.join("\n")).toMatch(/BUGBASH_BUDGET_USD \(\$2\) is too small to cover/);
+  expect(lines.join("\n")).toMatch(/BUGBASH_BUDGET_USD \(\$10\) is too small to cover/);
   expect(lines.join("\n")).not.toMatch(/Fix the key/);
 });
 
@@ -393,9 +397,86 @@ test.each([
     const stop = new AbortController().signal;
     expect(await runBugBash([...charters("a")], { ...ENV, ...over }, stop, deps)).toBe(2);
     expect(calls).toHaveLength(0);
-    // It names the variable and the host to check, never the URL's path.
-    expect(lines).toHaveLength(1);
-    expect(lines[0]).toContain("ANTHROPIC_BASE_URL (127.0.0.1:9)");
-    expect(lines[0]).not.toContain("/v1");
+    // One refusal (after U4's estimate line). It names the variable and the host to check,
+    // never the URL's path.
+    const refusals = lines.filter((line) => line.startsWith("make bug-bash:"));
+    expect(refusals).toHaveLength(1);
+    expect(refusals[0]).toContain("ANTHROPIC_BASE_URL (127.0.0.1:9)");
+    expect(refusals[0]).not.toContain("/v1");
   }
 );
+
+const SONNET = "anthropic:claude-sonnet-5-5";
+const OPUS = "anthropic:claude-opus-5-5";
+const run6 = { models: [SONNET], charters: 6, parallel: 8, maxSteps: 6, effort: "medium" };
+
+test("U4: the estimate scales the measured cost, and its headroom covers the calls in flight", () => {
+  const base = estimateRun(run6);
+  // Sonnet's measured $0.45 per charter; 6 calls in flight at Sonnet's worst bound.
+  expect(base.estimateUsd).toBeCloseTo(6 * 0.45);
+  expect(base.inFlight).toBe(6);
+  expect(base.recommendedUsd).toBe(Math.ceil(base.estimateUsd + base.headroomUsd));
+  expect(base.perModel[0].measured).toBe(true);
+  // Twice the steps cost twice as much; fewer at once need less headroom, not less estimate.
+  expect(estimateRun({ ...run6, maxSteps: 12 }).estimateUsd).toBeCloseTo(2 * base.estimateUsd);
+  const two = estimateRun({ ...run6, parallel: 2 });
+  expect(two.estimateUsd).toBeCloseTo(base.estimateUsd);
+  expect(two.headroomUsd).toBeCloseTo((base.headroomUsd * 2) / 6);
+  // An unmeasured model is a guess above its list-price share; its bound sets the headroom.
+  const both = estimateRun({ ...run6, models: [SONNET, OPUS] });
+  expect(both.perModel[1].measured).toBe(false);
+  expect(both.perModel[1].perJob).toBeGreaterThan(2 * base.perModel[0].perJob); // Opus is 2x list
+  expect(both.headroomUsd).toBeGreaterThan(base.headroomUsd * (8 / 6));
+  // A higher effort is an unmeasured guess that costs more; it reserves no more per call.
+  const high = estimateRun({ ...run6, effort: "high" });
+  expect(high.estimateUsd).toBeGreaterThan(base.estimateUsd);
+  expect(high.perModel[0].measured).toBe(false);
+  expect(high.headroomUsd).toBeCloseTo(base.headroomUsd);
+  // Low effort is not measured either, and it is not priced below medium.
+  const low = estimateRun({ ...run6, effort: "low" });
+  expect(low.perModel[0].measured).toBe(false);
+  expect(low.estimateUsd).toBeCloseTo(base.estimateUsd);
+  // A real app AI with a costlier model adds its share and its larger bound; Haiku adds nothing.
+  const opusApp = estimateRun({ ...run6, appModel: OPUS });
+  expect(opusApp.estimateUsd).toBeGreaterThan(base.estimateUsd);
+  expect(opusApp.headroomUsd).toBeGreaterThan(base.headroomUsd);
+  const haiku = estimateRun({ ...run6, appModel: "anthropic:claude-haiku-4-5" });
+  expect(haiku.recommendedUsd).toBe(base.recommendedUsd);
+});
+
+test("U4: the default runs Sonnet alone, and a budget below the estimate refuses before the probe", async () => {
+  const stop = new AbortController().signal;
+  const { BUGBASH_MODELS: _models, ...defaults } = ENV;
+  const six = charters("a", "b", "c", "d", "e", "f");
+  const need = estimateRun(run6).recommendedUsd;
+  const low = fakeLaunch(() => ({ code: 0, cleanup: "removed" }));
+  const below = { ...defaults, BUGBASH_BUDGET_USD: String(need - 1) };
+  expect(await runBugBash(six, below, stop, low.deps)).toBe(2);
+  expect(low.calls).toHaveLength(0);
+  expect(low.probes).toHaveLength(0);
+  // The refusal names the variable and the number to set.
+  const refusal = low.lines.find((line) => line.includes("BUGBASH_BUDGET_USD"))!;
+  expect(refusal).toContain(`$${need}`);
+
+  const enough = fakeLaunch(() => ({ code: 0, cleanup: "removed" }));
+  const at = { ...defaults, BUGBASH_BUDGET_USD: String(need) };
+  expect(await runBugBash(six, at, stop, enough.deps)).toBe(0);
+  expect(new Set(enough.calls.map((c) => c.o.env.BUGBASH_MODEL))).toEqual(new Set([SONNET]));
+  expect(enough.calls).toHaveLength(6);
+});
+
+test("U4: the run passes its effort and a possibly real app model to the estimate", async () => {
+  const stop = new AbortController().signal;
+  // A budget that fits medium effort with the Haiku app, but not high effort or an Opus app.
+  const fits = estimateRun({ ...run6, charters: 1 }).recommendedUsd;
+  const env = { ...ENV, BUGBASH_BUDGET_USD: String(fits) };
+  for (const [over, code] of [
+    [{}, 0],
+    [{ BUGBASH_EFFORT: "max" }, 2],
+    [{ BUGBASH_APP_MODEL: OPUS }, 2],
+    [{ BUGBASH_APP_MODEL: OPUS, BUGBASH_AI: "mock" }, 0], // a mock app makes no calls
+  ] as const) {
+    const { deps } = fakeLaunch(() => ({ code: 0, cleanup: "removed" }));
+    expect(await runBugBash([...charters("a")], { ...env, ...over }, stop, deps)).toBe(code);
+  }
+});

tests/bugbash/run.ts

@@ -17,10 +17,15 @@
  *   e2e.config.ts. No other config runs in the sandbox (sandbox/inputs.ts exploreRefusal).
  *   Paths are relative to the current directory.
  *
- * Models: BUGBASH_MODELS, comma-separated `anthropic:<model>`, default Opus 5.5 and Sonnet 5.5.
- * In a three-model comparison (2026-10) these two found 14 of 15 distinct bugs and overlapped on
- * only 4, so the default runs both; GPT-6.1 Sol found 1. Every model runs every charter, all in
- * one pool of --parallel explorers. Reasoning: BUGBASH_EFFORT, default medium (e2e.config.ts).
+ * Models: BUGBASH_MODELS, comma-separated `anthropic:<model>`, default Sonnet 5.5 alone (6 jobs).
+ * In a three-model comparison (2026-10) Opus 5.5 and Sonnet 5.5 together found 14 of 15 distinct
+ * bugs and overlapped on only 4; GPT-6.1 Sol found 1. Opus about doubles the cost of a run, so it
+ * is opt-in: BUGBASH_MODELS=anthropic:claude-sonnet-5-5,anthropic:claude-opus-5-5. Every model
+ * runs every charter, all in one pool of --parallel explorers. Reasoning: BUGBASH_EFFORT, default
+ * medium (e2e.config.ts).
+ *
+ * Cost: before any job, the run prints its estimate and a recommended budget (estimateRun), and
+ * refuses a BUGBASH_BUDGET_USD below it.
  *
  * Charters: tests/bugbash/charters.txt. Config and personas: tests/bugbash/e2e.config.ts.
  * Output: tests/bugbash/.e2e/bugbash/<run>/<model>/ with one directory, log and app log per
@@ -56,7 +61,7 @@ import {
   probeApp,
   probeUnreachable,
 } from "./sandbox/launch";
-import { Ledger, priced } from "./sandbox/proxyPolicy";
+import { Ledger, listPrice, priced, worstCaseNanoUsd } from "./sandbox/proxyPolicy";
 import { Refusal, Stopped } from "./sandbox/runner";
 
 const projectDir = import.meta.dir;
@@ -232,8 +237,84 @@ function writeFindings(
   fs.writeFileSync(file, `${lines.join("\n")}\n`);
 }
 
-/** Opus 5.5 and Sonnet 5.5: see the header comment for why both. */
-const DEFAULT_MODELS = ["anthropic:claude-opus-5-5", "anthropic:claude-sonnet-5-5"];
+/** Sonnet 5.5 alone: see the header comment for why Opus is opt-in. */
+const DEFAULT_MODELS = ["anthropic:claude-sonnet-5-5"];
+
+/**
+ * Measured list-price cost of one charter at --max-steps 6 and medium effort, explorer and app AI
+ * together: four `make bug-bash` runs of Sonnet 5.5 on #5714, #6076 and #6079 averaged $0.34 to
+ * $0.52 per charter (21 charters, $9.50 in all).
+ */
+const MEASURED_USD_PER_JOB: Record<string, number> = { "claude-sonnet-5-5": 0.45 };
+const MEASURED_STEPS = 6;
+/** An unmeasured model: Sonnet's measured cost, scaled by list price, times this margin. */
+const UNMEASURED_MARGIN = 2;
+/**
+ * Explorer cost per charter relative to medium effort, where it was measured. Higher efforts add
+ * thinking tokens; nothing measured them, so these are guesses, and the run labels them so. Low
+ * is not lowered: an estimate must not undercount.
+ */
+const EFFORT_FACTOR: Record<string, number> = { low: 1, medium: 1, high: 2, xhigh: 3, max: 4 };
+
+/**
+ * The real app AI's own share per charter with Haiku 4.5, the default app model: #6079's five
+ * real-mode charters spent 19,545 input and 1,022 output tokens on it, about $0.005 each. That
+ * share is already in the explorer measurement above, so only another app model adds to it.
+ */
+const MEASURED_APP_USD_PER_JOB = 0.005;
+
+/**
+ * The run's cost estimate and the budget it needs. The estimate scales the measured cost per
+ * charter linearly with --max-steps. The headroom covers the reservations of the calls in flight:
+ * one per running explorer, each at the largest bound the proxy can reserve (proxyPolicy.ts).
+ * Without it, the last jobs of a run that fits its estimate find the budget reserved and fail.
+ * `appModel` is the app's model when its AI can be real (the probe decides later, so the check
+ * assumes it is): its calls share the ledger, so its bound counts too, and a model other than the
+ * measured Haiku 4.5 adds a guessed share per charter.
+ */
+export function estimateRun(o: {
+  models: string[];
+  charters: number;
+  parallel: number;
+  maxSteps: number;
+  effort: string;
+  appModel?: string;
+}) {
+  const { models, charters, parallel, maxSteps, appModel } = o;
+  const effortFactor = EFFORT_FACTOR[o.effort];
+  assert(effortFactor != null, `estimateRun: unknown effort ${o.effort}`);
+  const sonnet = listPrice("claude-sonnet-5-5")!;
+  const perModel = models.map((spec) => {
+    const id = spec.split(/:(.*)/s, 2)[1];
+    const measured = MEASURED_USD_PER_JOB[id];
+    const perJob =
+      (measured ??
+        (MEASURED_USD_PER_JOB["claude-sonnet-5-5"] * listPrice(id)!.output * UNMEASURED_MARGIN) /
+          sonnet.output) *
+      (maxSteps / MEASURED_STEPS) *
+      effortFactor;
+    // Measured means: this model, at the effort of the measurement.
+    return { spec, perJob, measured: measured != null && o.effort === "medium" };
+  });
+  const appId = appModel?.split(/:(.*)/s, 2)[1];
+  const haiku = listPrice(DEFAULT_APP_MODEL.split(":")[1])!;
+  const appPerJob =
+    appId == null || appModel === DEFAULT_APP_MODEL
+      ? 0
+      : ((MEASURED_APP_USD_PER_JOB * listPrice(appId)!.output * UNMEASURED_MARGIN) / haiku.output) *
+        (maxSteps / MEASURED_STEPS);
+  const jobs = charters * models.length;
+  // Each job runs its own app, so the app's share counts once per job.
+  const estimateUsd = perModel.reduce((sum, m) => sum + m.perJob * charters, 0) + appPerJob * jobs;
+  const inFlight = Math.min(parallel, jobs);
+  const ids = [...models.map((spec) => spec.split(/:(.*)/s, 2)[1]), ...(appId ? [appId] : [])];
+  const worst = Math.max(...ids.map(worstCaseNanoUsd));
+  const headroomUsd = (inFlight * worst) / 1e9;
+  // Whole dollars, so the number is easy to type.
+  const recommendedUsd = Math.ceil(estimateUsd + headroomUsd);
+  const app = appPerJob > 0 ? { spec: appModel!, perJob: appPerJob } : undefined;
+  return { perModel, app, estimateUsd, headroomUsd, inFlight, recommendedUsd };
+}
 
 function modelDirName(model: string): string {
   return model.replace(/[^a-zA-Z0-9.-]+/g, "_");
@@ -364,6 +445,37 @@ export async function runBugBash(
     return 2;
   }
   const budgetUsd = job0!.budgetUsd;
+  // The app AI can be real unless it is pinned to the mock (see the probe below).
+  const appReal = requested !== "mock" && !charters.every((c) => c.goal.includes("[mock:"));
+  const appSpec = env.BUGBASH_APP_MODEL ?? DEFAULT_APP_MODEL;
+  const appPriced = appSpec.startsWith("anthropic:") && priced(appSpec.slice("anthropic:".length));
+  const cost = estimateRun({
+    models,
+    charters: charters.length,
+    parallel,
+    maxSteps,
+    effort,
+    appModel: appReal && appPriced ? appSpec : undefined,
+  });
+  const per = cost.perModel.map(
+    (m) => `${m.spec} $${m.perJob.toFixed(2)}${m.measured ? "" : " (not measured: a guess)"}`
+  );
+  if (cost.app)
+    per.push(
+      `app ${cost.app.spec} +$${cost.app.perJob.toFixed(2)} per job (not measured: a guess)`
+    );
+  out(
+    `Estimate: $${cost.estimateUsd.toFixed(2)} for ${charters.length} charter(s) x ${models.length} model(s) ` +
+      `(per charter: ${per.join(", ")}), plus $${cost.headroomUsd.toFixed(2)} headroom for ` +
+      `${cost.inFlight} call(s) in flight. Recommended budget: $${cost.recommendedUsd}.`
+  );
+  if (budgetUsd < cost.recommendedUsd) {
+    err(
+      `make bug-bash: set BUGBASH_BUDGET_USD to at least $${cost.recommendedUsd} (it is $${budgetUsd}), ` +
+        "or run fewer charters (--only), fewer at a time (--parallel) or fewer models"
+    );
+    return 2;
+  }
   // One budget for the whole run: every job's proxy reserves from it (sandbox/proxyPolicy.ts).
   const ledger = new Ledger(budgetUsd);
   const mockPin: AppAi = { mode: "mock", reason: "the charter names a [mock:...] prompt" };

tests/bugbash/sandbox/proxyPolicy.ts

@@ -53,6 +53,21 @@ const VISUAL_TOKENS = 4784;
 
 /** Whether the proxy can price (and so bound) a call of this model. */
 export const priced = (model: string) => PRICES.has(model);
+/** A model's list price per token in nano-dollars (run.ts scales its cost estimate by it). */
+export const listPrice = (model: string) => PRICES.get(model);
+
+/** A call's upper bound: every input token at the cache-write rate (2x input), all output paid. */
+const callBound = (price: { input: number; output: number }, input: number, output: number) =>
+  input * 2 * price.input + output * price.output;
+/**
+ * The most one call of `model` can reserve: the bound at the largest input and output that
+ * checkRequest admits. run.ts reserves budget headroom for the calls in flight with it.
+ */
+export function worstCaseNanoUsd(model: string): number {
+  const price = PRICES.get(model);
+  if (!price) throw new Error(`worstCaseNanoUsd: no price for ${model}`);
+  return callBound(price, MAX_INPUT_TOKENS, MAX_OUTPUT_TOKENS);
+}
 
 export interface JobPolicy {
   /** Model IDs this job may call: its explorer model and the app model (P3). */
@@ -206,7 +221,7 @@ export function checkRequest(
   const inputTokens =
     Buffer.byteLength(body) - images.bytes + images.count * 2 * VISUAL_TOKENS + OVERHEAD_TOKENS;
   if (inputTokens > MAX_INPUT_TOKENS) return refuse("body: input bound above 200k tokens");
-  const maxCostNanoUsd = inputTokens * 2 * price.input + outputTokens * price.output;
+  const maxCostNanoUsd = callBound(price, inputTokens, outputTokens);
   return { ok: true, model, body, headers, maxCostNanoUsd };
 }