From 568ec037d093cb6ce695c4be63335734a3eff870 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Sun, 11 Oct 2026 17:44:21 +0000 Subject: [PATCH 1/2] tests: the bug bash defaults to Sonnet alone and refuses a budget below its estimate --- tests/bugbash/run.test.ts | 89 +++++++++++++++++-- tests/bugbash/run.ts | 126 +++++++++++++++++++++++++-- tests/bugbash/sandbox/proxyPolicy.ts | 17 +++- 3 files changed, 219 insertions(+), 13 deletions(-) diff --git a/tests/bugbash/run.test.ts b/tests/bugbash/run.test.ts index 3e7e401f1bd..8761d2a2b1c 100644 --- a/tests/bugbash/run.test.ts +++ b/tests/bugbash/run.test.ts @@ -3,7 +3,7 @@ import * as childProcess from "node:child_process"; import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; -import { type RunDeps, runBugBash } from "./run"; +import { estimateRun, type RunDeps, runBugBash } from "./run"; import type { JobOutcome, LaunchOptions } from "./sandbox/launch"; import type { Ledger } from "./sandbox/proxyPolicy"; import { Refusal, Stopped } from "./sandbox/runner"; @@ -12,7 +12,8 @@ import { Refusal, Stopped } from "./sandbox/runner"; // (sandbox/runner.test.ts) cover the job itself. const ENV = { BUGBASH_MODELS: "anthropic:claude-sonnet-5-5", - BUGBASH_BUDGET_USD: "2", + // Above the recommended budget of every run here (estimateRun): up to 4 charters at once. + BUGBASH_BUDGET_USD: "10", ANTHROPIC_API_KEY: "sk-run-test", ANTHROPIC_BASE_URL: "http://127.0.0.1:9/v1", }; @@ -93,7 +94,7 @@ test("B2: every charter runs through the launcher with one shared ledger, none o expect(spawn).not.toHaveBeenCalled(); const findings = lines.find((line) => line.startsWith("Findings: "))!.slice(10); expect(fs.readFileSync(findings, "utf8")).toMatch( - /Cost: 0 proxied calls, 0 refused by the budget, \$0\.0000 of \$2\.0000 at list price/ + /Cost: 0 proxied calls, 0 refused by the budget, \$0\.0000 of \$10\.0000 at list price/ ); } finally { spawn.mockRestore(); @@ -316,7 +317,10 @@ test("U1: Docker is checked once, before the probe and any folder; --preflight s expect(checks).toBe(1); expect(dead.probes).toHaveLength(0); expect(dead.calls).toHaveLength(0); - expect(dead.lines).toEqual(["make bug-bash: docker info: no daemon answered"]); + // The cost estimate prints first (U4); the only refusal is Docker's. + expect(dead.lines.filter((line) => line.startsWith("make bug-bash:"))).toEqual([ + "make bug-bash: docker info: no daemon answered", + ]); const live = fakeLaunch(() => ({ code: 0, cleanup: "removed" })); let liveChecks = 0; @@ -341,7 +345,7 @@ test("U1: a probe the budget refused (HTTP 402) blames the budget, not the key", const stop = new AbortController().signal; expect(await runBugBash([...charters("a")], ENV, stop, deps)).toBe(2); expect(calls).toHaveLength(0); - expect(lines.join("\n")).toMatch(/BUGBASH_BUDGET_USD \(\$2\) is too small to cover/); + expect(lines.join("\n")).toMatch(/BUGBASH_BUDGET_USD \(\$10\) is too small to cover/); expect(lines.join("\n")).not.toMatch(/Fix the key/); }); @@ -399,3 +403,78 @@ test.each([ expect(lines[0]).not.toContain("/v1"); } ); + +const SONNET = "anthropic:claude-sonnet-5-5"; +const OPUS = "anthropic:claude-opus-5-5"; +const run6 = { models: [SONNET], charters: 6, parallel: 8, maxSteps: 6, effort: "medium" }; + +test("U4: the estimate scales the measured cost, and its headroom covers the calls in flight", () => { + const base = estimateRun(run6); + // Sonnet's measured $0.45 per charter; 6 calls in flight at Sonnet's worst bound. + expect(base.estimateUsd).toBeCloseTo(6 * 0.45); + expect(base.inFlight).toBe(6); + expect(base.recommendedUsd).toBe(Math.ceil(base.estimateUsd + base.headroomUsd)); + expect(base.perModel[0].measured).toBe(true); + // Twice the steps cost twice as much; fewer at once need less headroom, not less estimate. + expect(estimateRun({ ...run6, maxSteps: 12 }).estimateUsd).toBeCloseTo(2 * base.estimateUsd); + const two = estimateRun({ ...run6, parallel: 2 }); + expect(two.estimateUsd).toBeCloseTo(base.estimateUsd); + expect(two.headroomUsd).toBeCloseTo((base.headroomUsd * 2) / 6); + // An unmeasured model is a guess above its list-price share; its bound sets the headroom. + const both = estimateRun({ ...run6, models: [SONNET, OPUS] }); + expect(both.perModel[1].measured).toBe(false); + expect(both.perModel[1].perJob).toBeGreaterThan(2 * base.perModel[0].perJob); // Opus is 2x list + expect(both.headroomUsd).toBeGreaterThan(base.headroomUsd * (8 / 6)); + // A higher effort is an unmeasured guess that costs more; it reserves no more per call. + const high = estimateRun({ ...run6, effort: "high" }); + expect(high.estimateUsd).toBeGreaterThan(base.estimateUsd); + expect(high.perModel[0].measured).toBe(false); + expect(high.headroomUsd).toBeCloseTo(base.headroomUsd); + // Low effort is not measured either, and it is not priced below medium. + const low = estimateRun({ ...run6, effort: "low" }); + expect(low.perModel[0].measured).toBe(false); + expect(low.estimateUsd).toBeCloseTo(base.estimateUsd); + // A real app AI with a costlier model adds its share and its larger bound; Haiku adds nothing. + const opusApp = estimateRun({ ...run6, appModel: OPUS }); + expect(opusApp.estimateUsd).toBeGreaterThan(base.estimateUsd); + expect(opusApp.headroomUsd).toBeGreaterThan(base.headroomUsd); + const haiku = estimateRun({ ...run6, appModel: "anthropic:claude-haiku-4-5" }); + expect(haiku.recommendedUsd).toBe(base.recommendedUsd); +}); + +test("U4: the default runs Sonnet alone, and a budget below the estimate refuses before the probe", async () => { + const stop = new AbortController().signal; + const { BUGBASH_MODELS: _models, ...defaults } = ENV; + const six = charters("a", "b", "c", "d", "e", "f"); + const need = estimateRun(run6).recommendedUsd; + const low = fakeLaunch(() => ({ code: 0, cleanup: "removed" })); + const below = { ...defaults, BUGBASH_BUDGET_USD: String(need - 1) }; + expect(await runBugBash(six, below, stop, low.deps)).toBe(2); + expect(low.calls).toHaveLength(0); + expect(low.probes).toHaveLength(0); + // The refusal names the variable and the number to set. + const refusal = low.lines.find((line) => line.includes("BUGBASH_BUDGET_USD"))!; + expect(refusal).toContain(`$${need}`); + + const enough = fakeLaunch(() => ({ code: 0, cleanup: "removed" })); + const at = { ...defaults, BUGBASH_BUDGET_USD: String(need) }; + expect(await runBugBash(six, at, stop, enough.deps)).toBe(0); + expect(new Set(enough.calls.map((c) => c.o.env.BUGBASH_MODEL))).toEqual(new Set([SONNET])); + expect(enough.calls).toHaveLength(6); +}); + +test("U4: the run passes its effort and a possibly real app model to the estimate", async () => { + const stop = new AbortController().signal; + // A budget that fits medium effort with the Haiku app, but not high effort or an Opus app. + const fits = estimateRun({ ...run6, charters: 1 }).recommendedUsd; + const env = { ...ENV, BUGBASH_BUDGET_USD: String(fits) }; + for (const [over, code] of [ + [{}, 0], + [{ BUGBASH_EFFORT: "max" }, 2], + [{ BUGBASH_APP_MODEL: OPUS }, 2], + [{ BUGBASH_APP_MODEL: OPUS, BUGBASH_AI: "mock" }, 0], // a mock app makes no calls + ] as const) { + const { deps } = fakeLaunch(() => ({ code: 0, cleanup: "removed" })); + expect(await runBugBash([...charters("a")], { ...env, ...over }, stop, deps)).toBe(code); + } +}); diff --git a/tests/bugbash/run.ts b/tests/bugbash/run.ts index 5ee793c67ba..5e09e2ab785 100644 --- a/tests/bugbash/run.ts +++ b/tests/bugbash/run.ts @@ -17,10 +17,15 @@ * e2e.config.ts. No other config runs in the sandbox (sandbox/inputs.ts exploreRefusal). * Paths are relative to the current directory. * - * Models: BUGBASH_MODELS, comma-separated `anthropic:`, default Opus 5.5 and Sonnet 5.5. - * In a three-model comparison (2026-10) these two found 14 of 15 distinct bugs and overlapped on - * only 4, so the default runs both; GPT-6.1 Sol found 1. Every model runs every charter, all in - * one pool of --parallel explorers. Reasoning: BUGBASH_EFFORT, default medium (e2e.config.ts). + * Models: BUGBASH_MODELS, comma-separated `anthropic:`, default Sonnet 5.5 alone (6 jobs). + * In a three-model comparison (2026-10) Opus 5.5 and Sonnet 5.5 together found 14 of 15 distinct + * bugs and overlapped on only 4; GPT-6.1 Sol found 1. Opus about doubles the cost of a run, so it + * is opt-in: BUGBASH_MODELS=anthropic:claude-sonnet-5-5,anthropic:claude-opus-5-5. Every model + * runs every charter, all in one pool of --parallel explorers. Reasoning: BUGBASH_EFFORT, default + * medium (e2e.config.ts). + * + * Cost: before any job, the run prints its estimate and a recommended budget (estimateRun), and + * refuses a BUGBASH_BUDGET_USD below it. * * Charters: tests/bugbash/charters.txt. Config and personas: tests/bugbash/e2e.config.ts. * Output: tests/bugbash/.e2e/bugbash/// with one directory, log and app log per @@ -56,7 +61,7 @@ import { probeApp, probeUnreachable, } from "./sandbox/launch"; -import { Ledger, priced } from "./sandbox/proxyPolicy"; +import { Ledger, listPrice, priced, worstCaseNanoUsd } from "./sandbox/proxyPolicy"; import { Refusal, Stopped } from "./sandbox/runner"; const projectDir = import.meta.dir; @@ -232,8 +237,84 @@ function writeFindings( fs.writeFileSync(file, `${lines.join("\n")}\n`); } -/** Opus 5.5 and Sonnet 5.5: see the header comment for why both. */ -const DEFAULT_MODELS = ["anthropic:claude-opus-5-5", "anthropic:claude-sonnet-5-5"]; +/** Sonnet 5.5 alone: see the header comment for why Opus is opt-in. */ +const DEFAULT_MODELS = ["anthropic:claude-sonnet-5-5"]; + +/** + * Measured list-price cost of one charter at --max-steps 6 and medium effort, explorer and app AI + * together: four `make bug-bash` runs of Sonnet 5.5 on #5714, #6076 and #6079 averaged $0.34 to + * $0.52 per charter (21 charters, $9.50 in all). + */ +const MEASURED_USD_PER_JOB: Record = { "claude-sonnet-5-5": 0.45 }; +const MEASURED_STEPS = 6; +/** An unmeasured model: Sonnet's measured cost, scaled by list price, times this margin. */ +const UNMEASURED_MARGIN = 2; +/** + * Explorer cost per charter relative to medium effort, where it was measured. Higher efforts add + * thinking tokens; nothing measured them, so these are guesses, and the run labels them so. Low + * is not lowered: an estimate must not undercount. + */ +const EFFORT_FACTOR: Record = { low: 1, medium: 1, high: 2, xhigh: 3, max: 4 }; + +/** + * The real app AI's own share per charter with Haiku 4.5, the default app model: #6079's five + * real-mode charters spent 19,545 input and 1,022 output tokens on it, about $0.005 each. That + * share is already in the explorer measurement above, so only another app model adds to it. + */ +const MEASURED_APP_USD_PER_JOB = 0.005; + +/** + * The run's cost estimate and the budget it needs. The estimate scales the measured cost per + * charter linearly with --max-steps. The headroom covers the reservations of the calls in flight: + * one per running explorer, each at the largest bound the proxy can reserve (proxyPolicy.ts). + * Without it, the last jobs of a run that fits its estimate find the budget reserved and fail. + * `appModel` is the app's model when its AI can be real (the probe decides later, so the check + * assumes it is): its calls share the ledger, so its bound counts too, and a model other than the + * measured Haiku 4.5 adds a guessed share per charter. + */ +export function estimateRun(o: { + models: string[]; + charters: number; + parallel: number; + maxSteps: number; + effort: string; + appModel?: string; +}) { + const { models, charters, parallel, maxSteps, appModel } = o; + const effortFactor = EFFORT_FACTOR[o.effort]; + assert(effortFactor != null, `estimateRun: unknown effort ${o.effort}`); + const sonnet = listPrice("claude-sonnet-5-5")!; + const perModel = models.map((spec) => { + const id = spec.split(/:(.*)/s, 2)[1]; + const measured = MEASURED_USD_PER_JOB[id]; + const perJob = + (measured ?? + (MEASURED_USD_PER_JOB["claude-sonnet-5-5"] * listPrice(id)!.output * UNMEASURED_MARGIN) / + sonnet.output) * + (maxSteps / MEASURED_STEPS) * + effortFactor; + // Measured means: this model, at the effort of the measurement. + return { spec, perJob, measured: measured != null && o.effort === "medium" }; + }); + const appId = appModel?.split(/:(.*)/s, 2)[1]; + const haiku = listPrice(DEFAULT_APP_MODEL.split(":")[1])!; + const appPerJob = + appId == null || appModel === DEFAULT_APP_MODEL + ? 0 + : ((MEASURED_APP_USD_PER_JOB * listPrice(appId)!.output * UNMEASURED_MARGIN) / haiku.output) * + (maxSteps / MEASURED_STEPS); + const jobs = charters * models.length; + // Each job runs its own app, so the app's share counts once per job. + const estimateUsd = perModel.reduce((sum, m) => sum + m.perJob * charters, 0) + appPerJob * jobs; + const inFlight = Math.min(parallel, jobs); + const ids = [...models.map((spec) => spec.split(/:(.*)/s, 2)[1]), ...(appId ? [appId] : [])]; + const worst = Math.max(...ids.map(worstCaseNanoUsd)); + const headroomUsd = (inFlight * worst) / 1e9; + // Whole dollars, so the number is easy to type. + const recommendedUsd = Math.ceil(estimateUsd + headroomUsd); + const app = appPerJob > 0 ? { spec: appModel!, perJob: appPerJob } : undefined; + return { perModel, app, estimateUsd, headroomUsd, inFlight, recommendedUsd }; +} function modelDirName(model: string): string { return model.replace(/[^a-zA-Z0-9.-]+/g, "_"); @@ -364,6 +445,37 @@ export async function runBugBash( return 2; } const budgetUsd = job0!.budgetUsd; + // The app AI can be real unless it is pinned to the mock (see the probe below). + const appReal = requested !== "mock" && !charters.every((c) => c.goal.includes("[mock:")); + const appSpec = env.BUGBASH_APP_MODEL ?? DEFAULT_APP_MODEL; + const appPriced = appSpec.startsWith("anthropic:") && priced(appSpec.slice("anthropic:".length)); + const cost = estimateRun({ + models, + charters: charters.length, + parallel, + maxSteps, + effort, + appModel: appReal && appPriced ? appSpec : undefined, + }); + const per = cost.perModel.map( + (m) => `${m.spec} $${m.perJob.toFixed(2)}${m.measured ? "" : " (not measured: a guess)"}` + ); + if (cost.app) + per.push( + `app ${cost.app.spec} +$${cost.app.perJob.toFixed(2)} per job (not measured: a guess)` + ); + out( + `Estimate: $${cost.estimateUsd.toFixed(2)} for ${charters.length} charter(s) x ${models.length} model(s) ` + + `(per charter: ${per.join(", ")}), plus $${cost.headroomUsd.toFixed(2)} headroom for ` + + `${cost.inFlight} call(s) in flight. Recommended budget: $${cost.recommendedUsd}.` + ); + if (budgetUsd < cost.recommendedUsd) { + err( + `make bug-bash: set BUGBASH_BUDGET_USD to at least $${cost.recommendedUsd} (it is $${budgetUsd}), ` + + "or run fewer charters (--only), fewer at a time (--parallel) or fewer models" + ); + return 2; + } // One budget for the whole run: every job's proxy reserves from it (sandbox/proxyPolicy.ts). const ledger = new Ledger(budgetUsd); const mockPin: AppAi = { mode: "mock", reason: "the charter names a [mock:...] prompt" }; diff --git a/tests/bugbash/sandbox/proxyPolicy.ts b/tests/bugbash/sandbox/proxyPolicy.ts index 3b1fc74b774..5df337ec1c8 100644 --- a/tests/bugbash/sandbox/proxyPolicy.ts +++ b/tests/bugbash/sandbox/proxyPolicy.ts @@ -53,6 +53,21 @@ const VISUAL_TOKENS = 4784; /** Whether the proxy can price (and so bound) a call of this model. */ export const priced = (model: string) => PRICES.has(model); +/** A model's list price per token in nano-dollars (run.ts scales its cost estimate by it). */ +export const listPrice = (model: string) => PRICES.get(model); + +/** A call's upper bound: every input token at the cache-write rate (2x input), all output paid. */ +const callBound = (price: { input: number; output: number }, input: number, output: number) => + input * 2 * price.input + output * price.output; +/** + * The most one call of `model` can reserve: the bound at the largest input and output that + * checkRequest admits. run.ts reserves budget headroom for the calls in flight with it. + */ +export function worstCaseNanoUsd(model: string): number { + const price = PRICES.get(model); + if (!price) throw new Error(`worstCaseNanoUsd: no price for ${model}`); + return callBound(price, MAX_INPUT_TOKENS, MAX_OUTPUT_TOKENS); +} export interface JobPolicy { /** Model IDs this job may call: its explorer model and the app model (P3). */ @@ -206,7 +221,7 @@ export function checkRequest( const inputTokens = Buffer.byteLength(body) - images.bytes + images.count * 2 * VISUAL_TOKENS + OVERHEAD_TOKENS; if (inputTokens > MAX_INPUT_TOKENS) return refuse("body: input bound above 200k tokens"); - const maxCostNanoUsd = inputTokens * 2 * price.input + outputTokens * price.output; + const maxCostNanoUsd = callBound(price, inputTokens, outputTokens); return { ok: true, model, body, headers, maxCostNanoUsd }; } From d50869f534948171ff644d78351ab85e78caaa39 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Sun, 11 Oct 2026 18:08:04 +0000 Subject: [PATCH 2/2] tests: the unreachable-upstream test reads the refusal after the estimate line --- tests/bugbash/run.test.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/bugbash/run.test.ts b/tests/bugbash/run.test.ts index 8761d2a2b1c..48da293b957 100644 --- a/tests/bugbash/run.test.ts +++ b/tests/bugbash/run.test.ts @@ -397,10 +397,12 @@ test.each([ const stop = new AbortController().signal; expect(await runBugBash([...charters("a")], { ...ENV, ...over }, stop, deps)).toBe(2); expect(calls).toHaveLength(0); - // It names the variable and the host to check, never the URL's path. - expect(lines).toHaveLength(1); - expect(lines[0]).toContain("ANTHROPIC_BASE_URL (127.0.0.1:9)"); - expect(lines[0]).not.toContain("/v1"); + // One refusal (after U4's estimate line). It names the variable and the host to check, + // never the URL's path. + const refusals = lines.filter((line) => line.startsWith("make bug-bash:")); + expect(refusals).toHaveLength(1); + expect(refusals[0]).toContain("ANTHROPIC_BASE_URL (127.0.0.1:9)"); + expect(refusals[0]).not.toContain("/v1"); } );