From 049f6e7857a4bcd7bbffd169dd9d4b71091aac92 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 6 Oct 2026 22:15:13 +0000 Subject: [PATCH 1/4] Initial plan From a4551e0b03e2e66f77964b878bb6304c26d53b54 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 6 Oct 2026 22:23:42 +0000 Subject: [PATCH 2/4] Count displayed daily AIC component jobs and invalidate stale scans Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- actions/setup/js/daily_aic_cache_helpers.cjs | 4 ++-- actions/setup/js/daily_aic_component_coverage.cjs | 7 ++++--- .../setup/js/daily_aic_component_coverage.test.cjs | 12 ++++++++++++ actions/setup/js/daily_aic_scan.test.cjs | 13 +++++++++++++ 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/actions/setup/js/daily_aic_cache_helpers.cjs b/actions/setup/js/daily_aic_cache_helpers.cjs index 95f4d1452b7..964abb3d9af 100644 --- a/actions/setup/js/daily_aic_cache_helpers.cjs +++ b/actions/setup/js/daily_aic_cache_helpers.cjs @@ -32,7 +32,7 @@ function readScanCache(content, repository, workflowId, now = Date.now()) { } if ( entry?.version !== 2 || - entry.coverage_version !== 1 || + entry.coverage_version !== 2 || entry.repository !== repository || entry.workflow_id !== workflowId || !Number.isSafeInteger(entry.run_id) || @@ -72,7 +72,7 @@ function scanCacheEntry(run, aic, repository, workflowId, now = Date.now(), sour } return { version: 2, - coverage_version: 1, + coverage_version: 2, repository, workflow_id: workflowId, run_id: run.id, diff --git a/actions/setup/js/daily_aic_component_coverage.cjs b/actions/setup/js/daily_aic_component_coverage.cjs index 21e49dc600b..a375b822554 100644 --- a/actions/setup/js/daily_aic_component_coverage.cjs +++ b/actions/setup/js/daily_aic_component_coverage.cjs @@ -28,16 +28,17 @@ async function loadBillableJobs({ github, budget }, owner, repo, run) { const jobs = response.data.jobs; if (!Array.isArray(jobs)) throw new Error("Incomplete daily AIC job metadata"); for (const job of jobs) { - if (!Object.hasOwn(COMPONENT_FILES, job.name)) continue; + const name = typeof job.name === "string" ? job.name.toLowerCase() : ""; + if (!Object.hasOwn(COMPONENT_FILES, name)) continue; if (!Number.isSafeInteger(job.run_attempt) || job.run_attempt < 1 || job.run_attempt > run.run_attempt || job.status !== "completed" || !job.conclusion) { throw new Error("Incomplete daily AIC component attempt metadata"); } - const prior = components.get(job.name); + const prior = components.get(name); if (prior && job.run_attempt === prior.run_attempt && job.id !== prior.id) { throw new Error("Ambiguous daily AIC component jobs"); } if (!prior || (prior.conclusion === "skipped" && job.conclusion !== "skipped") || (job.conclusion !== "skipped" && job.run_attempt > prior.run_attempt) || (prior.conclusion === "skipped" && job.run_attempt > prior.run_attempt)) { - components.set(job.name, job); + components.set(name, job); } } if (jobs.length < 100) { diff --git a/actions/setup/js/daily_aic_component_coverage.test.cjs b/actions/setup/js/daily_aic_component_coverage.test.cjs index 1a4fd01e0b5..77900d40c88 100644 --- a/actions/setup/js/daily_aic_component_coverage.test.cjs +++ b/actions/setup/js/daily_aic_component_coverage.test.cjs @@ -391,6 +391,18 @@ it("accepts a completed run with no jobs as zero usage", async () => { expect(global.core.info).toHaveBeenCalledWith(expect.stringContaining('"aic":0,"reason":"no_billable_jobs"')); }); +it("counts usage for compiler-generated job display names", async () => { + const f = evaluate({ "agent_usage.jsonl": '{"aic":18.02604}', "detection_usage.jsonl": '{"aic":30.08748}' }, [job("Agent"), job("Detection")]); + await expect(f.result).resolves.toBeCloseTo(48.11352); + expect(global.core.info).toHaveBeenCalledWith(expect.stringContaining('"component":"agent"')); + expect(global.core.info).toHaveBeenCalledWith(expect.stringContaining('"component":"detection"')); +}); + +it("requires accounting for executed jobs with generated display names", async () => { + const f = evaluate({}, [job("Agent")]); + await expect(f.result).rejects.toThrow("Missing accounting for executed agent component"); +}); + it("accepts a completed run with all billable jobs skipped as zero usage", async () => { const f = evaluate({}, [job("agent", { conclusion: "skipped" }), job("detection", { conclusion: "skipped" })]); await expect(f.result).resolves.toBe(0); diff --git a/actions/setup/js/daily_aic_scan.test.cjs b/actions/setup/js/daily_aic_scan.test.cjs index 03c0d7f9037..3d8bca1d673 100644 --- a/actions/setup/js/daily_aic_scan.test.cjs +++ b/actions/setup/js/daily_aic_scan.test.cjs @@ -188,6 +188,19 @@ describe("complete daily AIC scan observations", () => { expect(f.getRunAIC).toHaveBeenCalledOnce(); }); + it("reinspects recorded zero usage cached with the old component coverage", async () => { + const entry = scanCacheEntry(run(1), 0, repository, 7, now); + entry.coverage_version = 1; + writeEntries([entry]); + const f = fixture([run(1)]); + f.getRunAIC.mockResolvedValue(48.11352); + const result = await scanDailyAIC(f); + expect(result.cacheHits).toBe(0); + expect(result.countedRuns[0]).toMatchObject({ aic: 48.11352, source: "recorded" }); + expect(f.getRunAIC).toHaveBeenCalledOnce(); + expect(readScanCache(fs.readFileSync(cachePath, "utf8"), repository, 7, now).get(1)?.aic).toBe(48.11352); + }); + it("accepts corrected estimates without replacing recorded accounting for the same attempt", () => { const recorded = scanCacheEntry(run(1), 4, repository, 7, now - 2000, "recorded"); const estimated = scanCacheEntry(run(1), 1000, repository, 7, now - 1000, "estimated"); From 8d273c1061269e36211f1e67d38f41119baeff2b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:10:55 +0000 Subject: [PATCH 3/4] Prefer job_id and map Evaluations in daily AIC component lookup Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .../setup/js/daily_aic_component_coverage.cjs | 17 +++++++++++++++-- .../js/daily_aic_component_coverage.test.cjs | 7 +++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/actions/setup/js/daily_aic_component_coverage.cjs b/actions/setup/js/daily_aic_component_coverage.cjs index a375b822554..30787ae8d96 100644 --- a/actions/setup/js/daily_aic_component_coverage.cjs +++ b/actions/setup/js/daily_aic_component_coverage.cjs @@ -12,6 +12,19 @@ const COMPONENT_FILES = { evals: [["evals", "token_usage.jsonl"]], }; +// Generated jobs display as "Agent", "Detection" and "Evaluations". +const COMPONENT_DISPLAY_NAMES = { evaluations: "evals" }; + +function componentKey(job) { + for (const candidate of [job.job_id, job.name]) { + if (typeof candidate !== "string") continue; + const lower = candidate.toLowerCase(); + const key = COMPONENT_DISPLAY_NAMES[lower] ?? lower; + if (Object.hasOwn(COMPONENT_FILES, key)) return key; + } + return ""; +} + async function loadBillableJobs({ github, budget }, owner, repo, run) { const components = new Map(); let complete = false; @@ -28,8 +41,8 @@ async function loadBillableJobs({ github, budget }, owner, repo, run) { const jobs = response.data.jobs; if (!Array.isArray(jobs)) throw new Error("Incomplete daily AIC job metadata"); for (const job of jobs) { - const name = typeof job.name === "string" ? job.name.toLowerCase() : ""; - if (!Object.hasOwn(COMPONENT_FILES, name)) continue; + const name = componentKey(job); + if (!name) continue; if (!Number.isSafeInteger(job.run_attempt) || job.run_attempt < 1 || job.run_attempt > run.run_attempt || job.status !== "completed" || !job.conclusion) { throw new Error("Incomplete daily AIC component attempt metadata"); } diff --git a/actions/setup/js/daily_aic_component_coverage.test.cjs b/actions/setup/js/daily_aic_component_coverage.test.cjs index 77900d40c88..50c63ccbbc7 100644 --- a/actions/setup/js/daily_aic_component_coverage.test.cjs +++ b/actions/setup/js/daily_aic_component_coverage.test.cjs @@ -398,6 +398,13 @@ it("counts usage for compiler-generated job display names", async () => { expect(global.core.info).toHaveBeenCalledWith(expect.stringContaining('"component":"detection"')); }); +it("maps the Evaluations display name and job_id to their components", async () => { + const f = evaluate({ "agent_usage.jsonl": '{"aic":1}', "evals/token_usage.jsonl": '{"aic":2}' }, [job("Agent"), job("Evaluations")]); + await expect(f.result).resolves.toBeCloseTo(3); + const g = evaluate({ "agent_usage.jsonl": '{"aic":4}' }, [job("Display", { job_id: "agent" })]); + await expect(g.result).resolves.toBeCloseTo(4); +}); + it("requires accounting for executed jobs with generated display names", async () => { const f = evaluate({}, [job("Agent")]); await expect(f.result).rejects.toThrow("Missing accounting for executed agent component"); From 202d80b57591d6f7a820376251c277a8424050bc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 7 Oct 2026 01:10:06 +0000 Subject: [PATCH 4/4] Make weighted variant test deterministic Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com> --- actions/setup/js/pick_experiment.test.cjs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/actions/setup/js/pick_experiment.test.cjs b/actions/setup/js/pick_experiment.test.cjs index ae8ff3ce96f..0a7a201faca 100644 --- a/actions/setup/js/pick_experiment.test.cjs +++ b/actions/setup/js/pick_experiment.test.cjs @@ -802,13 +802,18 @@ describe("pick_experiment", () => { it("distributes variants proportionally across many runs", () => { const counts = { A: 0, B: 0 }; const N = 1000; - for (let i = 0; i < N; i++) { - const v = pickVariantWeighted(["A", "B"], [70, 30]); - counts[v]++; + let sample = 0; + const random = vi.spyOn(Math, "random").mockImplementation(() => (sample++ + 0.5) / N); + try { + for (let i = 0; i < N; i++) { + const v = pickVariantWeighted(["A", "B"], [70, 30]); + counts[v]++; + } + expect(counts["A"]).toBe(700); + expect(counts["B"]).toBe(300); + } finally { + random.mockRestore(); } - // With weights 70:30 we expect ~70% A and ~30% B. Allow 10% absolute tolerance. - expect(counts["A"] / N).toBeCloseTo(0.7, 1); - expect(counts["B"] / N).toBeCloseTo(0.3, 1); }); });