Count generated jobs in the daily AIC guardrail

actions/setup/js/daily_aic_cache_helpers.cjs

@@ -32,7 +32,7 @@ function readScanCache(content, repository, workflowId, now = Date.now()) {
     }
     if (
       entry?.version !== 2 ||
-      entry.coverage_version !== 1 ||
+      entry.coverage_version !== 2 ||
       entry.repository !== repository ||
       entry.workflow_id !== workflowId ||
       !Number.isSafeInteger(entry.run_id) ||
@@ -72,7 +72,7 @@ function scanCacheEntry(run, aic, repository, workflowId, now = Date.now(), sour
   }
   return {
     version: 2,
-    coverage_version: 1,
+    coverage_version: 2,
     repository,
     workflow_id: workflowId,
     run_id: run.id,

actions/setup/js/daily_aic_component_coverage.cjs

@@ -12,6 +12,19 @@ const COMPONENT_FILES = {
   evals: [["evals", "token_usage.jsonl"]],
 };
 
+// Generated jobs display as "Agent", "Detection" and "Evaluations".
+const COMPONENT_DISPLAY_NAMES = { evaluations: "evals" };
+
+function componentKey(job) {
+  for (const candidate of [job.job_id, job.name]) {
+    if (typeof candidate !== "string") continue;
+    const lower = candidate.toLowerCase();
+    const key = COMPONENT_DISPLAY_NAMES[lower] ?? lower;
+    if (Object.hasOwn(COMPONENT_FILES, key)) return key;
+  }
+  return "";
+}
+
 async function loadBillableJobs({ github, budget }, owner, repo, run) {
   const components = new Map();
   let complete = false;
@@ -28,16 +41,17 @@ async function loadBillableJobs({ github, budget }, owner, repo, run) {
     const jobs = response.data.jobs;
     if (!Array.isArray(jobs)) throw new Error("Incomplete daily AIC job metadata");
     for (const job of jobs) {
-      if (!Object.hasOwn(COMPONENT_FILES, job.name)) continue;
+      const name = componentKey(job);
+      if (!name) continue;
       if (!Number.isSafeInteger(job.run_attempt) || job.run_attempt < 1 || job.run_attempt > run.run_attempt || job.status !== "completed" || !job.conclusion) {
         throw new Error("Incomplete daily AIC component attempt metadata");
       }
-      const prior = components.get(job.name);
+      const prior = components.get(name);
       if (prior && job.run_attempt === prior.run_attempt && job.id !== prior.id) {
         throw new Error("Ambiguous daily AIC component jobs");
       }
       if (!prior || (prior.conclusion === "skipped" && job.conclusion !== "skipped") || (job.conclusion !== "skipped" && job.run_attempt > prior.run_attempt) || (prior.conclusion === "skipped" && job.run_attempt > prior.run_attempt)) {
-        components.set(job.name, job);
+        components.set(name, job);
       }
     }
     if (jobs.length < 100) {

actions/setup/js/daily_aic_component_coverage.test.cjs

@@ -391,6 +391,25 @@ it("accepts a completed run with no jobs as zero usage", async () => {
   expect(global.core.info).toHaveBeenCalledWith(expect.stringContaining('"aic":0,"reason":"no_billable_jobs"'));
 });
 
+it("counts usage for compiler-generated job display names", async () => {
+  const f = evaluate({ "agent_usage.jsonl": '{"aic":18.02604}', "detection_usage.jsonl": '{"aic":30.08748}' }, [job("Agent"), job("Detection")]);
+  await expect(f.result).resolves.toBeCloseTo(48.11352);
+  expect(global.core.info).toHaveBeenCalledWith(expect.stringContaining('"component":"agent"'));
+  expect(global.core.info).toHaveBeenCalledWith(expect.stringContaining('"component":"detection"'));
+});
+
+it("maps the Evaluations display name and job_id to their components", async () => {
+  const f = evaluate({ "agent_usage.jsonl": '{"aic":1}', "evals/token_usage.jsonl": '{"aic":2}' }, [job("Agent"), job("Evaluations")]);
+  await expect(f.result).resolves.toBeCloseTo(3);
+  const g = evaluate({ "agent_usage.jsonl": '{"aic":4}' }, [job("Display", { job_id: "agent" })]);
+  await expect(g.result).resolves.toBeCloseTo(4);
+});
+
+it("requires accounting for executed jobs with generated display names", async () => {
+  const f = evaluate({}, [job("Agent")]);
+  await expect(f.result).rejects.toThrow("Missing accounting for executed agent component");
+});
+
 it("accepts a completed run with all billable jobs skipped as zero usage", async () => {
   const f = evaluate({}, [job("agent", { conclusion: "skipped" }), job("detection", { conclusion: "skipped" })]);
   await expect(f.result).resolves.toBe(0);

actions/setup/js/daily_aic_scan.test.cjs

@@ -188,6 +188,19 @@ describe("complete daily AIC scan observations", () => {
     expect(f.getRunAIC).toHaveBeenCalledOnce();
   });
 
+  it("reinspects recorded zero usage cached with the old component coverage", async () => {
+    const entry = scanCacheEntry(run(1), 0, repository, 7, now);
+    entry.coverage_version = 1;
+    writeEntries([entry]);
+    const f = fixture([run(1)]);
+    f.getRunAIC.mockResolvedValue(48.11352);
+    const result = await scanDailyAIC(f);
+    expect(result.cacheHits).toBe(0);
+    expect(result.countedRuns[0]).toMatchObject({ aic: 48.11352, source: "recorded" });
+    expect(f.getRunAIC).toHaveBeenCalledOnce();
+    expect(readScanCache(fs.readFileSync(cachePath, "utf8"), repository, 7, now).get(1)?.aic).toBe(48.11352);
+  });
+
   it("accepts corrected estimates without replacing recorded accounting for the same attempt", () => {
     const recorded = scanCacheEntry(run(1), 4, repository, 7, now - 2000, "recorded");
     const estimated = scanCacheEntry(run(1), 1000, repository, 7, now - 1000, "estimated");

actions/setup/js/pick_experiment.test.cjs

@@ -802,13 +802,18 @@ describe("pick_experiment", () => {
     it("distributes variants proportionally across many runs", () => {
       const counts = { A: 0, B: 0 };
       const N = 1000;
-      for (let i = 0; i < N; i++) {
-        const v = pickVariantWeighted(["A", "B"], [70, 30]);
-        counts[v]++;
+      let sample = 0;
+      const random = vi.spyOn(Math, "random").mockImplementation(() => (sample++ + 0.5) / N);
+      try {
+        for (let i = 0; i < N; i++) {
+          const v = pickVariantWeighted(["A", "B"], [70, 30]);
+          counts[v]++;
+        }
+        expect(counts["A"]).toBe(700);
+        expect(counts["B"]).toBe(300);
+      } finally {
+        random.mockRestore();
       }
-      // With weights 70:30 we expect ~70% A and ~30% B.  Allow 10% absolute tolerance.
-      expect(counts["A"] / N).toBeCloseTo(0.7, 1);
-      expect(counts["B"] / N).toBeCloseTo(0.3, 1);
     });
   });