Skip to content

Commit 403d92e

Browse files
committed
fix: reconcile audit samples and final execution status
Require latency sample counts to match reported requests before preparing default comparisons. Keep final failed or timed-out executions stopped even after all scheduled trials are recorded, and preserve that state when rebuilding reports. Agent-assisted-by: Codex (author) Signed-off-by: GUOGUO <xwdong1998@163.com>
1 parent 3b44b62 commit 403d92e

5 files changed

Lines changed: 58 additions & 20 deletions

File tree

‎docs/examples/skill-comparison/report-example.json‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,11 @@
55
"skill": "selected",
66
"revisions": {
77
"baseline": {
8-
"git_revision": "c27d81552b9f28c1eb8fae9706d4890bf37e88b8",
8+
"git_revision": "953aa2ac39e46573caedfff5179a3e5d36c9be3e",
99
"bundle_revision": "sha256:db78dfd40c109c9e99bfefe5006812f2e492d4002c9f5a256f4ec424bed0b59d"
1010
},
1111
"candidate": {
12-
"git_revision": "f4ad23f807fc9df5824532e086d04a64a5b736a1",
12+
"git_revision": "c269dc05581f58232de90ee5958098620d2aa9cf",
1313
"bundle_revision": "sha256:7fbb9ba7387ce05ec0ef18b559fbb914b2c2371c537e19c2bceddf18cad334ae"
1414
}
1515
},
@@ -30,7 +30,7 @@
3030
"grader": "a11ef015770706aa33e4c801dcf31c0bb08870c80dc8aa5b8a713d049f16b7ff"
3131
},
3232
"frozen_controllers": {
33-
"tools/skill_eval_compare.py": "295ec1f97d9b2c42359437973259a1f8e62238c1333771f70a631060d40ce7e1",
33+
"tools/skill_eval_compare.py": "8a8b35a40d8dbc0c203b2867cbcd02357ca77d1f63eaf6b0925ca1b0e211600d",
3434
"tools/skill_eval_pilot.py": "c7ec6c91463df119c9a8adffd772a799fbfdd371c6d25d97740a4b713e0f15aa",
3535
"src/inferrsi/harness/skill_evaluation.py": "8af9faf0fd083aa2731d284cebcf8caed135d6b3723bf1c869d1d79f81fde19d",
3636
"src/inferrsi/infra/executors/local.py": "b089cdc7daaaf7834698c2d280a12c5e755915ded926aa039c8de5f521bd46f0",
@@ -68,15 +68,15 @@
6868
"output_sha256": "d42de8a3924cc9b0867cfbf510433b510acad688f5a872829b235b8224e0dc2e",
6969
"retained_output_sha256": "d42de8a3924cc9b0867cfbf510433b510acad688f5a872829b235b8224e0dc2e",
7070
"evidence_sha256": {
71-
"execution.json": "c2f893f850a311ae1c7c4ab71dbcafb6cd4c78e7b4958dadf771a420c1614b56",
71+
"execution.json": "a6eab3e94000c3aade3ac1cbf5e88370522d91d57e4b28c100ccdbd6e1db8ace",
7272
"agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
7373
"agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
7474
"verification/result.json": "8a64fcee2c58d57083f195a727ba7a1b35d2f400bdf0699ec8d0eaac72b14f73",
7575
"verification/stdout.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
7676
"verification/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
7777
},
7878
"retained_evidence_sha256": {
79-
"execution.json": "c2f893f850a311ae1c7c4ab71dbcafb6cd4c78e7b4958dadf771a420c1614b56",
79+
"execution.json": "a6eab3e94000c3aade3ac1cbf5e88370522d91d57e4b28c100ccdbd6e1db8ace",
8080
"agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
8181
"agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
8282
"verification/result.json": "8a64fcee2c58d57083f195a727ba7a1b35d2f400bdf0699ec8d0eaac72b14f73",
@@ -102,15 +102,15 @@
102102
"output_sha256": "0f0c9f0f76ff65f67c097c86422dab0927b3e777d881df0d19e964c56d10cb20",
103103
"retained_output_sha256": "0f0c9f0f76ff65f67c097c86422dab0927b3e777d881df0d19e964c56d10cb20",
104104
"evidence_sha256": {
105-
"execution.json": "7e3f57d22fd7d4f3093685dd2ab0307f458b6ada999d1c785fd19b5a82f16666",
105+
"execution.json": "a68d8651dea2a336c856639eb0122c4c0946cef05eb0ecfd6bea8e69bd752209",
106106
"agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
107107
"agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
108108
"verification/result.json": "6ee705eb8e072554678b86346a7d11edab8c1307eb88a31771af08a39fbc7820",
109109
"verification/stdout.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
110110
"verification/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
111111
},
112112
"retained_evidence_sha256": {
113-
"execution.json": "7e3f57d22fd7d4f3093685dd2ab0307f458b6ada999d1c785fd19b5a82f16666",
113+
"execution.json": "a68d8651dea2a336c856639eb0122c4c0946cef05eb0ecfd6bea8e69bd752209",
114114
"agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
115115
"agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
116116
"verification/result.json": "6ee705eb8e072554678b86346a7d11edab8c1307eb88a31771af08a39fbc7820",
@@ -141,15 +141,15 @@
141141
"output_sha256": "0f0c9f0f76ff65f67c097c86422dab0927b3e777d881df0d19e964c56d10cb20",
142142
"retained_output_sha256": "0f0c9f0f76ff65f67c097c86422dab0927b3e777d881df0d19e964c56d10cb20",
143143
"evidence_sha256": {
144-
"execution.json": "69550a36b6c940ab8bb3ce4935c8ae1c73b349fd8772889645f6a5e4f091f64b",
144+
"execution.json": "ff72cb6a28db5adf030609cbdd285a391475cee2dee9c8a965565238020fb172",
145145
"agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
146146
"agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
147147
"verification/result.json": "6ee705eb8e072554678b86346a7d11edab8c1307eb88a31771af08a39fbc7820",
148148
"verification/stdout.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
149149
"verification/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
150150
},
151151
"retained_evidence_sha256": {
152-
"execution.json": "69550a36b6c940ab8bb3ce4935c8ae1c73b349fd8772889645f6a5e4f091f64b",
152+
"execution.json": "ff72cb6a28db5adf030609cbdd285a391475cee2dee9c8a965565238020fb172",
153153
"agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
154154
"agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
155155
"verification/result.json": "6ee705eb8e072554678b86346a7d11edab8c1307eb88a31771af08a39fbc7820",
@@ -175,15 +175,15 @@
175175
"output_sha256": "d42de8a3924cc9b0867cfbf510433b510acad688f5a872829b235b8224e0dc2e",
176176
"retained_output_sha256": "d42de8a3924cc9b0867cfbf510433b510acad688f5a872829b235b8224e0dc2e",
177177
"evidence_sha256": {
178-
"execution.json": "db32f242cb50b77b121bf47ad4d931ee62d091a8b188d56469e1d2c71a75104b",
178+
"execution.json": "9191efb9ae1b040d0eeef520aeadbe91db378b461132d22e705e9c8e479cf0c6",
179179
"agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
180180
"agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
181181
"verification/result.json": "8a64fcee2c58d57083f195a727ba7a1b35d2f400bdf0699ec8d0eaac72b14f73",
182182
"verification/stdout.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
183183
"verification/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
184184
},
185185
"retained_evidence_sha256": {
186-
"execution.json": "db32f242cb50b77b121bf47ad4d931ee62d091a8b188d56469e1d2c71a75104b",
186+
"execution.json": "9191efb9ae1b040d0eeef520aeadbe91db378b461132d22e705e9c8e479cf0c6",
187187
"agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
188188
"agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
189189
"verification/result.json": "8a64fcee2c58d57083f195a727ba7a1b35d2f400bdf0699ec8d0eaac72b14f73",
@@ -214,15 +214,15 @@
214214
"output_sha256": "0f0c9f0f76ff65f67c097c86422dab0927b3e777d881df0d19e964c56d10cb20",
215215
"retained_output_sha256": "0f0c9f0f76ff65f67c097c86422dab0927b3e777d881df0d19e964c56d10cb20",
216216
"evidence_sha256": {
217-
"execution.json": "89d40e49eb4ace0f2c5710999feb14ad43756406be45af47c6716bbdd24edf47",
217+
"execution.json": "9b989e4d07a49ece73426c7aa83dddfb0b881e4fc46d77ef3668aad6c3cd44a3",
218218
"agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
219219
"agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
220220
"verification/result.json": "6ee705eb8e072554678b86346a7d11edab8c1307eb88a31771af08a39fbc7820",
221221
"verification/stdout.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
222222
"verification/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
223223
},
224224
"retained_evidence_sha256": {
225-
"execution.json": "89d40e49eb4ace0f2c5710999feb14ad43756406be45af47c6716bbdd24edf47",
225+
"execution.json": "9b989e4d07a49ece73426c7aa83dddfb0b881e4fc46d77ef3668aad6c3cd44a3",
226226
"agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
227227
"agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
228228
"verification/result.json": "6ee705eb8e072554678b86346a7d11edab8c1307eb88a31771af08a39fbc7820",
@@ -238,12 +238,12 @@
238238
"output_sha256": "be302461cda285db3ed2d45fb767b23e08ca8808b35a638009937df22b332d73",
239239
"retained_output_sha256": "be302461cda285db3ed2d45fb767b23e08ca8808b35a638009937df22b332d73",
240240
"evidence_sha256": {
241-
"execution.json": "7c42d15813dcf68fd41894c2d23a5590f659f81275fc8ce1b5a2c2f69fecf149",
241+
"execution.json": "abff190d9105af18941244f1469515f219c610e590263d9b2f5a59414b07d4b4",
242242
"agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
243243
"agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
244244
},
245245
"retained_evidence_sha256": {
246-
"execution.json": "7c42d15813dcf68fd41894c2d23a5590f659f81275fc8ce1b5a2c2f69fecf149",
246+
"execution.json": "abff190d9105af18941244f1469515f219c610e590263d9b2f5a59414b07d4b4",
247247
"agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
248248
"agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
249249
}

‎docs/examples/skill-comparison/report-example.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,15 +71,15 @@ Candidate output and grading evidence for the regression:
7171
"output_sha256": "d42de8a3924cc9b0867cfbf510433b510acad688f5a872829b235b8224e0dc2e",
7272
"retained_output_sha256": "d42de8a3924cc9b0867cfbf510433b510acad688f5a872829b235b8224e0dc2e",
7373
"evidence_sha256": {
74-
"execution.json": "db32f242cb50b77b121bf47ad4d931ee62d091a8b188d56469e1d2c71a75104b",
74+
"execution.json": "9191efb9ae1b040d0eeef520aeadbe91db378b461132d22e705e9c8e479cf0c6",
7575
"agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
7676
"agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
7777
"verification/result.json": "8a64fcee2c58d57083f195a727ba7a1b35d2f400bdf0699ec8d0eaac72b14f73",
7878
"verification/stdout.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
7979
"verification/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
8080
},
8181
"retained_evidence_sha256": {
82-
"execution.json": "db32f242cb50b77b121bf47ad4d931ee62d091a8b188d56469e1d2c71a75104b",
82+
"execution.json": "9191efb9ae1b040d0eeef520aeadbe91db378b461132d22e705e9c8e479cf0c6",
8383
"agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
8484
"agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
8585
"verification/result.json": "8a64fcee2c58d57083f195a727ba7a1b35d2f400bdf0699ec8d0eaac72b14f73",

‎docs/skill-eval-comparison.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,7 @@ accepted contract cannot add promises that this grader does not enforce.
9797
| `mode=extract` | Requires an empty `contract`; checks descriptive reduction only and returns no acceptance verdict or claim. |
9898
| `evidence_scope` | `http_fixed_workload` or `kernel`. Kernel-only evidence produces an inconclusive service audit rather than a service PASS. |
9999
| Both reports | Require latency groups, workload, quality, and all three request counts. Workload needs positive integer `steps` and `concurrency`; optional `shape` must have at least one dimension, all positive integers. Differing valid workloads remain negative audit cases. |
100+
| Measurement coverage | The total number of latency samples across all groups must equal `reported_requests` in each arm, including extract mode. Groups may have different sizes; partial or extra observations are rejected before creating a plan. |
100101
| Audit baseline | Requires `quality="passed"` and zero failed requests; existing request accounting then requires every baseline request to be reported. Failed/missing baseline quality or failed baseline requests cannot support this default comparison. Candidate quality/completion failures remain valid negative audit cases. |
101102
| `evidence_keys` | Audit requires the complete reference set listed below, so negative cases remain answerable. Extract requires both latency paths only. Additional references may name existing report fields; unknown paths are rejected. The grader checks the answer's references and its computed verdict, findings, scope and reduction. |
102103

@@ -156,6 +157,10 @@ and elapsed time bound the run. The token threshold stops new trials using
156157
observed usage; it is not a provider-side hard token or monetary cap, and one
157158
trial can cross it. Missing/partial usage and monetary cost remain unknown.
158159
An execution failure stops new trials and retains the remaining missing sides.
160+
Run status is `completed` only when every scheduled trial is recorded and no
161+
execution stop reason exists. A failed or timed-out final trial remains `stopped`
162+
with `client_failed` or `client_timeout`, even when recorded and expected trial
163+
counts are equal.
159164
Before starting a real trial, execution resolves the configured client executable,
160165
probes its version without a model call, and uses that resolved executable for
161166
every trial. Execution evidence records its full command path locally; JSON and

‎tests/test_skill_eval_compare.py‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -242,6 +242,17 @@ def test_client_errors_cannot_be_fixes_or_regressions(tmp_path, failure, error):
242242
assert result["conditions"]["candidate"]["evaluation_errors"] == 1
243243
persisted = json.loads((output / "results/one-candidate-1/result.json").read_text())
244244
assert persisted["evaluation_error"] == error
245+
status = json.loads((output / "results/status.json").read_text())
246+
assert status["recorded_trials"] == status["expected_trials"] == len(client.calls) == 2
247+
if failure in ("timeout", "nonzero"):
248+
assert status["execution_state"] == "stopped"
249+
assert status["stop_reason"] == (
250+
"client_timeout" if failure == "timeout" else "client_failed")
251+
else:
252+
assert status["execution_state"] == "completed" and status["stop_reason"] is None
253+
regenerated = compare.report(output)
254+
for name in ("execution_state", "stop_reason", "recorded_trials", "expected_trials"):
255+
assert regenerated["status"][name] == result["status"][name] == status[name]
245256
if failure == "invalid_utf8":
246257
assert trial["output"] == "\\xff\\xfe"
247258
assert compare.report(output)["pairs"][0]["candidate"]["output"] == "\\xff\\xfe"
@@ -399,7 +410,7 @@ def test_default_audit_requires_complete_inputs():
399410

400411
@pytest.mark.parametrize("changes", [
401412
{"quality": "failed"}, {"quality": "missing"}, {"failed_requests": 1},
402-
{"failed_requests": 1, "reported_requests": 5},
413+
{"failed_requests": 1, "reported_requests": 5, "latencies_ms": [[100] * 5]},
403414
])
404415
def test_default_audit_rejects_unsupported_baseline_evidence(changes):
405416
task = copy.deepcopy(DATA["task"])
@@ -423,6 +434,12 @@ def test_default_audit_supported_gates_match_the_existing_grader():
423434
target[keys[-1]] = value
424435
if task["mode"] == "extract":
425436
task["contract"] = {}
437+
# Preserve the legacy verifier's branch expectations with complete comparison inputs.
438+
if case["id"] in ("below-threshold", "above-threshold"):
439+
task["candidate"]["latencies_ms"] = [group * 2
440+
for group in task["candidate"]["latencies_ms"]]
441+
if case["id"] == "omitted-failures":
442+
task["candidate"].update(total_requests=7, reported_requests=6)
426443
compare.validate_input(task, compare.CASE_INPUT)
427444
compare.validate_audit_values(task)
428445
compare.validate_default_audit(task)
@@ -437,6 +454,17 @@ def test_default_audit_supported_gates_match_the_existing_grader():
437454
compare.validate_default_audit(task)
438455

439456

457+
@pytest.mark.parametrize("role", compare.ROLES)
458+
@pytest.mark.parametrize("latencies", [[[1]], [[100] * 5], [[100] * 7]])
459+
def test_default_audit_requires_one_latency_sample_per_reported_request(role, latencies):
460+
task = copy.deepcopy(DATA["task"])
461+
task[role]["latencies_ms"] = latencies
462+
compare.validate_input(task, compare.CASE_INPUT)
463+
compare.validate_audit_values(task)
464+
with pytest.raises(ValueError, match="latency samples must match reported_requests"):
465+
compare.validate_default_audit(task)
466+
467+
440468
def test_default_audit_boundary_runs_before_output_and_covers_explicit_grader_copies(tmp_path):
441469
config, repo = configuration(tmp_path, ids=("one",))
442470
original = json.loads((tmp_path / "cases.json").read_text())
@@ -449,7 +477,8 @@ def test_default_audit_boundary_runs_before_output_and_covers_explicit_grader_co
449477
write_json(config, raw)
450478
for field, value in (("contract.statistical_significance_required", True),
451479
("contract.min_reduction_pct", float("nan")), ("evidence_keys", []),
452-
("baseline.quality", "failed"), ("baseline.failed_requests", 1)):
480+
("baseline.quality", "failed"), ("baseline.failed_requests", 1),
481+
("baseline.latencies_ms", [[1]]), ("candidate.latencies_ms", [[1]])):
453482
tasks = copy.deepcopy(original)
454483
target = tasks[0]
455484
keys = field.split(".")

‎tools/skill_eval_compare.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,9 @@ def validate_default_audit(task: dict) -> None:
130130
report = task[role]
131131
if REPORT_INPUT.keys() - report.keys():
132132
raise ValueError(f"Default audit requires complete report fields at {role}")
133+
if sum(len(group) for group in report["latencies_ms"]) != report["reported_requests"]:
134+
raise ValueError(
135+
f"Default audit latency samples must match reported_requests at {role}")
133136
workload = report["workload"]
134137
if {"steps", "concurrency"} - workload.keys() or any(
135138
workload.get(name, 0) <= 0 for name in ("steps", "concurrency")
@@ -429,7 +432,8 @@ def execute(output: Path, authorization: str, *, client=None) -> dict:
429432
status = {"client_runtime": runtime,
430433
"recorded_trials": completed, "expected_trials": len(plan["schedule"]),
431434
"observed_tokens": tokens, "stop_reason": stop_reason,
432-
"execution_state": "completed" if completed == len(plan["schedule"]) else "stopped"}
435+
"execution_state": "completed" if completed == len(plan["schedule"])
436+
and stop_reason is None else "stopped"}
433437
atomic_write_json(output / "results/status.json", status)
434438
atomic_write_json(output / "results/status.receipt.json",
435439
{"result_sha256": sha256(output / "results/status.json")})

0 commit comments

Comments
 (0)