Skip to content

Commit 3ce8e2c

Browse files
committed
fix: persist partial usage and invalid grader outcomes
Keep the controller's observed-token counter usable for partial usage in both new and recovered trials, and expose whether all usage fields were recorded. Retain malformed verifier payloads as raw artifacts while persisting only validated boolean checks. Cover both paths through real offline execution. Agent-assisted-by: Codex (author) Signed-off-by: GUOGUO <xwdong1998@163.com>
1 parent 9e5415f commit 3ce8e2c

6 files changed

Lines changed: 59 additions & 16 deletions

File tree

‎docs/skill-evaluation.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,10 @@ verdicts. Missing token/cost observations stay unknown, including partially
7777
reported token metrics. Cached input is a subset of
7878
input tokens, not an additional charge. Paired regressions and recoveries remain
7979
visible, including neutral/negative results.
80+
The controller's `observed_tokens` counts only known token fields and is a lower
81+
bound when `token_usage_complete` is false; unknown totals remain null in the
82+
summary. Invalid verifier payloads remain in their raw verification artifact;
83+
only validated boolean checks enter the trial record.
8084

8185
## CPU checks and paid execution
8286

‎src/inferrsi/harness/skill_evaluation.py‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -222,14 +222,17 @@ def run_trial(
222222
evaluation_error = "missing_verifier_result"
223223
else:
224224
try:
225-
checks = json.loads(result_path.read_text(encoding="utf-8"))
225+
raw_checks = json.loads(result_path.read_text(encoding="utf-8"))
226226
except ValueError:
227227
evaluation_error = "invalid_verifier_result"
228-
if evaluation_error is None:
229-
if not valid_checks(checks):
228+
else:
229+
if not valid_checks(raw_checks):
230230
evaluation_error = "invalid_verifier_result"
231-
elif verification["returncode"] != (0 if checks["passed"] else 1):
232-
evaluation_error = "verifier_exit"
231+
else:
232+
# 原始结果保留在 verification/result.json;记录只接收已校验字段。
233+
checks = {key: raw_checks[key] for key in ("passed", "assertions")}
234+
if verification["returncode"] != (0 if checks["passed"] else 1):
235+
evaluation_error = "verifier_exit"
233236
after = {
234237
p.relative_to(workspace).as_posix(): sha256(p) for p in workspace.rglob("*") if p.is_file()
235238
}

‎tests/fixtures/skill_eval_fake_client.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,5 +20,7 @@
2020
event = {"type": "turn.completed"}
2121
if mode != "unknown_usage":
2222
event["usage"] = {"input_tokens": 17, "output_tokens": 3}
23+
if mode == "partial_usage":
24+
del event["usage"]["output_tokens"]
2325
print("{broken" if mode == "invalid_trace" else json.dumps(event), flush=True)
2426
raise SystemExit(7 if mode == "nonzero" else 0)

‎tests/fixtures/skill_eval_faulty_verifier.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,14 @@
1818
"string_assertion": {"passed": True, "assertions": {"check": "false"}},
1919
"inconsistent": {"passed": True, "assertions": {"check": False}},
2020
"wrong_exit": {"passed": True, "assertions": {"check": True}},
21+
"nan_assertion": {"passed": False, "assertions": {"check": float("nan")}},
22+
"infinite_assertion": {"passed": False, "assertions": {"check": float("inf")}},
2123
}
24+
if mode == "overflow_assertion":
25+
payload = '{"passed":false,"assertions":{"check":1e999}}'
26+
else:
27+
payload = "{broken" if mode == "invalid_json" else json.dumps(results[mode])
2228
Path(os.environ["INFERRSI_RESULT_PATH"]).write_text(
23-
"{broken" if mode == "invalid_json" else json.dumps(results[mode]), encoding="utf-8"
29+
payload, encoding="utf-8"
2430
)
2531
raise SystemExit(7 if mode == "wrong_exit" else 0)

‎tests/test_skill_eval_pipeline.py‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,10 @@ def test_client_failures_are_persisted_evaluation_errors(tmp_path, mode, state,
109109
("missing_assertions", "invalid_verifier_result"),
110110
("empty_assertions", "invalid_verifier_result"),
111111
("string_assertion", "invalid_verifier_result"),
112-
("inconsistent", "invalid_verifier_result"), ("wrong_exit", "verifier_exit")],
112+
("inconsistent", "invalid_verifier_result"), ("wrong_exit", "verifier_exit"),
113+
("nan_assertion", "invalid_verifier_result"),
114+
("infinite_assertion", "invalid_verifier_result"),
115+
("overflow_assertion", "invalid_verifier_result")],
113116
)
114117
def test_verifier_faults_cannot_be_completed_passes(tmp_path, fault, error):
115118
record = run_case(tmp_path, fault=fault)
@@ -128,7 +131,7 @@ def test_pilot_persists_mixed_summary_without_model_calls(tmp_path, monkeypatch)
128131
for name in ("plan", "validator", "results", "patch"):
129132
(tmp_path / name).mkdir()
130133
tasks = []
131-
for mode in ("correct", "incorrect", "missing_output", "unknown_usage"):
134+
for mode in ("correct", "incorrect", "missing_output", "unknown_usage", "partial_usage"):
132135
tasks.append({**copy.deepcopy(DATA["task"]), "id": mode, "client_mode": mode})
133136
atomic_write_json(tmp_path / "plan/tasks.json", tasks)
134137
atomic_write_json(tmp_path / "plan/schema.json", skill_eval_pilot.SCHEMA)
@@ -152,16 +155,26 @@ def test_pilot_persists_mixed_summary_without_model_calls(tmp_path, monkeypatch)
152155
skill_eval_pilot.execute(tmp_path)
153156
summary_path = tmp_path / "results/summary.json"
154157
group = json.loads(summary_path.read_text(encoding="utf-8"))["conditions"]["N"]
155-
assert group["trials"] == 4
156-
assert group["verdicts"] == {"PASS": 2, "FAIL": 1, "INCONCLUSIVE": 1}
158+
assert group["trials"] == 5
159+
assert group["verdicts"] == {"PASS": 3, "FAIL": 1, "INCONCLUSIVE": 1}
157160
assert group["evaluation_errors"] == 1
158161
assert group["input_tokens"] is None and group["output_tokens"] is None
159162
assert group["cost"] is None
160163
trials = json.loads((tmp_path / "results/trials.json").read_text(encoding="utf-8"))
161-
assert [trial["verdict"] for trial in trials] == ["PASS", "FAIL", "INCONCLUSIVE", "PASS"]
164+
assert [trial["verdict"] for trial in trials] == [
165+
"PASS", "FAIL", "INCONCLUSIVE", "PASS", "PASS",
166+
]
162167
for trial in trials:
163168
persisted = tmp_path / "results" / (trial["task_id"] + "-N-1") / "result.json"
164169
assert json.loads(persisted.read_text(encoding="utf-8")) == trial
170+
status_path = tmp_path / "results/status.json"
171+
status = json.loads(status_path.read_text(encoding="utf-8"))
172+
assert status["observed_tokens"] == 77
173+
assert status["token_usage_complete"] is False
174+
# 恢复已完成记录也必须能读取部分 usage,且不能重跑客户端。
175+
monkeypatch.setattr(FakeClient, "command", lambda *args: pytest.fail("Unexpected rerun"))
176+
skill_eval_pilot.execute(tmp_path)
177+
assert json.loads(status_path.read_text(encoding="utf-8"))["observed_tokens"] == 77
165178

166179

167180
def test_partial_usage_stays_unknown_per_metric():

‎tools/skill_eval_pilot.py‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -279,6 +279,16 @@ def recover_attempts(output: Path, schedule: list[dict]) -> list[dict]:
279279
return trials
280280

281281

282+
def observed_tokens(usage: dict | None) -> int:
283+
# 预算计数只累计已观测字段;完整性在状态与 summary 中单独表达。
284+
if usage is None:
285+
return 0
286+
return sum(
287+
value for key, value in usage.items()
288+
if key in ("input_tokens", "output_tokens") and value is not None
289+
)
290+
291+
282292
def execute(output: Path, max_new_trials: int | None = None):
283293
plan = json.loads((output / "plan/freeze.json").read_text(encoding="utf-8"))
284294
assert sha256(output / "plan/tasks.json") == plan["tasks_sha256"]
@@ -297,9 +307,7 @@ def execute(output: Path, max_new_trials: int | None = None):
297307
assert reader.read("checkout:" + name).revision == bundle["revision"]
298308
trials = recover_attempts(output, plan["schedule"])
299309
tokens = sum(
300-
t["observations"]["usage"]["input_tokens"] + t["observations"]["usage"]["output_tokens"]
301-
for t in trials
302-
if t["observations"]["usage"]
310+
observed_tokens(t["observations"]["usage"]) for t in trials
303311
)
304312
prior = {(t["task_id"], t["condition"], t["repeat"]) for t in trials}
305313
start_path = output / "results/start.json"
@@ -387,8 +395,7 @@ def execute(output: Path, max_new_trials: int | None = None):
387395
trials.append(result)
388396
new_trials += 1
389397
usage = result["observations"]["usage"]
390-
if usage:
391-
tokens += usage["input_tokens"] + usage["output_tokens"]
398+
tokens += observed_tokens(usage)
392399
write_json(trial_dir / "result.json", result)
393400
write_json(output / "results/trials.json", trials)
394401
write_json(output / "results/summary.json", summarize(trials))
@@ -413,6 +420,14 @@ def execute(output: Path, max_new_trials: int | None = None):
413420
"expected_trials": len(plan["schedule"]),
414421
"recorded_trials": len(trials),
415422
"observed_tokens": tokens,
423+
"token_usage_complete": all(
424+
trial["observations"]["usage"] is not None
425+
and all(
426+
trial["observations"]["usage"].get(key) is not None
427+
for key in ("input_tokens", "output_tokens")
428+
)
429+
for trial in trials
430+
),
416431
"execution_state": "completed" if len(trials) == len(plan["schedule"]) else "stopped",
417432
"stop_reason": stop_reason,
418433
},

0 commit comments

Comments
 (0)