Skip to content

Commit bcbfbde

Browse files
committed
fix(harness): keep a protocol's own exception type in the coordinator record (R3)
Review (zengchuang-hw, 096f93c): when a frozen optimization.py raised in its `arguments` operation, coordinator-error.json recorded type "TaskProtocolError" instead of the protocol's own type, which the in-process call recorded before R3. The settlement reason on that path was and stays the existing coordinator behaviour, "ValueError: <message>". - TaskProtocolError gains `type_name`: the protocol's own exception type for kind protocol_exception, "TaskProtocolError" for every other kind. It replaces the `reason` attribute added in 096f93c. - optimization._type_name() feeds both _failure and coordinator-error.json, so all three operations leave the records they left before R3. - New regression drives `session start` and a real `round` with an `arguments` TypeError: coordinator-error.json is {"type": "TypeError", "error": "bad arguments"}, the attempt settles as "ValueError: bad arguments" and the benchmark never starts. - CHANGELOG states the record per operation instead of one broad claim. Refs #412 Agent-assisted-by: Claude Code / claude-opus-5-5 [effort=xhigh] (author) Signed-off-by: GUOGUO <xwdong1998@163.com>
1 parent 096f93c commit bcbfbde

5 files changed

Lines changed: 51 additions & 18 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,12 @@
1313
its own process group or Windows Job, and returns one strict JSON envelope.
1414
The child compiles only protocol bytes that match the frozen SHA-256, and a
1515
protocol `SystemExit` keeps its exit code (argparse usage errors still exit 2).
16-
An exception the protocol raises keeps its own message and its
17-
`"<Type>: <message>"` settlement reason, as before; transport, identity and
18-
result failures are reported as `TaskProtocolError` with their kind.
16+
An exception the protocol raises is recorded as before R3, under its own type and
17+
message: an `assess` failure settles as `"<Type>: <message>"`, an `arguments`
18+
failure keeps its type and message in `coordinator-error.json` and settles through
19+
the existing coordinator path, and a `validate_arguments` failure prints its own
20+
message at `session start`. Transport, identity and result failures are recorded
21+
as `TaskProtocolError` with their kind.
1922
After every operation the last 4 KiB of the child's stderr, protocol prints
2023
included, is copied to the caller's stderr; the scratch directory is then
2124
deleted. The 16 MiB envelope limit and the stderr tail apply when Harness reads

‎src/inferrsi/harness/optimization.py‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -210,13 +210,15 @@ def _summary(root, plan, state):
210210
}
211211

212212

213+
def _type_name(error):
214+
# A task protocol that raised is recorded under its own exception type across the
215+
# child transport (#412 R3); any other error under its own Python type.
216+
return error.type_name if isinstance(error, TaskProtocolError) else type(error).__name__
217+
218+
213219
def _failure(error):
214-
# A task protocol that raised keeps its own "<Type>: <message>" across the child
215-
# transport (#412 R3); any other error is named by its own type.
216-
reason = (error.reason if isinstance(error, TaskProtocolError)
217-
else f"{type(error).__name__}: {error}")
218220
return {"status": "infrastructure_failure", "score": None,
219-
"verdict": "INCONCLUSIVE", "reason": reason}
221+
"verdict": "INCONCLUSIVE", "reason": f"{_type_name(error)}: {error}"}
220222

221223

222224
def _settle(root, plan, state, pending, execution, assess):
@@ -348,7 +350,7 @@ def evaluate_round(root: Path, *, candidate: Path | None, hypothesis: str,
348350
# failed. Collect it without launching the same candidate again.
349351
path = directory / "execution/execution.json"
350352
atomic_write_json(directory / "coordinator-error.json", {
351-
"type": type(error).__name__, "error": str(error)})
353+
"type": _type_name(error), "error": str(error)})
352354
if path.is_file():
353355
execution = json.loads(path.read_text(encoding="utf-8"))
354356
else:

‎src/inferrsi/harness/task_protocol.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -34,17 +34,17 @@
3434
class TaskProtocolError(ValueError):
3535
"""A protocol operation gave no valid result. ``kind`` names the failure class.
3636
37-
For ``protocol_exception`` the protocol itself raised: ``str()`` is its own message
38-
and ``reason`` its ``"<Type>: <message>"``, the text the in-process call gave before
39-
#412 R3. For every other kind both name the transport failure, so the two stay
40-
distinguishable.
37+
``type_name`` is the type a caller records for this failure. For
38+
``protocol_exception`` the protocol itself raised: ``str()`` is its own message and
39+
``type_name`` its own exception type, the record the in-process call left before
40+
#412 R3. For every other kind ``type_name`` is ``TaskProtocolError`` and ``str()``
41+
starts with the kind, so transport, identity and result failures stay distinct.
4142
"""
4243

4344
def __init__(self, kind: str, message: str, *, error_type: str | None = None):
4445
super().__init__(message if kind == "protocol_exception" else f"{kind}: {message}")
4546
self.kind = kind
46-
self.reason = (f"{error_type}: {message}" if kind == "protocol_exception"
47-
else f"{type(self).__name__}: {self}")
47+
self.type_name = error_type if kind == "protocol_exception" else type(self).__name__
4848

4949

5050
def _tail(path: Path) -> str:

‎tests/test_optimization_session.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -337,6 +337,34 @@ def test_child_assessor_exception_keeps_its_reason_through_a_real_round(task, tm
337337
assert (root / "calls.log").read_text().splitlines() == ["2"]
338338

339339

340+
def test_child_arguments_exception_keeps_the_coordinator_record_through_a_real_round(
341+
task, tmp_path, capsys,
342+
):
343+
# #412 R3: a frozen protocol's `arguments` failure leaves the same coordinator record
344+
# as the in-process call did: the protocol's own type and message, and the attempt
345+
# settles through the existing coordinator path. The benchmark never starts.
346+
path = task / "optimization.py"
347+
path.write_text(path.read_text(encoding="utf-8").replace(
348+
"def arguments(candidate, output, settings):\n",
349+
"def arguments(candidate, output, settings):\n"
350+
" raise TypeError('bad arguments')\n"), encoding="utf-8")
351+
root = tmp_path / "session"
352+
assert main(start_args(task, root)) == 0
353+
capsys.readouterr()
354+
candidate = root / "work/candidate.py"
355+
candidate.write_text("FACTOR = 2\n", encoding="utf-8")
356+
assert main(["benchmarks", "session", "round", "--session", str(root),
357+
"--candidate", str(candidate), "--hypothesis", "Child arguments error"]) == 2
358+
view = json.loads(capsys.readouterr().out)
359+
directory = root / view["latest_observation"]["path"]
360+
assert json.loads((directory / "coordinator-error.json").read_text()) == {
361+
"type": "TypeError", "error": "bad arguments"}
362+
assessment = view["latest_observation"]["assessment"]
363+
assert assessment["reason"] == "ValueError: bad arguments"
364+
assert assessment["verdict"] == "INCONCLUSIVE" and assessment["score"] is None
365+
assert not (root / "calls.log").exists()
366+
367+
340368
def test_child_argument_error_keeps_its_cli_message_at_start(task, tmp_path, capsys):
341369
path = task / "optimization.py"
342370
path.write_text(path.read_text(encoding="utf-8").replace(

‎tests/test_task_protocol_child.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -70,15 +70,15 @@ def test_protocol_exception_keeps_its_type_and_message(tmp_path):
7070
assert error.kind == "protocol_exception"
7171
# The protocol's own message and "<Type>: <message>" survive the child transport.
7272
assert str(error) == "bad settings"
73-
assert error.reason == "TypeError: bad settings"
73+
assert error.type_name == "TypeError"
7474

7575

7676
def test_transport_failures_stay_distinguishable_from_protocol_exceptions(tmp_path):
7777
digest = write_protocol(tmp_path,
7878
"def validate_arguments(a):\n return {'x': float('nan')}\n")
7979
error = failure(tmp_path, digest)
8080
assert error.kind == "result"
81-
assert error.reason == f"TaskProtocolError: {error}"
81+
assert error.type_name == "TaskProtocolError"
8282
assert str(error).startswith("result: ValueError: ")
8383

8484

@@ -131,7 +131,7 @@ def test_diagnostics_are_forwarded_when_the_protocol_raises(tmp_path, capsys):
131131
" print('context before failure', file=sys.stderr)\n"
132132
" raise ValueError('rejected')\n"))
133133
error = failure(tmp_path, digest)
134-
assert str(error) == "rejected" and error.reason == "ValueError: rejected"
134+
assert str(error) == "rejected" and error.type_name == "ValueError"
135135
assert "context before failure" in capsys.readouterr().err
136136

137137

0 commit comments

Comments
 (0)