Skip to content

Commit 199d700

Browse files
committed
fix: retain skill comparison runtime and status provenance
Reject executable Git entries rather than losing their modes in byte-only bundles. Resolve and record the client executable and version. Verify retained run status receipts before reporting completion or budgets. Agent-assisted-by: Codex (author) Signed-off-by: GUOGUO <xwdong1998@163.com>
1 parent 4b987fc commit 199d700

5 files changed

Lines changed: 246 additions & 46 deletions

File tree

‎docs/examples/skill-comparison/report-example.json‎

Lines changed: 48 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,11 @@
55
"skill": "selected",
66
"revisions": {
77
"baseline": {
8-
"git_revision": "3e4d897c5cdf80a1130cabe3bfcea3634b1e3aa1",
8+
"git_revision": "1ffced6b9f36571d802bfcd7a5b4a1907d2a1793",
99
"bundle_revision": "sha256:db78dfd40c109c9e99bfefe5006812f2e492d4002c9f5a256f4ec424bed0b59d"
1010
},
1111
"candidate": {
12-
"git_revision": "7dd28d4689de834e5fecc0c4e4785c3eb78ec8c5",
12+
"git_revision": "a419a9d87f5e7b7e740b15f9a227b45fc59fb19c",
1313
"bundle_revision": "sha256:7fbb9ba7387ce05ec0ef18b559fbb914b2c2371c537e19c2bceddf18cad334ae"
1414
}
1515
},
@@ -30,7 +30,7 @@
3030
"grader": "a11ef015770706aa33e4c801dcf31c0bb08870c80dc8aa5b8a713d049f16b7ff"
3131
},
3232
"frozen_controllers": {
33-
"tools/skill_eval_compare.py": "70c9e107c0197e8e9e3ffb26e2f2121bb678a1c76baabcb2bddd41f4f2e70e47",
33+
"tools/skill_eval_compare.py": "5f7c93265e148bb73bf908de81a1045e5bd37783929541f81205aedd2874058e",
3434
"tools/skill_eval_pilot.py": "c7ec6c91463df119c9a8adffd772a799fbfdd371c6d25d97740a4b713e0f15aa",
3535
"src/inferrsi/harness/skill_evaluation.py": "8af9faf0fd083aa2731d284cebcf8caed135d6b3723bf1c869d1d79f81fde19d",
3636
"src/inferrsi/infra/executors/local.py": "b089cdc7daaaf7834698c2d280a12c5e755915ded926aa039c8de5f521bd46f0",
@@ -67,15 +67,15 @@
6767
"output_sha256": "d42de8a3924cc9b0867cfbf510433b510acad688f5a872829b235b8224e0dc2e",
6868
"retained_output_sha256": "d42de8a3924cc9b0867cfbf510433b510acad688f5a872829b235b8224e0dc2e",
6969
"evidence_sha256": {
70-
"execution.json": "28d4d2dc461b8cf7f4e49928d925c0836ea1b895f4e44211f6513cc22e3b0d7e",
70+
"execution.json": "53d75f3e21860a95b134a87f02f09802b7215e74b7ae9c4d78f13e52917229f5",
7171
"agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
7272
"agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
7373
"verification/result.json": "8a64fcee2c58d57083f195a727ba7a1b35d2f400bdf0699ec8d0eaac72b14f73",
7474
"verification/stdout.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
7575
"verification/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
7676
},
7777
"retained_evidence_sha256": {
78-
"execution.json": "28d4d2dc461b8cf7f4e49928d925c0836ea1b895f4e44211f6513cc22e3b0d7e",
78+
"execution.json": "53d75f3e21860a95b134a87f02f09802b7215e74b7ae9c4d78f13e52917229f5",
7979
"agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
8080
"agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
8181
"verification/result.json": "8a64fcee2c58d57083f195a727ba7a1b35d2f400bdf0699ec8d0eaac72b14f73",
@@ -101,15 +101,15 @@
101101
"output_sha256": "0f0c9f0f76ff65f67c097c86422dab0927b3e777d881df0d19e964c56d10cb20",
102102
"retained_output_sha256": "0f0c9f0f76ff65f67c097c86422dab0927b3e777d881df0d19e964c56d10cb20",
103103
"evidence_sha256": {
104-
"execution.json": "f4f84a2d3a529723d400de7532afb850ab43e9889b754ebc63973c89320eb04f",
104+
"execution.json": "da576f9307b0358fc7fc587b3be26245639b13080ba48e2eaaca865817505999",
105105
"agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
106106
"agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
107107
"verification/result.json": "6ee705eb8e072554678b86346a7d11edab8c1307eb88a31771af08a39fbc7820",
108108
"verification/stdout.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
109109
"verification/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
110110
},
111111
"retained_evidence_sha256": {
112-
"execution.json": "f4f84a2d3a529723d400de7532afb850ab43e9889b754ebc63973c89320eb04f",
112+
"execution.json": "da576f9307b0358fc7fc587b3be26245639b13080ba48e2eaaca865817505999",
113113
"agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
114114
"agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
115115
"verification/result.json": "6ee705eb8e072554678b86346a7d11edab8c1307eb88a31771af08a39fbc7820",
@@ -140,15 +140,15 @@
140140
"output_sha256": "0f0c9f0f76ff65f67c097c86422dab0927b3e777d881df0d19e964c56d10cb20",
141141
"retained_output_sha256": "0f0c9f0f76ff65f67c097c86422dab0927b3e777d881df0d19e964c56d10cb20",
142142
"evidence_sha256": {
143-
"execution.json": "4fc1dbc26407ab588214097dda9e06029f95719d248e152c481efa3f66fb9ea3",
143+
"execution.json": "95407df91716ea943a50a659bfc42766f4939d849e6e533198467ea4ad68e0da",
144144
"agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
145145
"agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
146146
"verification/result.json": "6ee705eb8e072554678b86346a7d11edab8c1307eb88a31771af08a39fbc7820",
147147
"verification/stdout.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
148148
"verification/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
149149
},
150150
"retained_evidence_sha256": {
151-
"execution.json": "4fc1dbc26407ab588214097dda9e06029f95719d248e152c481efa3f66fb9ea3",
151+
"execution.json": "95407df91716ea943a50a659bfc42766f4939d849e6e533198467ea4ad68e0da",
152152
"agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
153153
"agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
154154
"verification/result.json": "6ee705eb8e072554678b86346a7d11edab8c1307eb88a31771af08a39fbc7820",
@@ -174,15 +174,15 @@
174174
"output_sha256": "d42de8a3924cc9b0867cfbf510433b510acad688f5a872829b235b8224e0dc2e",
175175
"retained_output_sha256": "d42de8a3924cc9b0867cfbf510433b510acad688f5a872829b235b8224e0dc2e",
176176
"evidence_sha256": {
177-
"execution.json": "5db7e5532b0d2b3df10860daceedeafb35a4bfd0a6dc39ec2019972cd6fabf05",
177+
"execution.json": "b5d2582493d136cc83753173f097d17ca8a74092a02eda9a3e19f42876a028b0",
178178
"agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
179179
"agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
180180
"verification/result.json": "8a64fcee2c58d57083f195a727ba7a1b35d2f400bdf0699ec8d0eaac72b14f73",
181181
"verification/stdout.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
182182
"verification/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
183183
},
184184
"retained_evidence_sha256": {
185-
"execution.json": "5db7e5532b0d2b3df10860daceedeafb35a4bfd0a6dc39ec2019972cd6fabf05",
185+
"execution.json": "b5d2582493d136cc83753173f097d17ca8a74092a02eda9a3e19f42876a028b0",
186186
"agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
187187
"agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
188188
"verification/result.json": "8a64fcee2c58d57083f195a727ba7a1b35d2f400bdf0699ec8d0eaac72b14f73",
@@ -213,15 +213,15 @@
213213
"output_sha256": "0f0c9f0f76ff65f67c097c86422dab0927b3e777d881df0d19e964c56d10cb20",
214214
"retained_output_sha256": "0f0c9f0f76ff65f67c097c86422dab0927b3e777d881df0d19e964c56d10cb20",
215215
"evidence_sha256": {
216-
"execution.json": "58629bf12679bb85048989603ea612b8594bebb41a2d1bda8a219c4b72650080",
216+
"execution.json": "e241f6c962c125501ef5ea5078864ad88d60e7191effde2b912c2af8239a6979",
217217
"agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
218218
"agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
219219
"verification/result.json": "6ee705eb8e072554678b86346a7d11edab8c1307eb88a31771af08a39fbc7820",
220220
"verification/stdout.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
221221
"verification/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
222222
},
223223
"retained_evidence_sha256": {
224-
"execution.json": "58629bf12679bb85048989603ea612b8594bebb41a2d1bda8a219c4b72650080",
224+
"execution.json": "e241f6c962c125501ef5ea5078864ad88d60e7191effde2b912c2af8239a6979",
225225
"agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
226226
"agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
227227
"verification/result.json": "6ee705eb8e072554678b86346a7d11edab8c1307eb88a31771af08a39fbc7820",
@@ -231,19 +231,19 @@
231231
},
232232
"candidate": {
233233
"verdict": "INCONCLUSIVE",
234-
"evaluation_error": "agent_timeout",
235-
"output": null,
234+
"evaluation_error": "invalid_output_json",
235+
"output": "{broken",
236236
"checks": null,
237-
"output_sha256": null,
238-
"retained_output_sha256": null,
237+
"output_sha256": "be302461cda285db3ed2d45fb767b23e08ca8808b35a638009937df22b332d73",
238+
"retained_output_sha256": "be302461cda285db3ed2d45fb767b23e08ca8808b35a638009937df22b332d73",
239239
"evidence_sha256": {
240-
"execution.json": "a386f33880adb31e245b4bfe22ffd7bbd0a4337adbd646ad974abf2291f2d0bb",
241-
"agent/stdout.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
240+
"execution.json": "e9742ec3244160b2bcf655594b693834b64592a232c33dff104ec7dee518085b",
241+
"agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
242242
"agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
243243
},
244244
"retained_evidence_sha256": {
245-
"execution.json": "a386f33880adb31e245b4bfe22ffd7bbd0a4337adbd646ad974abf2291f2d0bb",
246-
"agent/stdout.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
245+
"execution.json": "e9742ec3244160b2bcf655594b693834b64592a232c33dff104ec7dee518085b",
246+
"agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
247247
"agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
248248
}
249249
}
@@ -289,5 +289,32 @@
289289
},
290290
"extra_model_calls": 0,
291291
"reports_identical_after_restore": true
292+
},
293+
"status": {
294+
"client_runtime": {
295+
"implementation": "tests.fixtures.skill_eval_compare_support.FixedOutputClient",
296+
"executable": null,
297+
"executable_sha256": null,
298+
"version": null
299+
},
300+
"recorded_trials": 6,
301+
"expected_trials": 8,
302+
"observed_tokens": 120,
303+
"stop_reason": "token_budget",
304+
"execution_state": "stopped"
305+
},
306+
"status_integrity_control": {
307+
"mutation": "Only results/status.json changed",
308+
"status": {
309+
"execution_state": "unverified",
310+
"evaluation_error": "retained_status_record_hash_mismatch",
311+
"expected_trials": 8,
312+
"recorded_trials": null,
313+
"observed_tokens": null,
314+
"stop_reason": null,
315+
"client_runtime": null
316+
},
317+
"extra_model_calls": 0,
318+
"reports_identical_after_restore": true
292319
}
293320
}

‎docs/examples/skill-comparison/report-example.md‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,24 @@ JSON includes the recorded/current answer and execution/grader evidence hashes,
3232

3333
Each completed record also has a separate SHA-256 receipt. Changing only the regression candidate's recorded verdict/checks produced INCONCLUSIVE (`retained_trial_record_hash_mismatch`) and no accepted assertions; unchanged grader evidence could not become a positive pair. Restoring the original record regenerated identical reports without model or grader calls.
3434

35+
Client runtime and completion are retained in the verified run status. This injected offline client has unknown executable/version metadata. Editing only the status file produced unverified status with unknown budget fields; restoration reproduced identical reports without client or grader calls.
36+
37+
```json
38+
{
39+
"client_runtime": {
40+
"implementation": "tests.fixtures.skill_eval_compare_support.FixedOutputClient",
41+
"executable": null,
42+
"executable_sha256": null,
43+
"version": null
44+
},
45+
"recorded_trials": 6,
46+
"expected_trials": 8,
47+
"observed_tokens": 120,
48+
"stop_reason": "token_budget",
49+
"execution_state": "stopped"
50+
}
51+
```
52+
3553
Candidate output and grading evidence for the regression:
3654

3755
```json
@@ -53,15 +71,15 @@ Candidate output and grading evidence for the regression:
5371
"output_sha256": "d42de8a3924cc9b0867cfbf510433b510acad688f5a872829b235b8224e0dc2e",
5472
"retained_output_sha256": "d42de8a3924cc9b0867cfbf510433b510acad688f5a872829b235b8224e0dc2e",
5573
"evidence_sha256": {
56-
"execution.json": "5db7e5532b0d2b3df10860daceedeafb35a4bfd0a6dc39ec2019972cd6fabf05",
74+
"execution.json": "b5d2582493d136cc83753173f097d17ca8a74092a02eda9a3e19f42876a028b0",
5775
"agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
5876
"agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
5977
"verification/result.json": "8a64fcee2c58d57083f195a727ba7a1b35d2f400bdf0699ec8d0eaac72b14f73",
6078
"verification/stdout.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
6179
"verification/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
6280
},
6381
"retained_evidence_sha256": {
64-
"execution.json": "5db7e5532b0d2b3df10860daceedeafb35a4bfd0a6dc39ec2019972cd6fabf05",
82+
"execution.json": "b5d2582493d136cc83753173f097d17ca8a74092a02eda9a3e19f42876a028b0",
6583
"agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf",
6684
"agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
6785
"verification/result.json": "8a64fcee2c58d57083f195a727ba7a1b35d2f400bdf0699ec8d0eaac72b14f73",

‎docs/skill-eval-comparison.md‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,9 @@ Git's [`--no-replace-objects`](https://git-scm.com/docs/git#Documentation/git.tx
5151
The retained bundles include the full body, owner-local resources and declared
5252
references/dependencies. `reader.read(..., revision=...)` checks those bytes;
5353
it never retrieves a historical Git version.
54+
This entry point accepts regular non-executable Git blobs only. Executable
55+
entries (`100755`), symlinks and submodules are rejected during preparation;
56+
their semantics must not be silently changed by byte-only bundle export.
5457

5558
In addition mode, the baseline workspace contains only `task.json`, even if the
5659
source revision itself contains the target skill. That baseline commit may also
@@ -104,6 +107,13 @@ and elapsed time bound the run. The token threshold stops new trials using
104107
observed usage; it is not a provider-side hard token or monetary cap, and one
105108
trial can cross it. Missing/partial usage and monetary cost remain unknown.
106109
An execution failure stops new trials and retains the remaining missing sides.
110+
Before starting a real trial, execution resolves the configured client executable,
111+
probes its version without a model call, and uses that resolved executable for
112+
every trial. Execution evidence records its full command path locally; JSON and
113+
Markdown reports expose its basename, SHA-256 and version. An unavailable client
114+
or failed version probe fails before starting trials. Injected offline clients
115+
are identified by their implementation class; unavailable executable/version
116+
metadata remains null.
107117

108118
`CodexClient` disables global/bundled skills, apps, plugins, memories and web,
109119
and creates fresh sessions. Its controlled exposure is not OS read confinement.
@@ -152,6 +162,12 @@ assertions or usage, even when its answer and grader files are unchanged.
152162
Report regeneration leaves the original record and receipt untouched.
153163
These local checks detect missing or changed artifacts; they are not signatures
154164
against coordinated modification of both an artifact and its recorded hash.
165+
The final run status has its own `results/status.receipt.json`. Reporting checks
166+
it before using completion, trial counts, stop reason, observed tokens or client
167+
runtime. Missing/invalid proof or changed status bytes produce an `unverified`
168+
status with unknown completion/budget fields, while individually verified trial
169+
grades remain available. Neither report regeneration nor integrity checking
170+
invokes the client or grader.
155171

156172
`report.json` and `report.md` retain revisions, settings, limits, source/resource
157173
changes, case outputs, grading assertions and relative evidence links. Raw

0 commit comments

Comments
 (0)