|
5 | 5 | "skill": "selected", |
6 | 6 | "revisions": { |
7 | 7 | "baseline": { |
8 | | - "git_revision": "3e4d897c5cdf80a1130cabe3bfcea3634b1e3aa1", |
| 8 | + "git_revision": "1ffced6b9f36571d802bfcd7a5b4a1907d2a1793", |
9 | 9 | "bundle_revision": "sha256:db78dfd40c109c9e99bfefe5006812f2e492d4002c9f5a256f4ec424bed0b59d" |
10 | 10 | }, |
11 | 11 | "candidate": { |
12 | | - "git_revision": "7dd28d4689de834e5fecc0c4e4785c3eb78ec8c5", |
| 12 | + "git_revision": "a419a9d87f5e7b7e740b15f9a227b45fc59fb19c", |
13 | 13 | "bundle_revision": "sha256:7fbb9ba7387ce05ec0ef18b559fbb914b2c2371c537e19c2bceddf18cad334ae" |
14 | 14 | } |
15 | 15 | }, |
|
30 | 30 | "grader": "a11ef015770706aa33e4c801dcf31c0bb08870c80dc8aa5b8a713d049f16b7ff" |
31 | 31 | }, |
32 | 32 | "frozen_controllers": { |
33 | | - "tools/skill_eval_compare.py": "70c9e107c0197e8e9e3ffb26e2f2121bb678a1c76baabcb2bddd41f4f2e70e47", |
| 33 | + "tools/skill_eval_compare.py": "5f7c93265e148bb73bf908de81a1045e5bd37783929541f81205aedd2874058e", |
34 | 34 | "tools/skill_eval_pilot.py": "c7ec6c91463df119c9a8adffd772a799fbfdd371c6d25d97740a4b713e0f15aa", |
35 | 35 | "src/inferrsi/harness/skill_evaluation.py": "8af9faf0fd083aa2731d284cebcf8caed135d6b3723bf1c869d1d79f81fde19d", |
36 | 36 | "src/inferrsi/infra/executors/local.py": "b089cdc7daaaf7834698c2d280a12c5e755915ded926aa039c8de5f521bd46f0", |
|
67 | 67 | "output_sha256": "d42de8a3924cc9b0867cfbf510433b510acad688f5a872829b235b8224e0dc2e", |
68 | 68 | "retained_output_sha256": "d42de8a3924cc9b0867cfbf510433b510acad688f5a872829b235b8224e0dc2e", |
69 | 69 | "evidence_sha256": { |
70 | | - "execution.json": "28d4d2dc461b8cf7f4e49928d925c0836ea1b895f4e44211f6513cc22e3b0d7e", |
| 70 | + "execution.json": "53d75f3e21860a95b134a87f02f09802b7215e74b7ae9c4d78f13e52917229f5", |
71 | 71 | "agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf", |
72 | 72 | "agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", |
73 | 73 | "verification/result.json": "8a64fcee2c58d57083f195a727ba7a1b35d2f400bdf0699ec8d0eaac72b14f73", |
74 | 74 | "verification/stdout.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", |
75 | 75 | "verification/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" |
76 | 76 | }, |
77 | 77 | "retained_evidence_sha256": { |
78 | | - "execution.json": "28d4d2dc461b8cf7f4e49928d925c0836ea1b895f4e44211f6513cc22e3b0d7e", |
| 78 | + "execution.json": "53d75f3e21860a95b134a87f02f09802b7215e74b7ae9c4d78f13e52917229f5", |
79 | 79 | "agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf", |
80 | 80 | "agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", |
81 | 81 | "verification/result.json": "8a64fcee2c58d57083f195a727ba7a1b35d2f400bdf0699ec8d0eaac72b14f73", |
|
101 | 101 | "output_sha256": "0f0c9f0f76ff65f67c097c86422dab0927b3e777d881df0d19e964c56d10cb20", |
102 | 102 | "retained_output_sha256": "0f0c9f0f76ff65f67c097c86422dab0927b3e777d881df0d19e964c56d10cb20", |
103 | 103 | "evidence_sha256": { |
104 | | - "execution.json": "f4f84a2d3a529723d400de7532afb850ab43e9889b754ebc63973c89320eb04f", |
| 104 | + "execution.json": "da576f9307b0358fc7fc587b3be26245639b13080ba48e2eaaca865817505999", |
105 | 105 | "agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf", |
106 | 106 | "agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", |
107 | 107 | "verification/result.json": "6ee705eb8e072554678b86346a7d11edab8c1307eb88a31771af08a39fbc7820", |
108 | 108 | "verification/stdout.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", |
109 | 109 | "verification/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" |
110 | 110 | }, |
111 | 111 | "retained_evidence_sha256": { |
112 | | - "execution.json": "f4f84a2d3a529723d400de7532afb850ab43e9889b754ebc63973c89320eb04f", |
| 112 | + "execution.json": "da576f9307b0358fc7fc587b3be26245639b13080ba48e2eaaca865817505999", |
113 | 113 | "agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf", |
114 | 114 | "agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", |
115 | 115 | "verification/result.json": "6ee705eb8e072554678b86346a7d11edab8c1307eb88a31771af08a39fbc7820", |
|
140 | 140 | "output_sha256": "0f0c9f0f76ff65f67c097c86422dab0927b3e777d881df0d19e964c56d10cb20", |
141 | 141 | "retained_output_sha256": "0f0c9f0f76ff65f67c097c86422dab0927b3e777d881df0d19e964c56d10cb20", |
142 | 142 | "evidence_sha256": { |
143 | | - "execution.json": "4fc1dbc26407ab588214097dda9e06029f95719d248e152c481efa3f66fb9ea3", |
| 143 | + "execution.json": "95407df91716ea943a50a659bfc42766f4939d849e6e533198467ea4ad68e0da", |
144 | 144 | "agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf", |
145 | 145 | "agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", |
146 | 146 | "verification/result.json": "6ee705eb8e072554678b86346a7d11edab8c1307eb88a31771af08a39fbc7820", |
147 | 147 | "verification/stdout.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", |
148 | 148 | "verification/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" |
149 | 149 | }, |
150 | 150 | "retained_evidence_sha256": { |
151 | | - "execution.json": "4fc1dbc26407ab588214097dda9e06029f95719d248e152c481efa3f66fb9ea3", |
| 151 | + "execution.json": "95407df91716ea943a50a659bfc42766f4939d849e6e533198467ea4ad68e0da", |
152 | 152 | "agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf", |
153 | 153 | "agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", |
154 | 154 | "verification/result.json": "6ee705eb8e072554678b86346a7d11edab8c1307eb88a31771af08a39fbc7820", |
|
174 | 174 | "output_sha256": "d42de8a3924cc9b0867cfbf510433b510acad688f5a872829b235b8224e0dc2e", |
175 | 175 | "retained_output_sha256": "d42de8a3924cc9b0867cfbf510433b510acad688f5a872829b235b8224e0dc2e", |
176 | 176 | "evidence_sha256": { |
177 | | - "execution.json": "5db7e5532b0d2b3df10860daceedeafb35a4bfd0a6dc39ec2019972cd6fabf05", |
| 177 | + "execution.json": "b5d2582493d136cc83753173f097d17ca8a74092a02eda9a3e19f42876a028b0", |
178 | 178 | "agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf", |
179 | 179 | "agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", |
180 | 180 | "verification/result.json": "8a64fcee2c58d57083f195a727ba7a1b35d2f400bdf0699ec8d0eaac72b14f73", |
181 | 181 | "verification/stdout.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", |
182 | 182 | "verification/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" |
183 | 183 | }, |
184 | 184 | "retained_evidence_sha256": { |
185 | | - "execution.json": "5db7e5532b0d2b3df10860daceedeafb35a4bfd0a6dc39ec2019972cd6fabf05", |
| 185 | + "execution.json": "b5d2582493d136cc83753173f097d17ca8a74092a02eda9a3e19f42876a028b0", |
186 | 186 | "agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf", |
187 | 187 | "agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", |
188 | 188 | "verification/result.json": "8a64fcee2c58d57083f195a727ba7a1b35d2f400bdf0699ec8d0eaac72b14f73", |
|
213 | 213 | "output_sha256": "0f0c9f0f76ff65f67c097c86422dab0927b3e777d881df0d19e964c56d10cb20", |
214 | 214 | "retained_output_sha256": "0f0c9f0f76ff65f67c097c86422dab0927b3e777d881df0d19e964c56d10cb20", |
215 | 215 | "evidence_sha256": { |
216 | | - "execution.json": "58629bf12679bb85048989603ea612b8594bebb41a2d1bda8a219c4b72650080", |
| 216 | + "execution.json": "e241f6c962c125501ef5ea5078864ad88d60e7191effde2b912c2af8239a6979", |
217 | 217 | "agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf", |
218 | 218 | "agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", |
219 | 219 | "verification/result.json": "6ee705eb8e072554678b86346a7d11edab8c1307eb88a31771af08a39fbc7820", |
220 | 220 | "verification/stdout.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", |
221 | 221 | "verification/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" |
222 | 222 | }, |
223 | 223 | "retained_evidence_sha256": { |
224 | | - "execution.json": "58629bf12679bb85048989603ea612b8594bebb41a2d1bda8a219c4b72650080", |
| 224 | + "execution.json": "e241f6c962c125501ef5ea5078864ad88d60e7191effde2b912c2af8239a6979", |
225 | 225 | "agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf", |
226 | 226 | "agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", |
227 | 227 | "verification/result.json": "6ee705eb8e072554678b86346a7d11edab8c1307eb88a31771af08a39fbc7820", |
|
231 | 231 | }, |
232 | 232 | "candidate": { |
233 | 233 | "verdict": "INCONCLUSIVE", |
234 | | - "evaluation_error": "agent_timeout", |
235 | | - "output": null, |
| 234 | + "evaluation_error": "invalid_output_json", |
| 235 | + "output": "{broken", |
236 | 236 | "checks": null, |
237 | | - "output_sha256": null, |
238 | | - "retained_output_sha256": null, |
| 237 | + "output_sha256": "be302461cda285db3ed2d45fb767b23e08ca8808b35a638009937df22b332d73", |
| 238 | + "retained_output_sha256": "be302461cda285db3ed2d45fb767b23e08ca8808b35a638009937df22b332d73", |
239 | 239 | "evidence_sha256": { |
240 | | - "execution.json": "a386f33880adb31e245b4bfe22ffd7bbd0a4337adbd646ad974abf2291f2d0bb", |
241 | | - "agent/stdout.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", |
| 240 | + "execution.json": "e9742ec3244160b2bcf655594b693834b64592a232c33dff104ec7dee518085b", |
| 241 | + "agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf", |
242 | 242 | "agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" |
243 | 243 | }, |
244 | 244 | "retained_evidence_sha256": { |
245 | | - "execution.json": "a386f33880adb31e245b4bfe22ffd7bbd0a4337adbd646ad974abf2291f2d0bb", |
246 | | - "agent/stdout.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", |
| 245 | + "execution.json": "e9742ec3244160b2bcf655594b693834b64592a232c33dff104ec7dee518085b", |
| 246 | + "agent/stdout.log": "cae0a0049534a8720917a328866e40de12e3a95495ab4f5c07cbeb525eb7ccdf", |
247 | 247 | "agent/stderr.log": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" |
248 | 248 | } |
249 | 249 | } |
|
289 | 289 | }, |
290 | 290 | "extra_model_calls": 0, |
291 | 291 | "reports_identical_after_restore": true |
| 292 | + }, |
| 293 | + "status": { |
| 294 | + "client_runtime": { |
| 295 | + "implementation": "tests.fixtures.skill_eval_compare_support.FixedOutputClient", |
| 296 | + "executable": null, |
| 297 | + "executable_sha256": null, |
| 298 | + "version": null |
| 299 | + }, |
| 300 | + "recorded_trials": 6, |
| 301 | + "expected_trials": 8, |
| 302 | + "observed_tokens": 120, |
| 303 | + "stop_reason": "token_budget", |
| 304 | + "execution_state": "stopped" |
| 305 | + }, |
| 306 | + "status_integrity_control": { |
| 307 | + "mutation": "Only results/status.json changed", |
| 308 | + "status": { |
| 309 | + "execution_state": "unverified", |
| 310 | + "evaluation_error": "retained_status_record_hash_mismatch", |
| 311 | + "expected_trials": 8, |
| 312 | + "recorded_trials": null, |
| 313 | + "observed_tokens": null, |
| 314 | + "stop_reason": null, |
| 315 | + "client_runtime": null |
| 316 | + }, |
| 317 | + "extra_model_calls": 0, |
| 318 | + "reports_identical_after_restore": true |
292 | 319 | } |
293 | 320 | } |
0 commit comments