Skip to content

Commit 2ad3e4f

Browse files
committed
fix: restore comparison results after plan artifact download
Recreate empty result directories in the PR adapter before invoking the unchanged executor. Cover addition and update through a file-only artifact round trip while retaining duplicate-attempt rejection. Agent-assisted-by: Codex / unknown [effort=unknown] (implementation) Signed-off-by: GUOGUO <xwdong1998@163.com>
1 parent eba5942 commit 2ad3e4f

3 files changed

Lines changed: 33 additions & 0 deletions

File tree

‎docs/skill-eval-pr.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,10 @@ The execute job has read-only repository permissions, no publication token in
9090
its model step, and checkout uses `persist-credentials: false`. The privileged
9191
publisher is a separate fresh job. Model access must not also grant PR publishing.
9292
The existing client's controlled skill exposure is not OS read confinement.
93+
Plan artifacts contain files rather than empty directories. The adapter restores
94+
each comparison's empty results directory after download; the existing executor
95+
still rejects nonempty results, and the exclusive execution claim still rejects
96+
duplicate attempts.
9397

9498
## Duplicate prevention and interrupted execution
9599

‎tests/test_skill_eval_pr.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -232,6 +232,33 @@ def trigger():
232232
integration.execute(output, approved, client=client)
233233

234234

235+
@pytest.mark.parametrize("mode", ["addition", "update"])
236+
def test_file_only_plan_artifact_round_trip_can_execute(tmp_path, mode):
237+
repo, mapping, identity = setup(tmp_path, mode, ids=("one",))
238+
prepared = tmp_path / "prepared"
239+
plan = integration.prepare(repo, mapping, identity, prepared)
240+
write_json(prepared / "approval.json", approval(plan))
241+
buffer = io.BytesIO()
242+
with zipfile.ZipFile(buffer, "w") as archive:
243+
for path in prepared.rglob("*"):
244+
if path.is_file():
245+
archive.write(path, path.relative_to(prepared).as_posix())
246+
downloaded = tmp_path / "downloaded"
247+
with zipfile.ZipFile(io.BytesIO(buffer.getvalue())) as archive:
248+
assert all(not entry.is_dir() for entry in archive.infolist())
249+
archive.extractall(downloaded)
250+
assert not (downloaded / "selected/results").exists()
251+
assert compare.load_plan(downloaded / "selected") == compare.load_plan(prepared / "selected")
252+
client = FixedOutputClient(tmp_path)
253+
integration.execute(downloaded, compare.read_json(downloaded / "approval.json"), client=client)
254+
status = compare.read_json(downloaded / "selected/results/status.json")
255+
assert status["execution_state"] == "completed" and status["recorded_trials"] == 2
256+
assert len(client.calls) == 2
257+
with pytest.raises(FileExistsError):
258+
integration.execute(downloaded, approval(plan), client=client)
259+
assert len(client.calls) == 2
260+
261+
235262
@pytest.mark.parametrize("mode", ["addition", "update"])
236263
def test_retained_results_publish_once_retry_without_model_and_reject_stale_jobs(tmp_path, mode):
237264
repo, mapping, identity = setup(tmp_path, mode, token_stop_new_trials=200)

‎tools/skill_eval_pr.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,8 @@ def execute(output, approval, *, client=None):
162162
handle.write(canonical(approval))
163163
for item in plan["comparisons"]:
164164
if "plan_sha256" in item:
165+
# Artifact downloads omit empty directories required by the executor.
166+
(output / item["skill"] / "results").mkdir(exist_ok=True)
165167
compare.execute(output / item["skill"], item["plan_sha256"], client=client)
166168

167169

0 commit comments

Comments
 (0)