From 7b29c02b6cfe8a255d5f62e2d3e393c619e75ba6 Mon Sep 17 00:00:00 2001 From: Alex Chi Z Date: Wed, 12 Aug 2026 04:48:29 -0700 Subject: [PATCH 1/4] course: add Week 4 outcome evaluation AI-Assisted: GPT-5.6 Sol + Forge --- README.md | 11 +- book/src/SUMMARY.md | 1 + book/src/week4-06-steering.md | 5 +- book/src/week4-07-evaluation.md | 385 +++++++++-------------- book/src/week4-overview.md | 17 +- src/tiny_llm/agent/__init__.py | 16 + src/tiny_llm/agent/evaluation.py | 94 ++++++ src/tiny_llm_ref/agent/__init__.py | 16 + src/tiny_llm_ref/agent/evaluation.py | 239 ++++++++++++++ tests_refsol/test_week_4_day_7.py | 270 ++++++++++++++++ tests_refsol/test_week_4_starter_sync.py | 32 +- 11 files changed, 826 insertions(+), 260 deletions(-) create mode 100644 src/tiny_llm/agent/evaluation.py create mode 100644 src/tiny_llm_ref/agent/evaluation.py create mode 100644 tests_refsol/test_week_4_day_7.py diff --git a/README.md b/README.md index ed46e4db..a7ce72e1 100644 --- a/README.md +++ b/README.md @@ -37,10 +37,11 @@ The course follows a four-week learning path: the scheduler does not rebuild dense history on every step. - **Week 4: Build a Coding Agent.** Start with a bounded, validated agent loop, then connect it to a small workspace. The course is publishing one reviewed - checkpoint at a time; Days 1 through 6 now cover inspection, approved edits, + checkpoint at a time; Days 1 through 7 now cover inspection, approved edits, one validation command, simple effect receipts, and one visible checkpoint-and-resume boundary, receipt-backed context compaction, and one - visible inspect-and-steer pause. + visible inspect-and-steer pause, and deterministic evaluation of observable + outcomes. ## Why MLX and Qwen3? @@ -73,7 +74,7 @@ implementation, test, and publication readiness is tracked below. ## Roadmap -The table tracks implementation (`Code`), tests (`Test`), rendered chapters (`Doc`), and Chi's review of learner-facing material (`Audit`). Week 4 is publishing one reviewed day at a time; Days 1 through 6 are currently available to learners. The Audit column reflects Chi's personal editorial pass on the published course content and is independent of code/test/doc readiness. +The table tracks implementation (`Code`), tests (`Test`), rendered chapters (`Doc`), and Chi's review of learner-facing material (`Audit`). Week 4 is publishing one reviewed day at a time; Days 1 through 7 are currently available to learners. The Audit column reflects Chi's personal editorial pass on the published course content and is independent of code/test/doc readiness. Day 3 can send file contents to the model, modify files after approval, and run one exact configured command. Use a disposable workspace without secrets and @@ -85,6 +86,8 @@ Day 5 compacts older completed effects in the model-visible transcript while their exact receipts retain the full action, result, and changed artifacts. Day 6 inspects one complete-observation checkpoint, appends one visible operator instruction, and resumes a fresh model without replaying the completed effect. +Day 7 evaluates one completed run from declared final, file, result, and receipt +facts without grading hidden reasoning or exact transcript shape. | Week + Chapter | Topic | Code | Test | Doc | Audit | |---|---|---|---|---|---| @@ -115,7 +118,7 @@ instruction, and resumes a fresh model without replaying the completed effect. | 4.4 | Checkpoint and Resume | ✅ | ✅ | ✅ | 🚧 | | 4.5 | Compact Completed Work | ✅ | ✅ | ✅ | 🚧 | | 4.6 | Inspect and Steer a Paused Agent | ✅ | ✅ | ✅ | 🚧 | -| 4.7 | Later agent checkpoint | — | — | — | 🚧 | +| 4.7 | Evaluate Observable Outcomes | ✅ | ✅ | ✅ | 🚧 | Other topics not covered include quantized or compressed KV caches, cross-request prefix caching, fine-tuning, and long-context techniques. diff --git a/book/src/SUMMARY.md b/book/src/SUMMARY.md index 9cea7c68..944cf3fa 100644 --- a/book/src/SUMMARY.md +++ b/book/src/SUMMARY.md @@ -37,6 +37,7 @@ - [🚧 Day 4: Checkpoint and Resume](./week4-04-sessions.md) - [🚧 Day 5: Compact Completed Work](./week4-05-compaction.md) - [🚧 Day 6: Inspect and Steer a Paused Agent](./week4-06-steering.md) + - [🚧 Day 7: Evaluate Observable Outcomes](./week4-07-evaluation.md) - [🚧 Appendix: Performance Evidence Ledger](./appendix-performance.md) - [Sponsored by Raft.build](./sponsor.md) diff --git a/book/src/week4-06-steering.md b/book/src/week4-06-steering.md index a390b2b7..630b9517 100644 --- a/book/src/week4-06-steering.md +++ b/book/src/week4-06-steering.md @@ -190,7 +190,8 @@ fresh model without replaying the completed effect. Inspect the model inputs in the focused test to verify the original task, saved evidence, and steering stay in order through a later tool turn and final answer. -Additional capabilities remain unpublished until their own complete learner -checkpoints are ready. +Continue with [Day 7: Evaluate Observable Outcomes](week4-07-evaluation.md) to +turn the final workspace, tool results, and durable receipts into a structured +pass/fail report without grading hidden reasoning or exact transcript shape. {{#include copyright.md}} diff --git a/book/src/week4-07-evaluation.md b/book/src/week4-07-evaluation.md index 6fe619d4..11fa407a 100644 --- a/book/src/week4-07-evaluation.md +++ b/book/src/week4-07-evaluation.md @@ -1,275 +1,170 @@ -# Day 7: Safe Deterministic Evaluation +# Day 7: Evaluate Observable Outcomes > 🚧 **Early-review WIP:** This chapter is public for early review and may > change. Use a disposable workspace when running the agent or enabling writes > or commands. -An agent saying that it finished is not evidence that its change is correct. -Day 7 adds a small, deterministic evaluation harness that grades the resulting -files independently of the model's final message. The checkpoint is deliberately -static: it can inspect text, JSON, and Python syntax, but it never imports or -executes candidate code. +The first six days built a small coding-agent loop, connected it to a workspace, +recorded approved effects, and added checkpoint, compaction, and steering +boundaries. The closing question is practical: did one run produce the outcome +the task asked for? -> **Implementation status:** The reference implementation, learner API surface, -> inert task packages, and focused tests implement the static evaluation boundary -> described here. The chapter remains WIP even though the checkpoint is -> executable. Executable Python tests and general coding-task grading remain -> deferred until the candidate can run inside a container or virtual machine. +Day 7 answers with a small deterministic evaluation harness. It checks declared +observable facts: the final answer, exact file contents, tool-result evidence, +and named durable receipts. It does not grade hidden reasoning or require one +exact transcript shape. -## Check the Chapter +## The Starter Surface -Implement the evaluation APIs under `src/tiny_llm/agent/`, then run: +Day 7 adds one module: + +| File | Public names | Purpose | +| --- | --- | --- | +| `src/tiny_llm/agent/evaluation.py` | `FileExpectation`, `ResultExpectation`, `ReceiptExpectation`, `EvaluationCase`, `EvaluationCheck`, `EvaluationReport`, `evaluate_run` | Describe required observable facts and produce a stable pass/fail report. | +| `src/tiny_llm/agent/__init__.py` | the names above | Export the cumulative Day 7 API. | + +Copy and run the seven learner tasks: ```bash +pdm run copy-test --week 4 --day 7 pdm run test --week 4 --day 7 ``` -Use `pdm run test-refsol --week 4 --day 7` for the supplied implementation. The -focused suite uses scripted model actions and inert fixtures. Tripwires verify -that grading does not launch a process, import a candidate module, or call -`eval()` or `exec()`. - -You can inspect and grade an unchanged task fixture without loading a model: +Use this command for the supplied implementation: ```bash -pdm run evaluate-agent inspect evals/week4/localized-constant -pdm run evaluate-agent grade evals/week4/localized-constant +pdm run test-refsol --week 4 --day 7 ``` -The second command stages the fixture into a fresh temporary directory, freezes -that candidate, and runs only the declarative held-out checks. It does not run -the coding agent or approve a mutation. Use the Python API to evaluate a scripted -or model-backed trajectory. `inspect` exits zero for a valid package; `grade` -exits zero for a pass, one for a deterministic failure, and two for an invalid -package or CLI error. The shipped unchanged localized-constant baseline is -intentionally incorrect, so grading it returns a normal failed report. +Before you implement the TODOs, all seven Day 7 tasks are expected to fail. + +## Task 1: Declare the Outcome + +An evaluation case names only the facts that matter for this task: + +```python +case = EvaluationCase( + final_contains="validated", + files=(FileExpectation("app.py", "answer = 2\n"),), + results=( + ResultExpectation("run_command", "validation passed"), + ), + receipts=( + ReceiptExpectation( + "call-1", + "edit_file", + "ok", + "edited app.py", + ("app.py",), + ), + ReceiptExpectation( + "call-2", + "run_command", + "ok", + "validation passed", + ), + ), +) +``` -## A Sealed, Inert Task Package +The two receipt IDs are explicit inputs chosen for this deterministic case. A +different harness could discover or correlate effect records another way; Day +7 does not claim that every evaluation needs fixed call IDs. -Each task is a directory with one public manifest, a starting workspace, and a -held-out declarative check file: +Reject an invalid specification before evaluating: required strings cannot be +blank, file paths must be relative and remain inside the workspace, file paths +and receipt IDs must be unique within their groups, and a receipt exit state is +either `ok` or `error`. -```text -localized-constant/ - task.json - workspace/ - answer.py - held_out_tests/ - checks.json -``` +## Task 2: Check the Final Answer -`task.json` is strict and versioned: +Implement: -```json -{ - "schema_version": 1, - "id": "localized-constant", - "prompt": "Correct the answer without changing the public name.", - "max_steps": 6, - "editable_paths": ["answer.py"] -} +```python +evaluate_run(run, workspace, receipts, case) -> EvaluationReport ``` -The loader rejects missing and unknown fields, unsupported schema versions, -blank identifiers or prompts, invalid budgets, absolute paths, parent traversal, -duplicate editable paths, and overlapping editable path prefixes. It also -rejects symlinks, `.git`, excessive file counts or bytes, unknown package -entries, and executable held-out files. A package cannot contain `grade.py`, a -command, or another task-local program. - -`TaskPackage.stage()` copies only `workspace/` into a new narrow destination. -The destination must not already contain a candidate tree. Held-out checks, -solution history, and package metadata never enter the agent-visible workspace. -Staging records a sorted snapshot of every initial regular file and a canonical -tree SHA-256 that also covers directory presence, without modifying the original -package. - -These rules make the included fixtures inert and reproducible. They are not a -general way to make an untrusted repository safe. Parsing a package still reads -bytes supplied by its author, and the later agent run can only be as isolated as -its enabled tools. - -## Keep Commands Disabled and Writes Default-No - -`evaluate_task()` creates a `Workspace` whose command allowlist is empty. An -agent action requesting `run_command` is rejected before process launch. The -evaluation harness does not translate manifest data into an argv and does not -offer a task-local validation command. - -File writes remain subject to the normal workspace checks and a per-call human -confirmation callback. If no callback is supplied, the write is denied. The -evaluator never installs an always-yes callback on the caller's behalf. A CLI or -application that wants model-authored mutations must present the exact tool call -and implement the same explicit `y/N` policy as the main agent CLI: blank input, -EOF, non-interactive input, interruption, and every response except `y` or `yes` -mean No. - -This gives the evaluator a small capability boundary: +The first check requires a completed run whose public final answer contains the +declared substring. This is a small grounding signal, not a prose grader. Do +not inspect hidden reasoning, demand exact wording, or ask another model to +judge the answer. + +## Task 3: Check Workspace State + +For each `FileExpectation`, resolve the declared path through the existing +`Workspace` boundary, read it as UTF-8, and compare the exact content. Emit one +named check such as `file:app.py`. + +A missing file, directory, unreadable file, or content mismatch is observed +evidence that failed. Return a failed check instead of aborting the whole +report. That is different from an invalid case definition, which is rejected. + +## Task 4: Match Result Evidence Without Grading a Trace + +Each `ResultExpectation` requires at least one public `AgentEvent` with the +declared tool and result substring. Search the events as a set of observable +facts. Do not require an exact number of turns or an exact event order. + +This matters because two useful runs may phrase their final answers differently +or place unrelated read-only observations in a different order while producing +the same required outcome. + +## Task 5: Check Named Durable Receipts + +Use the public `ReceiptStore` passed to `evaluate_run`; do not reach through +private workspace state. For every declared call ID, require the expected tool, +exit state, result substring, and exact changed-artifact tuple. + +Absent receipts, tampered persistent logs, mismatched fields, and lookup errors +become failed checks. Evaluation must not append a receipt or rerun an effect. + +## Task 6: Produce a Stable Report + +Return checks in one deterministic order: + +1. final answer; +2. files in case order; +3. results in case order; and +4. receipts in case order. + +`EvaluationReport.passed` is true only when every check passes. Its `render()` +method should produce a compact summary: ```text -public manifest + staged workspace - | - v - bounded agent loop - writes: explicit y/N - commands: disabled - | - v - frozen candidate tree - | - v - static held-out grader +evaluation: PASS +- final: PASS (required final observed) +- file:app.py: PASS (content matches) +- result:run_command: PASS (required result observed) +- receipt:call-1: PASS (receipt facts match) +- receipt:call-2: PASS (receipt facts match) ``` -It does not make host execution safe. Do not enable `run_command` to make a task -more realistic: an approved Python program or test can delete files outside the -workspace, read credentials, access the network, or create detached processes. - -## Freeze Before Revealing Held-Out Checks - -The evaluation lifecycle is ordered so hidden expectations cannot influence the -agent trajectory: - -1. Validate the package and stage only its public workspace. -2. Record the initial file snapshots and tree hash. -3. Run the bounded agent with commands disabled and writes defaulting to No. -4. Stop after a valid final action, budget termination, interruption, or another - recorded terminal reason. -5. Copy the candidate into a separate, read-only-in-practice grading snapshot. -6. Record `evaluation_started` with the task ID and candidate tree hash. -7. Load `held_out_tests/checks.json` for the first time. -8. Run the allowlisted static checks over the frozen snapshot. -9. Record `evaluation_result` with the grade, ordered check outcomes, forbidden - modifications, and metrics. - -The grader never observes a live tree that the agent can continue changing. -This checkpoint treats one staged task and its session as single-use. A crash -after `evaluation_started` but before `evaluation_result` leaves an incomplete -record; restage and rerun the task instead of silently grading different bytes. -Hidden expected values are not included in model messages or pre-grade session -events. - -Freezing closes an evaluator consistency gap, not every host race. A separate -same-user process can still interfere with ordinary files while they are copied. -Use a container or virtual machine when task inputs or neighboring processes are -untrusted. - -## Declarative Held-Out Checks - -`StaticHeldOutGrader` accepts a versioned JSON document containing only a small -allowlist of checks: - -- `path_exists` requires a regular file at a relative path; -- `path_absent` requires that a relative path not exist; -- `text_equals` compares bounded UTF-8 content; -- `json_value` parses JSON and compares a value at a declared path; -- `python_constant` parses Python with `ast.parse()` and inspects one literal - assignment; and -- `unchanged` compares a path with its initial snapshot. - -Every check has strict fields and bounded inputs. Unsupported or malformed -checks produce an evaluation error, never a false pass. Results and paths are -sorted so the same manifest, check specification, and candidate bytes produce -the same `GradeReport`. - -The Python check is syntax inspection, not program execution. It does not import -the candidate, add the staged directory to `sys.path`, evaluate an expression, -execute a module, or start a child process. A file may contain dangerous-looking -top-level calls; `ast.parse()` represents them as nodes and does not run them. -The check accepts only the supported literal assignment shape and rejects other -expressions. - -After the declared checks, the grader compares the frozen tree with the initial -snapshot. Creating, changing, or deleting a path outside `editable_paths` is a -forbidden modification and fails the grade even when every held-out check passes. -The candidate tree hash and ordered forbidden-path list make that decision -auditable. - -Static checks intentionally cover less behavior than executable tests. They are -well suited to localized constants, exact text, repository configuration, file -presence, and small syntax-shape exercises. They cannot prove that arbitrary -Python code behaves correctly. - -## Completion Is Not Correctness - -The loop and grader answer different questions: - -| Signal | Meaning | -| --- | --- | -| `AgentRun.completed` | the model returned a valid final action | -| `AgentRun.reason` | why the bounded agent loop stopped | -| `GradeReport.status` | whether deterministic checks passed, failed, or errored | -| `EvaluatedRun.task_success` | whether `GradeReport.status == "passed"` | - -A fluent final message paired with wrong bytes is completed but failed. A run -that exhausts its step budget can still pass when its final frozen bytes satisfy -every check. A grader configuration error is distinct from an ordinary failed -check. Model prose, timestamps, latency, and formatting never decide the grade. - -Keep these fields separate in reports and dashboards. Conflating protocol -completion with task success rewards agents for stopping confidently rather -than for producing correct files. - -## Metrics From Durable Events - -`aggregate_metrics()` derives evaluation metrics from the retained session -events instead of mutable counters in the runner. It reports: - -- model turns, tool calls, malformed actions, and tool errors; -- input and output tokens when the generation backend reports them; -- reused, rewound, and newly prefetched tokens; -- visible tool-result bytes and compaction count; -- generation latency when available, wall-clock evaluation time, and terminal - reason. - -A scripted string-only generator has no token or cache measurements, so those -fields remain unknown rather than becoming misleading zeroes. Grade ordering and -status do not depend on performance metrics. - -The complete session trace is retained to explain a score and compare -trajectories. It can contain prompts, model output, source excerpts, tool -results, diffs, before-images, and local paths. Treat the session and evaluation -records as sensitive data: do not publish them merely because the aggregate -metrics are safe to share. - -## Exercises - -1. Inspect an inert package and confirm that held-out expected values are not - printed. -2. Grade its unchanged baseline twice and compare the ordered report and tree - hash. -3. Run a scripted trajectory that makes the expected edit and compare - `AgentRun.completed` with `EvaluatedRun.task_success`. -4. Return a valid final action without editing the file and observe a completed, - failed evaluation. -5. Try to modify a path outside `editable_paths` and inspect the forbidden-path - result. -6. Put a dangerous-looking call at Python module scope and verify that - `python_constant` parses but never executes it. -7. Request `run_command` from the scripted agent and verify that no process is - launched. -8. Deny a proposed edit with blank input and verify that grading uses the - unchanged frozen candidate. - -## What Remains Deferred - -This checkpoint does **not** execute model-authored Python, pytest, compilers, -build systems, task-local graders, or manifest-provided commands. A temporary -directory and `cwd` are path-selection conveniences, not confinement. Running -those programs on the host would give candidate code the evaluator's filesystem, -process, credential, and network authority. - -General behavioral coding evaluation therefore requires a later backend with a -real isolation boundary, such as a disposable container or virtual machine with -a read-only base image, an explicit writable mount, network disabled by default, -resource limits, process cleanup, and trusted tests supplied from outside the -candidate workspace. Only that backend should install and run held-out pytest or -attempt the broader parser capstone. - -Other useful extensions include repeated sampled runs, model comparisons, -statistical confidence intervals, leak-resistant external task distribution, -and durable restart of an in-progress grading job. None of them changes the Day -7 rule: task correctness comes from an independent, deterministic grader, and -executable candidate code must not run without actual isolation. +Stable names and ordering make failures easy to inspect without turning the +test into an exact transcript comparison. + +## Task 7: Keep Evaluation Read-Only + +The focused scenario asks the existing agent loop to set `answer = 2` in +`app.py`, run the exact configured validation command, and finish. The harness +then checks the final answer, final file bytes, validation result, edit receipt, +and command receipt. + +Calling `evaluate_run` must leave the run, workspace bytes, modified-file list, +approval history, and receipt bytes unchanged. It invokes no model, tool, or +approval callback. Independent wrong final, file, result, and receipt facts +each fail their own named check. An alternate final phrase and event order still +pass when the required behavioral evidence is present. + +## Checkpoint + +You can now turn one coding-agent run into a deterministic report over declared +observable outcomes. This harness samples the facts a particular case names. It +does not prove general task correctness, model quality, security, or production +safety, and it is not a hidden grader, benchmark suite, or LLM-as-judge system. + +That closes the seven-day Week 4 path: build a bounded loop, inspect and change +a workspace with approvals and receipts, pause and resume, compact completed +evidence, steer at a safe boundary, and finally evaluate one run by what it +actually left behind. {{#include copyright.md}} diff --git a/book/src/week4-overview.md b/book/src/week4-overview.md index c4db4b38..a7d3d04f 100644 --- a/book/src/week4-overview.md +++ b/book/src/week4-overview.md @@ -1,7 +1,7 @@ # 🚧 Week 4: Build a Coding Agent > **Course status:** Week 4 is being published one checkpoint at a time. Days 1 -> through 6 are ready to learn and review. Additional capabilities will appear +> through 7 are ready to learn and review. Additional capabilities will appear > only after their implementation, starter, and reviews are ready. Weeks 1 through 3 turn tokens into text and make serving that text efficient. @@ -15,7 +15,9 @@ metadata, then restores both into a fresh model without replaying effects. Day 5 derives a smaller model-visible transcript from older completed effects while their exact receipts remain unchanged. Day 6 inspects one safe checkpoint, adds one visible operator steering message, and resumes a fresh model without -replaying the completed effect. +replaying the completed effect. Day 7 evaluates one completed run from declared +final, file, result, and receipt facts without grading hidden reasoning or exact +transcript shape. ## What Day 1 Builds @@ -106,8 +108,15 @@ Agent](week4-06-steering.md). Its cumulative command is: pdm run test --week 4 --day 6 ``` -Only the Day 1 through Day 6 starter modules are published. Do not add session -trees, rewind, reconciliation, evaluation, or other later public APIs to your +After Day 6 passes, continue with [Day 7: Evaluate Observable +Outcomes](week4-07-evaluation.md). Its cumulative command is: + +```bash +pdm run test --week 4 --day 7 +``` + +Only the Day 1 through Day 7 starter modules are published. Do not add session +trees, rewind, reconciliation, an LLM judge, or other later public APIs to your solution. {{#include copyright.md}} diff --git a/src/tiny_llm/agent/__init__.py b/src/tiny_llm/agent/__init__.py index 059c7728..24768f52 100644 --- a/src/tiny_llm/agent/__init__.py +++ b/src/tiny_llm/agent/__init__.py @@ -2,6 +2,15 @@ from .checkpoint import AgentCheckpoint, ModelCheckpoint, create_checkpoint from .compaction import CompactionResult, compact_completed_interactions +from .evaluation import ( + EvaluationCase, + EvaluationCheck, + EvaluationReport, + FileExpectation, + ReceiptExpectation, + ResultExpectation, + evaluate_run, +) from .generation import generate_response, initial_messages from .loop import ( AgentEvent, @@ -31,16 +40,23 @@ "AgentRun", "AgentStatus", "CompactionResult", + "EvaluationCase", + "EvaluationCheck", + "EvaluationReport", "EffectReceipt", "FinalAction", + "FileExpectation", "ModelCheckpoint", "ReceiptStore", + "ReceiptExpectation", + "ResultExpectation", "ToolAction", "ToolPolicy", "Workspace", "build_system_prompt", "compact_completed_interactions", "create_checkpoint", + "evaluate_run", "generate_response", "initial_messages", "inspect_checkpoint", diff --git a/src/tiny_llm/agent/evaluation.py b/src/tiny_llm/agent/evaluation.py new file mode 100644 index 00000000..876518a4 --- /dev/null +++ b/src/tiny_llm/agent/evaluation.py @@ -0,0 +1,94 @@ +# WARNING: Under review - generated by LLM. + +"""Week 4, Day 7: evaluate one run from declared observable facts.""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from .loop import AgentRun +from .receipts import ReceiptStore +from .workspace import Workspace + + +@dataclass(frozen=True) +class FileExpectation: + """One exact UTF-8 file fact expected after the run.""" + + path: str + content: str + + def __post_init__(self) -> None: + pass + + +@dataclass(frozen=True) +class ResultExpectation: + """One required substring in an observable tool result.""" + + tool: str + contains: str + + def __post_init__(self) -> None: + pass + + +@dataclass(frozen=True) +class ReceiptExpectation: + """Declared facts for one named durable effect receipt.""" + + tool_call_id: str + tool: str + exit_state: str + result_contains: str + changed_artifacts: tuple[str, ...] = field(default_factory=tuple) + + def __post_init__(self) -> None: + pass + + +@dataclass(frozen=True) +class EvaluationCase: + """The observable facts one deterministic course case requires.""" + + final_contains: str + files: tuple[FileExpectation, ...] = field(default_factory=tuple) + results: tuple[ResultExpectation, ...] = field(default_factory=tuple) + receipts: tuple[ReceiptExpectation, ...] = field(default_factory=tuple) + + def __post_init__(self) -> None: + pass + + +@dataclass(frozen=True) +class EvaluationCheck: + """One named PASS/FAIL result backed by an observable fact.""" + + name: str + passed: bool + detail: str + + +@dataclass(frozen=True) +class EvaluationReport: + """Stable ordered checks for one evaluation case.""" + + checks: tuple[EvaluationCheck, ...] + + @property + def passed(self) -> bool: + pass + + def render(self) -> str: + pass + + +def evaluate_run( + run: AgentRun, + workspace: Workspace, + receipts: ReceiptStore, + case: EvaluationCase, +) -> EvaluationReport: + """Judge one run only from the observable facts declared by ``case``.""" + + pass diff --git a/src/tiny_llm_ref/agent/__init__.py b/src/tiny_llm_ref/agent/__init__.py index 059c7728..24768f52 100644 --- a/src/tiny_llm_ref/agent/__init__.py +++ b/src/tiny_llm_ref/agent/__init__.py @@ -2,6 +2,15 @@ from .checkpoint import AgentCheckpoint, ModelCheckpoint, create_checkpoint from .compaction import CompactionResult, compact_completed_interactions +from .evaluation import ( + EvaluationCase, + EvaluationCheck, + EvaluationReport, + FileExpectation, + ReceiptExpectation, + ResultExpectation, + evaluate_run, +) from .generation import generate_response, initial_messages from .loop import ( AgentEvent, @@ -31,16 +40,23 @@ "AgentRun", "AgentStatus", "CompactionResult", + "EvaluationCase", + "EvaluationCheck", + "EvaluationReport", "EffectReceipt", "FinalAction", + "FileExpectation", "ModelCheckpoint", "ReceiptStore", + "ReceiptExpectation", + "ResultExpectation", "ToolAction", "ToolPolicy", "Workspace", "build_system_prompt", "compact_completed_interactions", "create_checkpoint", + "evaluate_run", "generate_response", "initial_messages", "inspect_checkpoint", diff --git a/src/tiny_llm_ref/agent/evaluation.py b/src/tiny_llm_ref/agent/evaluation.py new file mode 100644 index 00000000..1be628d1 --- /dev/null +++ b/src/tiny_llm_ref/agent/evaluation.py @@ -0,0 +1,239 @@ +# WARNING: Under review - generated by LLM. + +"""Week 4, Day 7: evaluate one run from declared observable facts.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path + +from .loop import AgentRun +from .protocol import AgentError, ToolAction +from .receipts import ReceiptStore +from .workspace import Workspace + + +def _expect_text(value: object, name: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{name} must not be blank") + return value + + +def _expect_path(value: object, name: str) -> str: + path = Path(_expect_text(value, name)) + if ( + path.is_absolute() + or path == Path(".") + or ".." in path.parts + or "\0" in str(path) + ): + raise ValueError(f"{name} must be a relative file path") + return str(path) + + +@dataclass(frozen=True) +class FileExpectation: + """One exact UTF-8 file fact expected after the run.""" + + path: str + content: str + + def __post_init__(self) -> None: + object.__setattr__(self, "path", _expect_path(self.path, "file path")) + if not isinstance(self.content, str): + raise ValueError("file content must be a string") + + +@dataclass(frozen=True) +class ResultExpectation: + """One required substring in an observable tool result.""" + + tool: str + contains: str + + def __post_init__(self) -> None: + _expect_text(self.tool, "result tool") + _expect_text(self.contains, "result substring") + + +@dataclass(frozen=True) +class ReceiptExpectation: + """Declared facts for one named durable effect receipt.""" + + tool_call_id: str + tool: str + exit_state: str + result_contains: str + changed_artifacts: tuple[str, ...] = field(default_factory=tuple) + + def __post_init__(self) -> None: + _expect_text(self.tool_call_id, "receipt call id") + _expect_text(self.tool, "receipt tool") + if self.exit_state not in {"ok", "error"}: + raise ValueError("receipt exit_state must be 'ok' or 'error'") + _expect_text(self.result_contains, "receipt result substring") + if not isinstance(self.changed_artifacts, tuple): + raise ValueError("receipt changed_artifacts must be a tuple") + object.__setattr__( + self, + "changed_artifacts", + tuple( + _expect_path(artifact, "changed artifact") + for artifact in self.changed_artifacts + ), + ) + + +@dataclass(frozen=True) +class EvaluationCase: + """The observable facts one deterministic course case requires.""" + + final_contains: str + files: tuple[FileExpectation, ...] = field(default_factory=tuple) + results: tuple[ResultExpectation, ...] = field(default_factory=tuple) + receipts: tuple[ReceiptExpectation, ...] = field(default_factory=tuple) + + def __post_init__(self) -> None: + self._validate() + + def _validate(self) -> None: + _expect_text(self.final_contains, "final substring") + groups = ( + (self.files, FileExpectation, "files"), + (self.results, ResultExpectation, "results"), + (self.receipts, ReceiptExpectation, "receipts"), + ) + for values, expected_type, name in groups: + if not isinstance(values, tuple) or any( + not isinstance(value, expected_type) for value in values + ): + raise ValueError(f"evaluation {name} must be a tuple of expectations") + file_paths = [expected.path for expected in self.files] + if len(file_paths) != len(set(file_paths)): + raise ValueError("evaluation file paths must be unique") + call_ids = [expected.tool_call_id for expected in self.receipts] + if len(call_ids) != len(set(call_ids)): + raise ValueError("evaluation receipt call ids must be unique") + + +@dataclass(frozen=True) +class EvaluationCheck: + """One named PASS/FAIL result backed by an observable fact.""" + + name: str + passed: bool + detail: str + + +@dataclass(frozen=True) +class EvaluationReport: + """Stable ordered checks for one evaluation case.""" + + checks: tuple[EvaluationCheck, ...] + + @property + def passed(self) -> bool: + return bool(self.checks) and all(check.passed for check in self.checks) + + def render(self) -> str: + lines = [f"evaluation: {'PASS' if self.passed else 'FAIL'}"] + for check in self.checks: + status = "PASS" if check.passed else "FAIL" + lines.append(f"- {check.name}: {status} ({check.detail})") + return "\n".join(lines) + + +def _file_check(workspace: Workspace, expected: FileExpectation) -> EvaluationCheck: + name = f"file:{expected.path}" + try: + observed = workspace.resolve_path(expected.path).read_text(encoding="utf-8") + except (AgentError, OSError, UnicodeError) as error: + return EvaluationCheck(name, False, f"evidence unavailable: {error}") + if observed == expected.content: + return EvaluationCheck(name, True, "content matches") + return EvaluationCheck(name, False, "content does not match") + + +def _result_check(run: AgentRun, expected: ResultExpectation) -> EvaluationCheck: + name = f"result:{expected.tool}" + matched = any( + isinstance(event.action, ToolAction) + and event.action.tool == expected.tool + and isinstance(event.result, str) + and expected.contains in event.result + for event in run.events + ) + detail = "required result observed" if matched else "required result not observed" + return EvaluationCheck(name, matched, detail) + + +def _receipt_reader(receipts: ReceiptStore) -> tuple[ReceiptStore | None, str | None]: + if receipts.path is None: + return receipts, None + try: + return ReceiptStore(receipts.path), None + except (AgentError, OSError, UnicodeError) as error: + return None, f"receipt evidence unavailable: {error}" + + +def _receipt_check( + receipts: ReceiptStore | None, + load_error: str | None, + expected: ReceiptExpectation, +) -> EvaluationCheck: + name = f"receipt:{expected.tool_call_id}" + if load_error is not None: + return EvaluationCheck(name, False, load_error) + assert receipts is not None + try: + receipt = receipts.get(expected.tool_call_id) + except (AgentError, OSError) as error: + return EvaluationCheck(name, False, f"receipt lookup failed: {error}") + if receipt is None: + return EvaluationCheck(name, False, "receipt is absent") + matched = ( + receipt.tool == expected.tool + and receipt.exit_state == expected.exit_state + and expected.result_contains in receipt.result + and receipt.changed_artifacts == expected.changed_artifacts + ) + detail = "receipt facts match" if matched else "receipt facts do not match" + return EvaluationCheck(name, matched, detail) + + +def evaluate_run( + run: AgentRun, + workspace: Workspace, + receipts: ReceiptStore, + case: EvaluationCase, +) -> EvaluationReport: + """Judge one run only from the observable facts declared by ``case``.""" + + if not isinstance(run, AgentRun): + raise ValueError("run must be an AgentRun") + if not isinstance(workspace, Workspace): + raise ValueError("workspace must be a Workspace") + if not isinstance(receipts, ReceiptStore): + raise ValueError("receipts must be a ReceiptStore") + if not isinstance(case, EvaluationCase): + raise ValueError("case must be an EvaluationCase") + case._validate() + + final_ok = ( + run.completed and run.final is not None and case.final_contains in run.final + ) + checks = [ + EvaluationCheck( + "final", + final_ok, + "required final observed" if final_ok else "required final not observed", + ) + ] + checks.extend(_file_check(workspace, expected) for expected in case.files) + checks.extend(_result_check(run, expected) for expected in case.results) + receipt_reader, load_error = _receipt_reader(receipts) + checks.extend( + _receipt_check(receipt_reader, load_error, expected) + for expected in case.receipts + ) + return EvaluationReport(tuple(checks)) diff --git a/tests_refsol/test_week_4_day_7.py b/tests_refsol/test_week_4_day_7.py new file mode 100644 index 00000000..362e8dab --- /dev/null +++ b/tests_refsol/test_week_4_day_7.py @@ -0,0 +1,270 @@ +# WARNING: Under review - generated by LLM. + +"""Week 4 Day 7 observable-outcome evaluation course-code tests.""" + +import json +import sys +from dataclasses import replace + +import pytest + +from .tiny_llm_base import ( + AgentError, + EvaluationCase, + FileExpectation, + ReceiptExpectation, + ReceiptStore, + ResultExpectation, + ToolPolicy, + Workspace, + evaluate_run, + run_agent, +) + + +class FakeEvaluationModel: + """Return one deterministic response per model call.""" + + def __init__(self, responses): + self.responses = iter(responses) + + def __call__(self, _messages): + return next(self.responses) + + +def _completed_case(tmp_path): + source = tmp_path / "app.py" + source.write_text("answer = 1\n", encoding="utf-8") + command = ( + sys.executable, + "-c", + ( + "from pathlib import Path; " + "assert Path('app.py').read_text() == 'answer = 2\\n'; " + "print('validation passed')" + ), + ) + approvals = [] + receipts = ReceiptStore(tmp_path / "receipts.jsonl") + workspace = Workspace( + ToolPolicy(tmp_path, allow_writes=True, allowed_commands=(command,)), + lambda action: approvals.append(action.tool) or True, + receipts, + ) + responses = ( + '{"tool":"read_file","path":"app.py"}', + '{"tool":"edit_file","path":"app.py","old":"1","new":"2"}', + json.dumps({"tool": "run_command", "argv": list(command)}), + '{"final":"updated app.py and validated the result"}', + ) + run = run_agent( + "set answer = 2 in app.py and validate", + FakeEvaluationModel(responses), + workspace, + ) + case = EvaluationCase( + final_contains="validated", + files=(FileExpectation("app.py", "answer = 2\n"),), + results=( + ResultExpectation("edit_file", "edited app.py"), + ResultExpectation("run_command", "validation passed"), + ), + receipts=( + ReceiptExpectation( + "call-1", "edit_file", "ok", "edited app.py", ("app.py",) + ), + ReceiptExpectation("call-2", "run_command", "ok", "validation passed"), + ), + ) + return run, workspace, receipts, case, approvals + + +def test_task_1_end_to_end_run_passes_declared_observable_facts(tmp_path): + run, workspace, receipts, case, approvals = _completed_case(tmp_path) + + report = evaluate_run(run, workspace, receipts, case) + + assert report.passed + assert [check.name for check in report.checks] == [ + "final", + "file:app.py", + "result:edit_file", + "result:run_command", + "receipt:call-1", + "receipt:call-2", + ] + assert approvals == ["edit_file", "run_command"] + assert report.render().splitlines() == [ + "evaluation: PASS", + "- final: PASS (required final observed)", + "- file:app.py: PASS (content matches)", + "- result:edit_file: PASS (required result observed)", + "- result:run_command: PASS (required result observed)", + "- receipt:call-1: PASS (receipt facts match)", + "- receipt:call-2: PASS (receipt facts match)", + ] + + +@pytest.mark.parametrize( + ("fact", "failed_name"), + [ + ("final", "final"), + ("file", "file:app.py"), + ("result", "result:run_command"), + ("receipt", "receipt:call-1"), + ], +) +def test_task_2_each_wrong_fact_fails_only_its_named_check(tmp_path, fact, failed_name): + run, workspace, receipts, case, _approvals = _completed_case(tmp_path) + if fact == "final": + case = replace(case, final_contains="not in the final") + elif fact == "file": + case = replace(case, files=(FileExpectation("app.py", "wrong\n"),)) + elif fact == "result": + case = replace( + case, + results=( + case.results[0], + ResultExpectation("run_command", "status: 9"), + ), + ) + else: + case = replace( + case, + receipts=(replace(case.receipts[0], tool="write_file"), case.receipts[1]), + ) + + report = evaluate_run(run, workspace, receipts, case) + + assert not report.passed + assert [check.name for check in report.checks if not check.passed] == [failed_name] + + +def test_task_3_nonessential_final_phrasing_and_event_order_are_not_graded(tmp_path): + run, workspace, receipts, case, _approvals = _completed_case(tmp_path) + alternate = replace( + run, + final="The requested outcome is validated with the recorded evidence.", + events=tuple(reversed(run.events)), + ) + + report = evaluate_run(alternate, workspace, receipts, case) + + assert report.passed + + +def test_task_4_missing_unreadable_and_absent_evidence_become_failed_checks(tmp_path): + run, workspace, _receipts, case, _approvals = _completed_case(tmp_path) + (tmp_path / "folder").mkdir() + unavailable = replace( + case, + files=( + FileExpectation("missing.txt", "missing"), + FileExpectation("folder", "not a file"), + ), + ) + + report = evaluate_run(run, workspace, ReceiptStore(), unavailable) + + assert [check.name for check in report.checks if not check.passed] == [ + "file:missing.txt", + "file:folder", + "receipt:call-1", + "receipt:call-2", + ] + assert report.render().startswith("evaluation: FAIL\n") + + +@pytest.mark.parametrize("tamper", ["digest", "utf8"]) +def test_task_5_tampered_receipt_log_becomes_failed_checks(tmp_path, tamper): + run, workspace, receipts, case, _approvals = _completed_case(tmp_path) + assert receipts.path is not None + if tamper == "digest": + original = receipts.path.read_text(encoding="utf-8") + receipts.path.write_text( + original.replace('"tool": "edit_file"', '"tool": "write_file"', 1), + encoding="utf-8", + ) + else: + receipts.path.write_bytes(b"\xff") + + report = evaluate_run(run, workspace, receipts, case) + + assert [check.name for check in report.checks if not check.passed] == [ + "receipt:call-1", + "receipt:call-2", + ] + assert all( + "receipt evidence unavailable" in check.detail for check in report.checks[-2:] + ) + + +def test_task_6_evaluation_is_pure(tmp_path): + run, workspace, receipts, case, approvals = _completed_case(tmp_path) + assert receipts.path is not None + before = ( + (tmp_path / "app.py").read_bytes(), + receipts.path.read_bytes(), + workspace.modified_files, + tuple(approvals), + run, + ) + + evaluate_run(run, workspace, receipts, case) + + after = ( + (tmp_path / "app.py").read_bytes(), + receipts.path.read_bytes(), + workspace.modified_files, + tuple(approvals), + run, + ) + assert after == before + + +def test_task_7_invalid_evaluation_specifications_fail_closed(tmp_path): + with pytest.raises(ValueError, match="file path"): + FileExpectation("../escape", "data") + with pytest.raises(ValueError, match="result substring"): + ResultExpectation("read_file", " ") + with pytest.raises(ValueError, match="receipt call id"): + ReceiptExpectation("", "edit_file", "ok", "edited") + with pytest.raises(ValueError, match="exit_state"): + ReceiptExpectation("call-1", "edit_file", "maybe", "edited") + with pytest.raises(ValueError, match="file paths must be unique"): + EvaluationCase( + "done", + files=(FileExpectation("a/b", "1"), FileExpectation("a//b", "2")), + ) + with pytest.raises(ValueError, match="receipt call ids must be unique"): + EvaluationCase( + "done", + receipts=( + ReceiptExpectation("call-1", "edit_file", "ok", "edited"), + ReceiptExpectation("call-1", "run_command", "ok", "status"), + ), + ) + + run, workspace, receipts, _case, _approvals = _completed_case(tmp_path) + with pytest.raises(ValueError, match="EvaluationCase"): + evaluate_run(run, workspace, receipts, object()) + with pytest.raises(ValueError, match="AgentRun"): + evaluate_run(object(), workspace, receipts, EvaluationCase("done")) + + +def test_task_7_receipt_lookup_errors_become_failed_checks(tmp_path, monkeypatch): + run, workspace, receipts, case, _approvals = _completed_case(tmp_path) + in_memory = ReceiptStore() + monkeypatch.setattr( + in_memory, + "get", + lambda _call_id: (_ for _ in ()).throw(AgentError("lookup broke")), + ) + + report = evaluate_run(run, workspace, in_memory, case) + + assert [check.name for check in report.checks if not check.passed] == [ + "receipt:call-1", + "receipt:call-2", + ] + assert all("receipt lookup failed" in check.detail for check in report.checks[-2:]) diff --git a/tests_refsol/test_week_4_starter_sync.py b/tests_refsol/test_week_4_starter_sync.py index 3754309e..c7ce0ad3 100644 --- a/tests_refsol/test_week_4_starter_sync.py +++ b/tests_refsol/test_week_4_starter_sync.py @@ -1,6 +1,6 @@ # WARNING: Under review - generated by LLM. -"""Course-code guards for the cumulative Week 4 Day 1--5 starter.""" +"""Course-code guards for the cumulative Week 4 Day 1--7 starter.""" import ast import importlib.util @@ -21,6 +21,7 @@ MODULES = ( "checkpoint", "compaction", + "evaluation", "generation", "loop", "protocol", @@ -125,7 +126,7 @@ def test_starter_is_solution_free_and_reference_is_implemented(module): ) -def test_package_exports_match_the_published_day_6_surface(): +def test_package_exports_match_the_published_day_7_surface(): import tiny_llm.agent as starter import tiny_llm_ref.agent as refsol @@ -137,16 +138,23 @@ def test_package_exports_match_the_published_day_6_surface(): "AgentRun", "AgentStatus", "CompactionResult", + "EvaluationCase", + "EvaluationCheck", + "EvaluationReport", "EffectReceipt", "FinalAction", + "FileExpectation", "ModelCheckpoint", "ReceiptStore", + "ReceiptExpectation", + "ResultExpectation", "ToolAction", "ToolPolicy", "Workspace", "build_system_prompt", "compact_completed_interactions", "create_checkpoint", + "evaluate_run", "generate_response", "initial_messages", "inspect_checkpoint", @@ -159,11 +167,12 @@ def test_package_exports_match_the_published_day_6_surface(): assert set(starter.__all__) == set(refsol.__all__) == expected -def test_only_day_1_through_day_6_modules_exist_in_the_starter(): +def test_only_day_1_through_day_7_modules_exist_in_the_starter(): allowed = { "__init__.py", "checkpoint.py", "compaction.py", + "evaluation.py", "generation.py", "loop.py", "protocol.py", @@ -218,17 +227,30 @@ def test_day_5_compaction_surface_is_complete(): assert {name for name, *_ in compaction["CompactionResult"]} == {"saved_tokens"} -def test_day_6_steering_surface_is_complete_and_has_no_day_7_api(): +def test_day_6_steering_surface_remains_complete(): steering = _public_surface(STARTER, "steering") assert set(steering) == { "AgentStatus", "inspect_checkpoint", "resume_with_steering", } + + +def test_day_7_evaluation_surface_is_complete_and_has_no_future_api(): + evaluation = _public_surface(STARTER, "evaluation") + assert set(evaluation) == { + "EvaluationCase", + "EvaluationCheck", + "EvaluationReport", + "FileExpectation", + "ReceiptExpectation", + "ResultExpectation", + "evaluate_run", + } source = "\n".join( path.read_text(encoding="utf-8") for path in STARTER.glob("*.py") ) - for future_name in ("Session", "rewind", "reconcile", "evaluation"): + for future_name in ("Session", "rewind", "reconcile", "llm_judge", "radix"): assert future_name not in source From fdf5f1431cbd6105981a42cd88af36e87169b147 Mon Sep 17 00:00:00 2001 From: Alex Chi Z Date: Wed, 12 Aug 2026 05:24:12 -0700 Subject: [PATCH 2/4] test: guard Day 7 evaluation invariants AI-Assisted: GPT-5.6 Sol + Forge --- tests_refsol/test_week_4_day_7.py | 91 +++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/tests_refsol/test_week_4_day_7.py b/tests_refsol/test_week_4_day_7.py index 362e8dab..bc081b1c 100644 --- a/tests_refsol/test_week_4_day_7.py +++ b/tests_refsol/test_week_4_day_7.py @@ -140,6 +140,73 @@ def test_task_2_each_wrong_fact_fails_only_its_named_check(tmp_path, fact, faile assert [check.name for check in report.checks if not check.passed] == [failed_name] +def test_task_2_incomplete_run_cannot_pass_from_matching_final_text(tmp_path): + run, workspace, receipts, case, _approvals = _completed_case(tmp_path) + incomplete = replace(run, completed=False, reason="step_limit") + + report = evaluate_run(incomplete, workspace, receipts, case) + + assert not report.passed + assert [check.name for check in report.checks if not check.passed] == ["final"] + + +def test_task_2_file_content_requires_exact_bytes(tmp_path): + run, workspace, receipts, case, _approvals = _completed_case(tmp_path) + (tmp_path / "app.py").write_text("answer = 2\nextra\n", encoding="utf-8") + + report = evaluate_run(run, workspace, receipts, case) + + assert not report.passed + assert [check.name for check in report.checks if not check.passed] == [ + "file:app.py" + ] + + +def test_task_2_tool_and_result_must_match_the_same_event(tmp_path): + run, workspace, receipts, case, _approvals = _completed_case(tmp_path) + split_evidence = replace( + run, + events=( + replace(run.events[0], result="edited app.py"), + replace(run.events[1], result="edit completed"), + *run.events[2:], + ), + ) + + report = evaluate_run(split_evidence, workspace, receipts, case) + + assert not report.passed + assert [check.name for check in report.checks if not check.passed] == [ + "result:edit_file" + ] + + +@pytest.mark.parametrize( + ("field_name", "mismatch"), + [ + ("exit_state", "error"), + ("result_contains", "not in the receipt"), + ("changed_artifacts", ("other.py",)), + ], +) +def test_task_2_each_receipt_fact_is_independently_required( + tmp_path, field_name, mismatch +): + run, workspace, receipts, case, _approvals = _completed_case(tmp_path) + wrong_receipt = replace(case.receipts[0], **{field_name: mismatch}) + changed_case = replace( + case, + receipts=(wrong_receipt, case.receipts[1]), + ) + + report = evaluate_run(run, workspace, receipts, changed_case) + + assert not report.passed + assert [check.name for check in report.checks if not check.passed] == [ + "receipt:call-1" + ] + + def test_task_3_nonessential_final_phrasing_and_event_order_are_not_graded(tmp_path): run, workspace, receipts, case, _approvals = _completed_case(tmp_path) alternate = replace( @@ -222,7 +289,31 @@ def test_task_6_evaluation_is_pure(tmp_path): assert after == before +def test_task_6_file_evidence_does_not_dispatch_the_stateful_read_tool( + tmp_path, monkeypatch +): + run, workspace, receipts, case, _approvals = _completed_case(tmp_path) + (tmp_path / "evidence.txt").write_text("declared evidence\n", encoding="utf-8") + changed_case = replace( + case, + files=(FileExpectation("evidence.txt", "declared evidence\n"),), + ) + monkeypatch.setattr( + workspace, + "read_file", + lambda _path: (_ for _ in ()).throw( + AssertionError("evaluation must not dispatch workspace.read_file") + ), + ) + + report = evaluate_run(run, workspace, receipts, changed_case) + + assert report.passed + + def test_task_7_invalid_evaluation_specifications_fail_closed(tmp_path): + with pytest.raises(ValueError, match="final substring"): + EvaluationCase(" ") with pytest.raises(ValueError, match="file path"): FileExpectation("../escape", "data") with pytest.raises(ValueError, match="result substring"): From c47407f19cd13580602fc64ae7dcce776b5d91bf Mon Sep 17 00:00:00 2001 From: Alex Chi Z Date: Wed, 12 Aug 2026 05:49:13 -0700 Subject: [PATCH 3/4] fix: preserve exact Day 7 file bytes AI-Assisted: GPT-5.6 Sol + Forge --- src/tiny_llm_ref/agent/evaluation.py | 2 +- tests_refsol/test_week_4_day_7.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/tiny_llm_ref/agent/evaluation.py b/src/tiny_llm_ref/agent/evaluation.py index 1be628d1..a07e1fea 100644 --- a/src/tiny_llm_ref/agent/evaluation.py +++ b/src/tiny_llm_ref/agent/evaluation.py @@ -146,7 +146,7 @@ def render(self) -> str: def _file_check(workspace: Workspace, expected: FileExpectation) -> EvaluationCheck: name = f"file:{expected.path}" try: - observed = workspace.resolve_path(expected.path).read_text(encoding="utf-8") + observed = workspace.resolve_path(expected.path).read_bytes().decode("utf-8") except (AgentError, OSError, UnicodeError) as error: return EvaluationCheck(name, False, f"evidence unavailable: {error}") if observed == expected.content: diff --git a/tests_refsol/test_week_4_day_7.py b/tests_refsol/test_week_4_day_7.py index bc081b1c..28827c43 100644 --- a/tests_refsol/test_week_4_day_7.py +++ b/tests_refsol/test_week_4_day_7.py @@ -162,6 +162,18 @@ def test_task_2_file_content_requires_exact_bytes(tmp_path): ] +def test_task_2_file_content_does_not_normalize_newlines(tmp_path): + run, workspace, receipts, case, _approvals = _completed_case(tmp_path) + (tmp_path / "app.py").write_bytes(b"answer = 2\r\n") + + report = evaluate_run(run, workspace, receipts, case) + + assert not report.passed + assert [check.name for check in report.checks if not check.passed] == [ + "file:app.py" + ] + + def test_task_2_tool_and_result_must_match_the_same_event(tmp_path): run, workspace, receipts, case, _approvals = _completed_case(tmp_path) split_evidence = replace( From 2a8942a6e29547e96730803e5dd5f0d01312e38e Mon Sep 17 00:00:00 2001 From: Alex Chi Z Date: Wed, 12 Aug 2026 06:13:46 -0700 Subject: [PATCH 4/4] test: reject lenient Day 7 UTF-8 decoding AI-Assisted: GPT-5.6 Sol + Forge --- tests_refsol/test_week_4_day_7.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests_refsol/test_week_4_day_7.py b/tests_refsol/test_week_4_day_7.py index 28827c43..1a866fed 100644 --- a/tests_refsol/test_week_4_day_7.py +++ b/tests_refsol/test_week_4_day_7.py @@ -174,6 +174,34 @@ def test_task_2_file_content_does_not_normalize_newlines(tmp_path): ] +def test_task_2_invalid_utf8_is_not_ignored(tmp_path): + run, workspace, receipts, case, _approvals = _completed_case(tmp_path) + (tmp_path / "app.py").write_bytes(b"answer = \xff2\n") + + report = evaluate_run(run, workspace, receipts, case) + + assert not report.passed + assert [check.name for check in report.checks if not check.passed] == [ + "file:app.py" + ] + + +def test_task_2_invalid_utf8_is_not_replaced(tmp_path): + run, workspace, receipts, case, _approvals = _completed_case(tmp_path) + (tmp_path / "app.py").write_bytes(b"answer = \xff2\n") + replacement_case = replace( + case, + files=(FileExpectation("app.py", "answer = \N{REPLACEMENT CHARACTER}2\n"),), + ) + + report = evaluate_run(run, workspace, receipts, replacement_case) + + assert not report.passed + assert [check.name for check in report.checks if not check.passed] == [ + "file:app.py" + ] + + def test_task_2_tool_and_result_must_match_the_same_event(tmp_path): run, workspace, receipts, case, _approvals = _completed_case(tmp_path) split_evidence = replace(