diff --git a/src/tiny_llm/agent/__init__.py b/src/tiny_llm/agent/__init__.py index c7c6ba25..886d2309 100644 --- a/src/tiny_llm/agent/__init__.py +++ b/src/tiny_llm/agent/__init__.py @@ -1,67 +1,53 @@ # WARNING: Under review - generated by LLM. -from .context import ( - ContextLimitError, - ContextManager, - ContextPolicy, - ContextWindow, - WorkingSummary, - append_tool_result, - compact_messages, -) from .branch import BranchStats, RewindError, SequentialBranch -from .checkpoint import CacheManifest, ManifestError, export_cache_manifest, validate_resume -from .compaction import CompactionError, CompactionResult, compact_tool_results, expand_receipt_range, reexpand_receipt_message -from .control import AgentInterrupted, CancellationToken, SteeringHandle -from .harness import EquivalenceReport, PlaneResult, RunSnapshot, compare_runs, snapshot_run -from .evaluation import ( - CandidateSnapshot, - CheckResult, - DirectorySnapshot, - EvaluatedRun, - EvaluationMetrics, - FileSnapshot, - GradeReport, - StaticHeldOutGrader, - StagedTask, - TaskManifest, - TaskPackage, - aggregate_metrics, - evaluate_task, +from .checkpoint import ( + CacheManifest, + ManifestError, + export_cache_manifest, + validate_resume, +) +from .compaction import ( + CompactionError, + CompactionResult, + compact_tool_results, + expand_receipt_range, + reexpand_receipt_message, ) +from .control import AgentInterrupted, CancellationToken, SteeringHandle from .generation import ( GenerationSession, GenerationStats, generate_response, initial_messages, ) -from .loop import AgentEvent, AgentLimits, AgentRun, run_agent -from .receipts import EffectReceipt, ReceiptStore -from .reconcile import ( - ReconciliationResult, - SafeCheckpoint, - largest_safe_checkpoint, - reconcile_effect, - reconcile_interrupted_effects, +from .harness import ( + EquivalenceReport, + PlaneResult, + RunSnapshot, + compare_runs, + snapshot_run, ) -from .status import AgentStateCard, StatusQuery, StatusQueryResult, build_state_card +from .loop import AgentEvent, AgentLimits, AgentRun, run_agent from .protocol import ( AgentError, FinalAction, + TOOL_CATALOG_HASH, ToolAction, build_system_prompt, parse_action, + tool_catalog_hash, ) -from .recovery import ( - Checkpoint, - MutationIntent, - MutationJournal, - PlannedRestore, - RecoveryResult, - UndoPlan, - UndoResult, +from .receipts import EffectReceipt, ReceiptStore +from .reconcile import ( + ReconciliationResult, + SafeCheckpoint, + largest_safe_checkpoint, + reconcile_effect, + reconcile_interrupted_effects, ) from .session import SessionEvent, SessionLog, SessionStore, memory_session +from .status import AgentStateCard, StatusQuery, StatusQueryResult, build_state_card from .workspace import ToolPolicy, Workspace @@ -75,64 +61,38 @@ "BranchStats", "CacheManifest", "CancellationToken", - "CandidateSnapshot", - "CheckResult", - "Checkpoint", "CompactionError", "CompactionResult", - "ContextLimitError", - "ContextManager", - "ContextPolicy", - "ContextWindow", - "DirectorySnapshot", "EffectReceipt", "EquivalenceReport", - "EvaluatedRun", - "EvaluationMetrics", - "FileSnapshot", "FinalAction", "GenerationSession", "GenerationStats", - "GradeReport", "ManifestError", - "MutationIntent", - "MutationJournal", "PlaneResult", - "PlannedRestore", "ReceiptStore", "ReconciliationResult", - "RecoveryResult", "RewindError", "RunSnapshot", "SafeCheckpoint", + "SequentialBranch", "SessionEvent", "SessionLog", "SessionStore", - "SequentialBranch", - "StaticHeldOutGrader", - "StagedTask", "StatusQuery", "StatusQueryResult", "SteeringHandle", + "TOOL_CATALOG_HASH", "ToolAction", "ToolPolicy", - "TaskManifest", - "TaskPackage", - "UndoPlan", - "UndoResult", "Workspace", - "WorkingSummary", - "append_tool_result", - "aggregate_metrics", "build_state_card", "build_system_prompt", "compare_runs", - "compact_messages", "compact_tool_results", "expand_receipt_range", "export_cache_manifest", "generate_response", - "evaluate_task", "initial_messages", "largest_safe_checkpoint", "memory_session", @@ -142,5 +102,6 @@ "reexpand_receipt_message", "run_agent", "snapshot_run", + "tool_catalog_hash", "validate_resume", ] diff --git a/src/tiny_llm/agent/context.py b/src/tiny_llm/agent/context.py deleted file mode 100644 index ee9a0c6b..00000000 --- a/src/tiny_llm/agent/context.py +++ /dev/null @@ -1,87 +0,0 @@ -# WARNING: Under review - generated by LLM. - -from collections.abc import Callable, Iterable -from dataclasses import dataclass - -from .generation import Message -from .protocol import AgentError -from .session import SessionLog - - -@dataclass(frozen=True) -class ContextPolicy: - """Exact token budgets used to derive one model-visible request.""" - - max_tokens: int = 32_768 - reserve_tokens: int = 8_192 - summary_max_tokens: int = 1_024 - max_tool_result_tokens: int = 4_096 - min_recent_turns: int = 2 - - -@dataclass(frozen=True) -class WorkingSummary: - """Stable semantic state retained when older complete turns are hidden.""" - - goal: str - constraints: tuple[str, ...] - facts: tuple[str, ...] - changed_files: tuple[str, ...] - validation: tuple[str, ...] - failed_approaches: tuple[str, ...] - next_step: str - - def __post_init__(self) -> None: - """Normalize and validate immutable summary fields.""" - - pass - - -@dataclass(frozen=True) -class ContextWindow: - """One immutable, exactly encoded request prepared for generation.""" - - messages: tuple[Message, ...] - token_ids: tuple[int, ...] - visible_tool_result_bytes: int - followed_compaction: bool - compaction_event_id: str | None - - -class ContextLimitError(AgentError): - """The required anchors and recent complete turns cannot fit safely.""" - - -class ContextManager: - """Build bounded semantic context without modifying canonical events.""" - - def __init__( - self, - encode_messages: Callable[[list[Message]], Iterable[int]], - policy: ContextPolicy | None = None, - ): - pass - - def prepare( - self, - session: SessionLog, - system_prompt: str, - summarize: Callable[[list[Message]], str] | None = None, - ) -> ContextWindow: - """Prepare one bounded request and durably record any new compaction.""" - - pass - - -def compact_messages(messages: list[Message], max_chars: int) -> list[Message]: - """Retain anchors, standalone messages, and complete recent tool turns.""" - - pass - - -def append_tool_result( - messages: list[Message], response: str, result: str -) -> list[Message]: - """Record an action and its observation as one complete semantic turn.""" - - pass diff --git a/src/tiny_llm/agent/evaluation.py b/src/tiny_llm/agent/evaluation.py deleted file mode 100644 index da67222c..00000000 --- a/src/tiny_llm/agent/evaluation.py +++ /dev/null @@ -1,195 +0,0 @@ -# WARNING: Under review - generated by LLM. - -from collections.abc import Callable -from dataclasses import dataclass -from pathlib import Path -from time import perf_counter - -from .generation import Generate -from .loop import AgentLimits, AgentRun -from .protocol import ToolAction -from .session import SessionLog - - -@dataclass(frozen=True) -class TaskManifest: - """The complete public contract for one sealed evaluation task.""" - - schema_version: int - id: str - prompt: str - max_steps: int - editable_paths: tuple[str, ...] - - -@dataclass(frozen=True) -class FileSnapshot: - """Content-and-mode identity for one regular file.""" - - path: str - mode: int - size: int - sha256: str - - -@dataclass(frozen=True) -class DirectorySnapshot: - """Path-and-mode identity for one directory.""" - - path: str - mode: int - - -@dataclass(frozen=True) -class CandidateSnapshot: - """An immutable in-memory view of a separately copied candidate tree.""" - - root: Path - task_id: str - files: tuple[FileSnapshot, ...] - directories: tuple[DirectorySnapshot, ...] - tree_sha256: str - initial_tree_sha256: str - _contents: tuple[tuple[str, bytes], ...] - - -@dataclass(frozen=True) -class CheckResult: - """One deterministic held-out check outcome.""" - - name: str - type: str - passed: bool - detail: str - - -@dataclass(frozen=True) -class GradeReport: - """A static grade, independent of whether the model returned a final.""" - - status: str - checks: tuple[CheckResult, ...] - forbidden_modifications: tuple[str, ...] - candidate_tree_sha256: str - error: str | None = None - - -@dataclass(frozen=True) -class EvaluationMetrics: - """Metrics reconstructed from the durable session event stream.""" - - model_turns: int - tool_calls: int - malformed_actions: int - tool_errors: int - input_tokens: int | None - output_tokens: int | None - reused_tokens: int | None - rewound_tokens: int | None - prefilled_tokens: int | None - visible_tool_result_bytes: int | None - compactions: int - generation_latency_seconds: float | None - wall_time_seconds: float - terminal_reason: str - - -@dataclass(frozen=True) -class EvaluatedRun: - """The agent trajectory, frozen candidate, independent grade, and metrics.""" - - agent_run: AgentRun - candidate: CandidateSnapshot - grade: GradeReport - metrics: EvaluationMetrics - task_success: bool - - -@dataclass(frozen=True) -class TaskPackage: - """A validated package whose held-out check bytes remain unopened.""" - - root: Path - manifest: TaskManifest - _workspace_capture: object - _held_out_metadata: object - - @classmethod - def load(cls, root: Path) -> "TaskPackage": - # TODO: validate the strict manifest and inert package tree without - # opening the held-out check bytes. - pass - - def stage(self, destination: Path) -> "StagedTask": - # TODO: copy only workspace/ and record a stable initial snapshot. - pass - - -@dataclass(frozen=True) -class StagedTask: - """The public agent workspace and its initial sealed snapshot.""" - - package: TaskPackage - destination: Path - workspace: Path - initial_files: tuple[FileSnapshot, ...] - initial_directories: tuple[DirectorySnapshot, ...] - initial_tree_sha256: str - _workspace_identity: tuple[int, int] - _destination_identity: tuple[int, int] - - def freeze(self, destination: Path) -> CandidateSnapshot: - # TODO: copy a stable, bounded candidate tree for static grading. - pass - - -class StaticHeldOutGrader: - """Grade only allowlisted declarative checks over frozen candidate bytes.""" - - def grade(self, staged: StagedTask, candidate: CandidateSnapshot) -> GradeReport: - # TODO: parse the strict held-out JSON schema and inspect frozen bytes. - pass - - -def aggregate_metrics( - session: SessionLog, - agent_run: AgentRun, - wall_time_seconds: float, -) -> EvaluationMetrics: - """Reconstruct comparable metrics from durable events, preserving unknowns.""" - - pass - - -def evaluate_task( - staged: StagedTask, - generate: Generate, - *, - model: str = "evaluation", - session: SessionLog | None = None, - limits: AgentLimits | None = None, - confirm_tool: Callable[[ToolAction], bool] | None = None, - clock: Callable[[], float] = perf_counter, -) -> EvaluatedRun: - """Run a command-free agent, freeze its bytes, then reveal static checks.""" - - # TODO: keep commands disabled, writes default-No, and reveal held-out checks - # only after the candidate is frozen and evaluation_started is durable. - pass - - -__all__ = [ - "CandidateSnapshot", - "CheckResult", - "DirectorySnapshot", - "EvaluatedRun", - "EvaluationMetrics", - "FileSnapshot", - "GradeReport", - "StaticHeldOutGrader", - "StagedTask", - "TaskManifest", - "TaskPackage", - "aggregate_metrics", - "evaluate_task", -] diff --git a/src/tiny_llm/agent/generation.py b/src/tiny_llm/agent/generation.py index e93253f5..389f069b 100644 --- a/src/tiny_llm/agent/generation.py +++ b/src/tiny_llm/agent/generation.py @@ -4,8 +4,6 @@ from dataclasses import dataclass from typing import Any -from .control import CancellationToken - Message = dict[str, str] Generate = Callable[[list[Message]], str] @@ -42,7 +40,7 @@ def __init__( cache_factory: Callable[[], Iterable[Any]], max_tokens: int, enable_thinking: bool = False, - cancellation: CancellationToken | None = None, + cancellation: Any | None = None, ): pass diff --git a/src/tiny_llm/agent/loop.py b/src/tiny_llm/agent/loop.py index 37a22de7..d83b3b45 100644 --- a/src/tiny_llm/agent/loop.py +++ b/src/tiny_llm/agent/loop.py @@ -1,20 +1,30 @@ # WARNING: Under review - generated by LLM. +"""Week 4, Day 1: the validated agent loop (starter). + +The loop turns model text into one bounded, schema-checked action or final +response. It stops by budget, returns invalid actions to the model, and +guards against repeated identical actions. When a ``session``-like object is +provided, every interaction is recorded as an immutable event in the durable +event tree (Day 3). + +Implement the bodies yourself; the reference solution lives in +``tiny_llm_ref``. +""" + from collections.abc import Callable from dataclasses import dataclass +from typing import Any -from .control import CancellationToken -from .context import ContextManager -from .generation import Generate -from .generation import Message -from .protocol import AgentAction -from .session import SessionLog -from .workspace import Workspace +from .generation import Generate, Message +from .protocol import ( + AgentAction, +) @dataclass(frozen=True) class AgentLimits: - """Week 4, Day 6: budgets that guarantee the loop eventually stops.""" + """Budgets that guarantee the loop eventually stops.""" max_steps: int = 8 max_context_chars: int = 48_000 @@ -22,14 +32,15 @@ class AgentLimits: max_identical_actions: int = 2 def __post_init__(self) -> None: - """Week 4, Day 6: reject budgets that could disable a stop condition.""" + """Reject budgets that could disable a stop condition.""" + # TODO: reject any non-positive budget. pass @dataclass(frozen=True) class AgentEvent: - """Week 4, Day 7: one auditable model/action/tool interaction.""" + """One auditable model/action/tool interaction.""" step: int response: str @@ -45,27 +56,42 @@ class AgentRun: reason: str final: str | None events: tuple[AgentEvent, ...] - modified_files: tuple[str, ...] - task_success: bool | None = None - command_side_effects_untracked: bool = False - uncertain_modified_files: tuple[str, ...] = () - retained_recovery_files: tuple[str, ...] = () - command_cleanup_incomplete: bool = False + modified_files: tuple[str, ...] = () session_id: str | None = None +def _append_tool_result( + messages: list[Message], response: str, result: str +) -> list[Message]: + """Append one assistant response and its tool observation.""" + + # TODO: append assistant + "Tool result: ..." user messages. + pass + + def run_agent( task: str | None, generate: Generate, - workspace: Workspace, + workspace: Any, limits: AgentLimits | None = None, on_event: Callable[[AgentEvent], None] | None = None, *, - session: SessionLog | None = None, - context_manager: ContextManager | None = None, - summarize: Callable[[list[Message]], str] | None = None, - cancellation: CancellationToken | None = None, + session: Any | None = None, ) -> AgentRun: - """Run a bounded loop, optionally recording its canonical durable events.""" - + """Run a bounded validated loop over one task. + + ``workspace`` must expose ``available_tools`` (a frozenset of tool names), + ``execute(action)`` (returning a result string), and ``modified_files`` + (an iterable of paths). When ``session`` is provided (a SessionLog, or + any object exposing ``append`` and ``session_id``), every interaction is + recorded as an immutable event in the durable event tree. + """ + + # TODO: reject an empty task; build the system prompt and initial + # messages; loop up to max_steps; parse each response with + # parse_action(response, workspace.available_tools); feed invalid + # actions back as "error: ..." observations; stop on FinalAction, + # invalid-action limit, identical-action limit, context limit, and step + # limit; record user_message/run_started/tool_call/tool_result/ + # run_finished events and session_id when a session is provided. pass diff --git a/src/tiny_llm/agent/receipts.py b/src/tiny_llm/agent/receipts.py index 5e313b51..3cd5fe78 100644 --- a/src/tiny_llm/agent/receipts.py +++ b/src/tiny_llm/agent/receipts.py @@ -93,3 +93,8 @@ def expand(self, receipt_id: str, *, start: int = 0, end: int | None = None) -> """Verify and re-expand a bounded byte range of one receipt's result.""" pass + + def close(self) -> None: + """Drop the in-memory index; the durable file is already fsynced.""" + + pass diff --git a/src/tiny_llm/agent/recovery.py b/src/tiny_llm/agent/recovery.py deleted file mode 100644 index f0c217a6..00000000 --- a/src/tiny_llm/agent/recovery.py +++ /dev/null @@ -1,133 +0,0 @@ -# WARNING: Under review - generated by LLM. - -from collections.abc import Callable -from dataclasses import dataclass -from pathlib import Path -from typing import Literal - -from .session import SessionLog - - -@dataclass(frozen=True) -class MutationIntent: - """A durable write-ahead description of one intended text-file replace.""" - - id: str - path: str - parent_dev: int - parent_ino: int - before_sha256: str | None - before_content: str | None - before_mode: int | None - after_sha256: str - after_mode: int - cause: Literal["tool", "undo"] - - -@dataclass(frozen=True) -class PlannedRestore: - """One conflict-checked file operation in an undo plan.""" - - intent_id: str - path: str - expected_parent_dev: int - expected_parent_ino: int - expected_sha256: str - expected_mode: int - restore_content: str | None - restore_mode: int | None - remove: bool - covered_intent_ids: tuple[str, ...] - - -@dataclass(frozen=True) -class Checkpoint: - """A named, branch-local boundary in the durable event stream.""" - - id: str - name: str - event_id: str - branch_id: str - - -@dataclass(frozen=True) -class RecoveryResult: - """The observed outcome of resolving one interrupted mutation intent.""" - - intent_id: str - path: str - status: Literal["committed", "not_applied", "conflict"] - - -@dataclass(frozen=True) -class UndoPlan: - """A reviewable, side-effect-free plan for restoring one checkpoint.""" - - checkpoint: Checkpoint - changes: tuple[PlannedRestore, ...] - warnings: tuple[str, ...] - - -@dataclass(frozen=True) -class UndoResult: - """An honest account of the file operations an undo did or did not apply.""" - - restored: tuple[str, ...] - removed: tuple[str, ...] - conflicts: tuple[str, ...] - warnings: tuple[str, ...] - retained_recovery_files: tuple[str, ...] = () - - -class MutationJournal: - """Durably journal bounded text mutations and undo them without clobbering.""" - - session: SessionLog - workspace_root: Path - - def __init__(self, session: SessionLog, workspace_root: Path): - # TODO: validate that the journal is bound to the session workspace. - pass - - def record_intent( - self, - path: str, - after_content: str, - cause: Literal["tool", "undo"] = "tool", - *, - after_mode: int | None = None, - ) -> MutationIntent: - # TODO: fsync a bounded before-image before changing the workspace. - pass - - def commit(self, intent: MutationIntent) -> RecoveryResult: - # TODO: commit only when the current hash matches the intended after-image. - pass - - def recover_pending(self) -> tuple[RecoveryResult, ...]: - # TODO: classify pending intents by hash without changing files. - pass - - def recover(self) -> tuple[RecoveryResult, ...]: - # TODO: provide the startup-recovery alias. - pass - - def create_checkpoint(self, name: str) -> Checkpoint: - # TODO: append a named checkpoint for the active session branch. - pass - - def checkpoint(self, name: str) -> Checkpoint: - # TODO: provide the interactive checkpoint alias. - pass - - def plan_undo(self, checkpoint: Checkpoint | str) -> UndoPlan: - # TODO: collapse committed post-checkpoint mutations by path. - pass - - def apply_undo( - self, - plan: UndoPlan, - confirm: Callable[[UndoPlan], bool] | None = None, - ) -> UndoResult: - # TODO: preflight, confirm, revalidate, then restore in reverse order. - pass diff --git a/src/tiny_llm/agent/workspace.py b/src/tiny_llm/agent/workspace.py index 839a94e3..f8b12432 100644 --- a/src/tiny_llm/agent/workspace.py +++ b/src/tiny_llm/agent/workspace.py @@ -1,18 +1,29 @@ # WARNING: Under review - generated by LLM. +"""Week 4, Day 2: authorize effects and record durable receipts (starter). + +This is the solution-free learner surface for the workspace: bounded tools +over one explicit root, protected/secret path rejection, symlink rejection, +inspection-before-overwrite with observed digests, atomic writes, +default-deny operator approvals, an exact command allowlist, and an +``execute`` dispatcher that records an immutable ``EffectReceipt`` per tool. + +Implement the bodies yourself; the reference solution lives in +``tiny_llm_ref``. This module deliberately has no mutation journal or undo +machinery — Day 6's exactly-once reconcile covers crash/effect recovery. +""" + from collections.abc import Callable from dataclasses import dataclass, field from pathlib import Path -from .control import CancellationToken from .protocol import ToolAction -from .recovery import MutationJournal, RecoveryResult -from .session import SessionLog +from .receipts import EffectReceipt, ReceiptStore @dataclass(frozen=True) class ToolPolicy: - """Week 4, Day 3: filesystem and command boundaries for one workspace.""" + """Filesystem and command boundaries for one workspace.""" root: Path allow_writes: bool = False @@ -25,110 +36,180 @@ class ToolPolicy: _root_identity: tuple[int, int] = field(init=False, repr=False, compare=False) def __post_init__(self) -> None: - """Week 4, Day 3: normalize the root and reject invalid limits.""" + """Normalize the root and reject invalid limits.""" + # TODO: resolve root, reject non-positive limits, too-broad roots, + # and protected roots; record (st_dev, st_ino) identity. pass @dataclass class Workspace: - """Week 4, Days 3--6: bounded tools over one explicit workspace root.""" + """Bounded tools over one explicit workspace root.""" policy: ToolPolicy confirm_tool: Callable[[ToolAction], bool] | None = None - session_log: SessionLog | None = field(default=None, kw_only=True) - cancellation: CancellationToken | None = field(default=None, kw_only=True) observed_files: dict[Path, bytes] = field(default_factory=dict, init=False) modified_files: set[Path] = field(default_factory=set, init=False) uncertain_modified_files: set[Path] = field(default_factory=set, init=False) - retained_recovery_files: set[Path] = field(default_factory=set, init=False) - command_side_effects_untracked: bool = field(default=False, init=False) - command_cleanup_incomplete: bool = field(default=False, init=False) - journal: MutationJournal | None = field(default=None, init=False) - recovery_results: tuple[RecoveryResult, ...] = field(default=(), init=False) + receipt_store: ReceiptStore | None = field(default=None, init=False) + last_receipt: EffectReceipt | None = field(default=None, init=False) - def __post_init__(self) -> None: - """Bind durable mutation recovery to the matching session, when present.""" + def bind_receipt_store(self, store: ReceiptStore) -> None: + """Attach a durable receipt store; every effect after this is recorded.""" - # TODO: create a journal and classify pending intents without changing files. + # TODO: bind before any effect; reject a second binding. pass - def bind_session(self, session_log: SessionLog) -> tuple[RecoveryResult, ...]: - """Bind a pristine workspace to the exact log used by its agent loop.""" + def _record_receipt( + self, + action: ToolAction, + result: str, + changed_artifacts: tuple[str, ...], + *, + tool_call_id: str | None = None, + ) -> None: + """Persist one immutable effect receipt for a dispatched tool.""" - # TODO: reject mismatches and bind an unjournaled workspace before tools run. + # TODO: derive exit_state (ok/error/uncertain), key the receipt to the + # session tool_call id when given, and store it if bound. pass - def _recover_command_state(self) -> None: - """Reconstruct conservative process-side-effect flags from durable events.""" + @property + def available_tools(self) -> frozenset[str]: + """Expose only tools enabled by operator policy.""" - # TODO: match command start/finish IDs and retain unmatched uncertainty. + # TODO: list_files/read_file always; write/edit only when + # allow_writes; run_command only when commands are allowlisted. pass - @property - def mutation_journal(self) -> MutationJournal | None: - """Expose the session-bound journal under a descriptive alias.""" + def resolve_path(self, raw: str, *, must_exist: bool = True) -> Path: + """Reject escapes, protected files, and symlink traversal.""" - # TODO: return the same journal exposed by ``journal``. + # TODO: resolve relative to root, reject escapes and protected + # components, reject symlinked components, require existence. pass - def recover_pending(self) -> tuple[RecoveryResult, ...]: - """Classify pending durable intents without changing workspace files.""" + def _reject_protected(self, relative: Path) -> None: + """Keep repository metadata and common secrets hidden.""" - # TODO: retain and return new recovery outcomes when a journal exists. + # TODO: reject .git, dotfile secrets, and *.pem/*.key components. pass - @property - def available_tools(self) -> frozenset[str]: - """Week 4, Day 3: expose only tools enabled by operator policy.""" + def list_files(self, raw: str = ".") -> str: + """List one directory without following symlinks.""" + # TODO: bounded, sorted listing; skip symlinks and protected entries. pass - def resolve_path(self, raw: str, *, must_exist: bool = True) -> Path: - """Week 4, Day 3: reject escapes, protected files, and symlink traversal.""" + def read_file(self, raw: str) -> str: + """Read one bounded UTF-8 regular file.""" + # TODO: reject symlinks/special files, bound the size, record the + # observed digest for stale-write detection. pass - def _reject_protected(self, relative: Path) -> None: - """Week 4, Day 3: keep repository metadata and common secrets hidden.""" + def write_file(self, raw: str, content: str) -> str: + """Atomically create or replace an inspected file.""" + # TODO: validate (allow_writes, size, inspection), then commit + # atomically; report "wrote ". pass - def list_files(self, raw: str = ".") -> str: - """Week 4, Day 3: list one directory without following symlinks.""" + def _prepare_write(self, raw: str, content: str) -> "_PreparedWrite": + """Validate a write completely before asking for operator approval.""" + # TODO: return a validated prepared write (see _PreparedWrite). pass - def read_file(self, raw: str) -> str: - """Week 4, Day 3: read one bounded UTF-8 regular file.""" + def _commit_write(self, prepared: "_PreparedWrite") -> str: + """Revalidate and commit one previously prepared write.""" + # TODO: revalidate digests, atomic-replace, track modified files. pass - def write_file(self, raw: str, content: str) -> str: - """Week 4, Day 3: atomically create or replace an inspected file.""" + def edit_file(self, raw: str, old: str, new: str) -> str: + """Make one exact, reviewable replacement in a read file.""" + # TODO: require prior read, require exactly one match, commit. pass - def edit_file(self, raw: str, old: str, new: str) -> str: - """Week 4, Day 5: make one exact, reviewable replacement in a read file.""" + def _prepare_edit(self, raw: str, old: str, new: str) -> "_PreparedWrite": + """Validate and compute an edit without changing the workspace.""" + # TODO: return a validated prepared write for the edited content. pass def run_command(self, argv: list[str]) -> str: - """Week 4, Day 5: run only an operator-approved exact argv without a shell.""" + """Run only an operator-approved exact argv without a shell.""" + # TODO: reject when commands are disabled or argv is not allowlisted; + # run with a bounded timeout and bounded output. pass - def bind_receipt_store(self, store) -> None: - """Attach a durable receipt store; every effect after this is recorded.""" + def _prepare_command(self, argv: list[str]) -> tuple[str, ...]: + """Validate one command against the exact operator allowlist.""" + # TODO: reject disabled/unallowlisted commands; return the tuple. pass - def execute( - self, action: ToolAction, *, tool_call_id: str | None = None - ) -> str: - """Week 4, Day 3: dispatch a validated action and return recoverable errors.""" + def _run_command(self, argv: tuple[str, ...]) -> str: + """Execute one command with a hard timeout, returning bounded output.""" + # TODO: Popen in the workspace root, drain output, enforce timeout, + # kill the process group on timeout, track uncertain modifications. + pass + + def _require_confirmation(self, action: ToolAction) -> None: + """Default-deny one model-requested side effect.""" + + # TODO: raise AgentError unless confirm_tool approves the action. + pass + + def execute(self, action: ToolAction, *, tool_call_id: str | None = None) -> str: + """Dispatch a validated action and return recoverable errors.""" + + # TODO: dispatch each tool, wrap recoverable errors as + # "error: ...", truncate output, record an effect receipt. + pass + + def _truncate_result(self, result: str) -> str: + """Bound a tool result to the configured output limit.""" + + # TODO: truncate with a visible marker when over the limit. + pass + + def _open_parent_directory(self, relative: Path) -> tuple[int, str]: + """Open the parent directory of a relative path without following links.""" + + # TODO: walk components with O_NOFOLLOW, reject protected/unsafe ones. + pass + + def _read_regular_at( + self, parent: int, name: str, *, tool: str + ) -> tuple[bytes, object] | None: + """Read one bounded regular file by parent dir fd, rejecting links.""" + + # TODO: O_NOFOLLOW open, reject non-regular/multi-link files, bound size. + pass + + def _revalidate_prepared_write(self, prepared: "_PreparedWrite") -> None: + """Reject a write whose target changed after preparation.""" + + # TODO: compare live digests/mode against the prepared expectation. + pass + + def _read_bounded_file(self, path: Path, *, tool: str) -> bytes: + """Read one bounded regular file, rejecting symlinks and special files.""" + + # TODO: O_NOFOLLOW open, reject non-regular files, bound the size. + pass + + @staticmethod + def _digest(content: bytes) -> bytes: + """Fingerprint observed bytes for stale-write detection.""" + + # TODO: return sha256(content).digest(). pass def _atomic_write( @@ -141,6 +222,19 @@ def _atomic_write( after_mode: int, parent_identity: tuple[int, int], ) -> None: - """Week 4, Day 3: replace a file without exposing a partial write.""" + """Replace a file without exposing a partial write.""" + # TODO: write temp in the same directory, fsync, rename, fsync parent. pass + + +@dataclass(frozen=True) +class _PreparedWrite: + """A validated write and the file state it is allowed to replace.""" + + path: Path + content: bytes + expected_digest: bytes | None + expected_mode: int | None + after_mode: int + parent_identity: tuple[int, int] diff --git a/tests_refsol/test_week_4_starter_sync.py b/tests_refsol/test_week_4_starter_sync.py new file mode 100644 index 00000000..d35288cc --- /dev/null +++ b/tests_refsol/test_week_4_starter_sync.py @@ -0,0 +1,181 @@ +# WARNING: Under review - generated by LLM. + +"""Fail-closed starter/reference sync and anti-leakage guards for Week 4. + +The starter (``tiny_llm.agent``) is the learner surface; the reference +(``tiny_llm_ref.agent``) is the solution. These tests keep them from +silently drifting: + +- every public class/method/function in the reference must exist in the + starter (and vice versa) — a missing or extra member fails; +- the starter must not contain solution logic: public method bodies must be + ``pass`` (or only TODO comments), so learners cannot be shown answers; +- every public member is mapped to a feature/day so the 7-day map stays + enforceable. +""" + +import ast +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) + +STARTER = ROOT / "src" / "tiny_llm" / "agent" +REFSOL = ROOT / "src" / "tiny_llm_ref" / "agent" + +# Module -> (feature, day) map per docs/week4-day-split.md. +FEATURE_MAP = { + "protocol": ("loop + tool protocol", "Day 1"), + "loop": ("loop + tool protocol", "Day 1"), + "generation": ( + "loop + tool protocol (minimal) / KV checkpoint (session)", + "Day 1 / Day 4", + ), + "receipts": ("effect receipts", "Day 2"), + "workspace": ("effect receipts (authorization)", "Day 2"), + "session": ("session tree", "Day 3"), + "checkpoint": ("KV checkpoint", "Day 4"), + "branch": ("sequential rewind", "Day 4"), + "compaction": ("receipt-backed compaction", "Day 5"), + "control": ("steering/status", "Day 6"), + "status": ("steering/status", "Day 6"), + "reconcile": ("exactly-once reconcile", "Day 6"), + "harness": ("equivalence harness", "Day 7"), +} + +MODULES = sorted(FEATURE_MAP) + + +def _public_api(path: Path): + """Return (classes -> public method names, top-level public functions).""" + + tree = ast.parse(path.read_text(encoding="utf-8")) + classes: dict[str, set[str]] = {} + funcs: set[str] = set() + for node in tree.body: + if isinstance(node, ast.ClassDef) and not node.name.startswith("_"): + methods = { + item.name + for item in node.body + if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)) + and not item.name.startswith("_") + } + classes[node.name] = methods + elif isinstance( + node, (ast.FunctionDef, ast.AsyncFunctionDef) + ) and not node.name.startswith("_"): + funcs.add(node.name) + return classes, funcs + + +def _public_bodies(path: Path): + """Return public (class, method) and function names with their bodies.""" + + tree = ast.parse(path.read_text(encoding="utf-8")) + bodies: list[tuple[str, ast.AST | None]] = [] + for node in tree.body: + if isinstance(node, ast.ClassDef) and not node.name.startswith("_"): + for item in node.body: + if isinstance( + item, (ast.FunctionDef, ast.AsyncFunctionDef) + ) and not item.name.startswith("_"): + bodies.append((f"{node.name}.{item.name}", item.body)) + elif isinstance( + node, (ast.FunctionDef, ast.AsyncFunctionDef) + ) and not node.name.startswith("_"): + bodies.append((node.name, node.body)) + return bodies + + +def _is_solution_free(body) -> bool: + """True when the body is a stub (pass / docstring-only / TODO only). + + A real implementation has at least one non-docstring statement (an + assignment, call, return, raise, control flow, etc.). + """ + + if body in (None, []): + return True + real = [ + stmt + for stmt in body + if not isinstance(stmt, ast.Pass) + and not ( + isinstance(stmt, ast.Expr) + and isinstance(stmt.value, ast.Constant) + and isinstance(stmt.value.value, str) + ) + and not ( + isinstance(stmt, ast.Expr) + and isinstance(stmt.value, ast.Constant) + and stmt.value.value is Ellipsis + ) + ] + return not real + + +@pytest.mark.parametrize("module", MODULES) +def test_starter_api_matches_reference(module): + starter = _public_api(STARTER / f"{module}.py") + refsol = _public_api(REFSOL / f"{module}.py") + + assert starter == refsol, ( + f"starter/refsol API drift in {module}: " + f"classes missing={sorted(set(refsol[0]) - set(starter[0]))} " + f"extra={sorted(set(starter[0]) - set(refsol[0]))}" + ) + + +@pytest.mark.parametrize("module", MODULES) +def test_starter_has_no_solution_logic(module): + for name, body in _public_bodies(STARTER / f"{module}.py"): + assert _is_solution_free(body), ( + f"starter {module}.{name} contains solution logic; " + "learner bodies must be pass/TODO-only" + ) + + +@pytest.mark.parametrize("module", MODULES) +def test_reference_implements_every_starter_member(module): + """The reference must implement (non-pass) every starter-declared member.""" + + starter = _public_bodies(STARTER / f"{module}.py") + refsol = dict(_public_bodies(REFSOL / f"{module}.py")) + for name, _body in starter: + assert name in refsol, f"reference missing starter member {module}.{name}" + assert not _is_solution_free(refsol[name]), ( + f"reference {module}.{name} is a stub; it must implement the feature" + ) + + +@pytest.mark.parametrize("module", MODULES) +def test_starter_members_are_mapped_to_features(module): + feature, day = FEATURE_MAP[module] + classes, funcs = _public_api(STARTER / f"{module}.py") + assert classes or funcs, f"{module} has no public surface" + # The map itself is the guard: module exists and is non-empty, and its + # feature/day annotation is recorded for reviewers. + assert feature and day + + +def test_starter_package_exports_match_reference(): + import tiny_llm.agent as starter + import tiny_llm_ref.agent as refsol + + assert set(starter.__all__) == set(refsol.__all__), ( + f"starter/refsol __all__ drift: " + f"missing={sorted(set(refsol.__all__) - set(starter.__all__))} " + f"extra={sorted(set(starter.__all__) - set(refsol.__all__))}" + ) + + +def test_no_stale_old_course_modules_in_starter(): + """The old 7-day modules (context/evaluation/recovery) must be gone.""" + + for stale in ("context.py", "evaluation.py", "recovery.py"): + assert not (STARTER / stale).exists(), ( + f"stale old-course module {stale} must not exist in the starter" + )