diff --git a/examples/loopx-turn-fake-host-walkthrough-smoke.py b/examples/loopx-turn-fake-host-walkthrough-smoke.py index 221fee524..0a204a14c 100644 --- a/examples/loopx-turn-fake-host-walkthrough-smoke.py +++ b/examples/loopx-turn-fake-host-walkthrough-smoke.py @@ -14,8 +14,11 @@ sys.path.insert(0, str(REPO_ROOT)) from loopx.control_plane.turn_driver import ( # noqa: E402 + TurnEffectEnvelope, + TurnLeaseController, build_loopx_turn_command_validator, build_loopx_turn_plan, + load_turn_events, load_loopx_turn_plan_from_journal, run_loopx_turn_once, ) @@ -39,6 +42,7 @@ def _envelope(*, action_hash: str, should_run: bool = True) -> dict[str, Any]: "selected_todo": { "todo_id": "todo_fakehost0001", "text": "Advance one synthetic public fixture.", + "required_write_scopes": ["synthetic/**"], } } if should_run @@ -135,17 +139,23 @@ def _validator_argv(effect_path: Path) -> list[str]: def _callbacks(calls: dict[str, int], phases: list[str]): - def writeback(_result: dict[str, Any]) -> dict[str, Any]: + def writeback( + _effect: TurnEffectEnvelope, + _result: dict[str, Any], + ) -> dict[str, Any]: calls["writeback"] += 1 phases.append("writeback") return {"ok": True, "appended": True, "classification": "fixture_progress"} - def spend() -> dict[str, Any]: + def spend(_effect: TurnEffectEnvelope) -> dict[str, Any]: calls["spend"] += 1 phases.append("spend") return {"ok": True, "appended": True, "slots": 1} - def scheduler(_spend: dict[str, Any]) -> dict[str, Any]: + def scheduler( + _effect: TurnEffectEnvelope, + _spend: dict[str, Any], + ) -> dict[str, Any]: calls["scheduler"] += 1 phases.append("scheduler") return { @@ -160,6 +170,7 @@ def scheduler(_spend: dict[str, Any]) -> dict[str, Any]: def _common( *, root: Path, + plan: dict[str, Any], calls: dict[str, int], phases: list[str], ) -> tuple[dict[str, Any], Path, Path]: @@ -167,11 +178,57 @@ def _common( project.mkdir(parents=True) effect_path = project / "synthetic-task.json" count_path = root / "host-count" + state_path = project / "ACTIVE_GOAL_STATE.md" + state_path.write_text( + "\n".join( + [ + "---", + "status: active", + "---", + "", + "# Fake Host Walkthrough", + "", + "## Agent Todo", + "", + "- [ ] [P0] Advance one synthetic public fixture.", + " ", + "", + ] + ), + encoding="utf-8", + ) + runtime_root = root / "runtime" + registry_path = root / "registry.json" + registry_path.write_text( + json.dumps( + { + "schema_version": 1, + "common_runtime_root": str(runtime_root), + "goals": [ + { + "id": "fake-host-walkthrough", + "status": "active", + "repo": str(project), + "state_file": state_path.name, + "coordination": { + "registered_agents": ["generic-fixture-host"], + }, + } + ], + } + ), + encoding="utf-8", + ) + transaction = plan.get("transaction") + assert isinstance(transaction, dict) + turn_key = str(transaction["turn_key"]) writeback, spend, scheduler = _callbacks(calls, phases) return { "host_argv": _host_argv(effect_path, count_path), "project": project, - "runtime_root": root / "runtime", + "runtime_root": runtime_root, "goal_id": "fake-host-walkthrough", "timeout_seconds": 5, "execute": True, @@ -183,6 +240,15 @@ def _common( "writeback": writeback, "spend": spend, "scheduler": scheduler, + "lease_controller": TurnLeaseController( + registry_path=registry_path, + runtime_root=runtime_root, + goal_id="fake-host-walkthrough", + todo_id="todo_fakehost0001", + owner="generic-fixture-host", + idempotency_key=f"turn:{turn_key}", + write_scopes=["synthetic/**"], + ), }, effect_path, count_path @@ -190,7 +256,12 @@ def _commit_replay_and_boundary(root: Path) -> dict[str, Any]: plan = _plan(action_hash="sha256:fake-host-commit") calls = {"writeback": 0, "spend": 0, "scheduler": 0} phases: list[str] = [] - kwargs, effect_path, count_path = _common(root=root, calls=calls, phases=phases) + kwargs, effect_path, count_path = _common( + root=root, + plan=plan, + calls=calls, + phases=phases, + ) preview = run_loopx_turn_once(plan, **{**kwargs, "execute": False}) committed = run_loopx_turn_once(plan, **kwargs) @@ -215,10 +286,19 @@ def _commit_replay_and_boundary(root: Path) -> dict[str, Any]: "synthetic_public_fixture" ) - journal = next( - (root / "runtime" / "goals" / "fake-host-walkthrough" / "turns").glob("*.json") + events = load_turn_events( + root / "runtime", + "fake-host-walkthrough", + committed["resume_turn_key"], ) - journal_payload = json.loads(journal.read_text(encoding="utf-8")) + states = [ + event["payload"]["state"] + for event in events + if isinstance(event.get("payload"), dict) + and isinstance(event["payload"].get("state"), dict) + ] + assert states + journal_payload = states[-1] assert journal_payload["host"] == { "executable": Path(sys.executable).name, "argv_count": 4, @@ -254,10 +334,15 @@ def _recover_after_writeback(root: Path) -> dict[str, Any]: plan = _plan(action_hash="sha256:fake-host-recovery") calls = {"writeback": 0, "spend": 0, "scheduler": 0} phases: list[str] = [] - kwargs, effect_path, count_path = _common(root=root, calls=calls, phases=phases) + kwargs, effect_path, count_path = _common( + root=root, + plan=plan, + calls=calls, + phases=phases, + ) healthy_spend = kwargs["spend"] - def interrupted_spend() -> dict[str, Any]: + def interrupted_spend(_effect: TurnEffectEnvelope) -> dict[str, Any]: calls["spend"] += 1 phases.append("spend-interrupted") raise SystemExit(8) diff --git a/loopx/benchmark_adapters/skillsbench_turn_runtime.py b/loopx/benchmark_adapters/skillsbench_turn_runtime.py index 71f09014d..ea266781d 100644 --- a/loopx/benchmark_adapters/skillsbench_turn_runtime.py +++ b/loopx/benchmark_adapters/skillsbench_turn_runtime.py @@ -16,13 +16,31 @@ import subprocess import time import uuid -from collections.abc import Callable, Mapping +from collections.abc import Callable, Iterator, Mapping +from contextlib import contextmanager from dataclasses import dataclass, replace from pathlib import Path from typing import Any, Union from ..benchmark_case_state import benchmark_case_loopx_command_prefix -from ..control_plane.turn_driver import run_loopx_turn_once +from ..control_plane.turn_driver import ( + TurnEffectEnvelope, + TurnFence, + TurnJournalError, + TurnJournalStore, + hold_turn_lease_heartbeat, + run_loopx_turn_once, + selected_turn_todo, + selected_turn_todo_write_scopes, +) +from ..control_plane.work_items.task_lease import ( + DEFAULT_TASK_LEASE_TTL_SECONDS, + MAX_TASK_LEASE_TTL_SECONDS, + TaskLeaseError, + require_task_lease_fence_value, + task_lease_fencing_generation, + task_lease_fencing_token, +) from .skillsbench_acp_failure_policy import ( RECOVERABLE_CODEX_TURN_FAILURE_PREFIX, recoverable_codex_turn_failure_message, @@ -114,9 +132,16 @@ def _loopx_cli_failure_category(stdout: Any, stderr: Any) -> str: payload = json.loads(str(stdout or "")) except json.JSONDecodeError: payload = {} - error = " ".join( - f"{payload.get('error') if isinstance(payload, dict) else ''} {stderr or ''}".lower().split() + typed_errors = ( + [ + value + for key in ("error_code", "reason", "error") + if isinstance((value := payload.get(key)), str) + ] + if isinstance(payload, dict) + else [] ) + error = " ".join(" ".join([*typed_errors, str(stderr or "")]).lower().split()) classifiers = ( (r"/app/\.local/bin/loopx.*not found", "case_loopx_cli_missing"), (r"no module named loopx", "case_loopx_source_missing"), @@ -128,6 +153,11 @@ def _loopx_cli_failure_category(stdout: Any, stderr: Any) -> str: (r"goal .*not found|no matching goal", "case_goal_not_found"), (r"public boundary|private material", "case_public_boundary_rejected"), (r"operator inbox|lark", "case_operator_inbox_projection_failed"), + (r"todo_lease_conflict|conflicts with active lease", "todo_lease_conflict"), + (r"stale_fencing_token|fencing token is stale", "stale_fencing_token"), + (r"lease_not_active|lease is missing or expired", "lease_not_active"), + (r"lease_cas_mismatch|lease owner or idempotency key mismatch", "lease_cas_mismatch"), + (r"turn journal|journal.*conflict|journal.*invalid", "journal_invariant_failed"), ) for pattern, category in classifiers: if re.search(pattern, error): @@ -230,6 +260,251 @@ def loopx_json(self, command: str) -> dict[str, Any]: return payload +def _bridge_authority_error(exc: SkillsBenchTurnBridgeError) -> None: + if exc.category in { + "todo_lease_conflict", + "stale_fencing_token", + "lease_not_active", + "lease_cas_mismatch", + }: + raise TaskLeaseError( + "scored-workspace Turn lease rejected the operation", + code=exc.category, + ) from exc + if exc.category == "journal_invariant_failed": + raise TurnJournalError( + "scored-workspace Turn journal operation failed" + ) from exc + raise exc + + +def _turn_fence_from_remote_lease(lease: Mapping[str, Any]) -> TurnFence: + normalized = dict(lease) + fencing_token = task_lease_fencing_token(normalized) + return TurnFence( + goal_id=str(normalized.get("goal_id") or ""), + todo_id=str(normalized.get("todo_id") or ""), + owner=str(normalized.get("owner") or ""), + idempotency_key=str(normalized.get("idempotency_key") or ""), + generation=task_lease_fencing_generation(normalized), + version=int(normalized.get("version") or 0), + **{"token": fencing_token}, + ) + + +class SkillsBenchTurnLeaseController: + """Task-lease adapter whose authority lives in the scored workspace.""" + + def __init__( + self, + *, + bridge: SkillsBenchTurnBridge, + config: SkillsBenchTurnRuntimeConfig, + turn_key: str, + todo_id: str, + write_scopes: list[str], + ) -> None: + self._bridge = bridge + self._prefix = _case_cli_prefix(config) + self._goal_id = config.goal_id + self._todo_id = todo_id + self._owner = config.agent_id + self._idempotency_key = f"turn:{turn_key}" + self._write_scopes = list(write_scopes) + self._ttl_seconds = min( + MAX_TASK_LEASE_TTL_SECONDS, + max( + DEFAULT_TASK_LEASE_TTL_SECONDS, + int(config.agent_timeout_seconds) + 300, + ), + ) + self._heartbeat_interval_seconds = self._ttl_seconds / 3 + + def _lease_command( + self, + action: str, + *, + expected_version: int | None = None, + ) -> dict[str, Any]: + command = ( + f"{self._prefix} task-lease {action} " + f"--goal-id {shlex.quote(self._goal_id)} " + f"--todo-id {shlex.quote(self._todo_id)}" + ) + if action != "inspect": + command += ( + f" --owner {shlex.quote(self._owner)} " + f"--idempotency-key {shlex.quote(self._idempotency_key)}" + ) + if action in {"acquire", "renew"}: + command += f" --ttl-seconds {self._ttl_seconds}" + if action == "acquire": + command += "".join( + f" --write-scope {shlex.quote(scope)}" + for scope in self._write_scopes + ) + if expected_version is not None: + command += f" --expected-version {expected_version}" + try: + payload = self._bridge.loopx_json(command) + except SkillsBenchTurnBridgeError as exc: + _bridge_authority_error(exc) + raise AssertionError("unreachable") + if payload.get("ok") is not True: + raise TaskLeaseError( + "scored-workspace Turn lease rejected the operation", + code=str(payload.get("error_code") or "task_lease_failed"), + ) + return payload + + def _fence(self, payload: Mapping[str, Any]) -> TurnFence: + lease = payload.get("lease") + if not isinstance(lease, Mapping): + raise TaskLeaseError( + "scored-workspace Turn lease response was malformed", + code="corrupt_lease", + ) + fence = _turn_fence_from_remote_lease(lease) + if ( + fence.goal_id != self._goal_id + or fence.todo_id != self._todo_id + or fence.owner != self._owner + or fence.idempotency_key != self._idempotency_key + ): + raise TaskLeaseError( + "scored-workspace Turn lease response had mismatched lineage", + code="lease_cas_mismatch", + ) + return fence + + def acquire(self) -> TurnFence: + return self._fence(self._lease_command("acquire")) + + def renew(self, fence: TurnFence) -> TurnFence: + return self._fence( + self._lease_command("renew", expected_version=fence.version) + ) + + def require_current(self, fence: TurnFence) -> None: + payload = self._lease_command("inspect") + lease = payload.get("lease") + if not isinstance(lease, dict): + raise TaskLeaseError( + "scored-workspace Turn lease response was malformed", + code="corrupt_lease", + ) + require_task_lease_fence_value( + lease, + owner=fence.owner, + idempotency_key=fence.idempotency_key, + fencing_token=fence.token, + ) + + def release(self, fence: TurnFence) -> None: + self._lease_command("release", expected_version=fence.version) + + @contextmanager + def heartbeat( + self, + fence: TurnFence, + ) -> Iterator[Callable[[], TurnFence]]: + with hold_turn_lease_heartbeat( + fence, + renew=self.renew, + interval_seconds=self._heartbeat_interval_seconds, + ) as latest: + yield latest + + @contextmanager + def effect_guard(self, _fence: TurnFence) -> Iterator[None]: + # Remote write callbacks carry the same fence into the scored-workspace + # command, which holds its task-lease lock around the durable effect. + yield + + +class SkillsBenchTurnJournalStore(TurnJournalStore): + """Turn journal adapter colocated with the scored-workspace task lease.""" + + def __init__( + self, + *, + bridge: SkillsBenchTurnBridge, + config: SkillsBenchTurnRuntimeConfig, + ) -> None: + self._bridge = bridge + self._prefix = _case_cli_prefix(config) + + def load_events( + self, + *, + runtime_root: Path, + goal_id: str, + turn_key: str, + ) -> list[dict[str, Any]]: + del runtime_root + command = ( + f"{self._prefix} turn journal-read " + f"--goal-id {shlex.quote(goal_id)} " + f"--turn-key {shlex.quote(turn_key)}" + ) + try: + payload = self._bridge.loopx_json(command) + except SkillsBenchTurnBridgeError as exc: + _bridge_authority_error(exc) + raise AssertionError("unreachable") + events = payload.get("events") + if payload.get("ok") is not True or not isinstance(events, list) or not all( + isinstance(event, dict) for event in events + ): + raise TurnJournalError( + "scored-workspace Turn journal read response was malformed" + ) + return [dict(event) for event in events] + + def append_event( + self, + *, + runtime_root: Path, + goal_id: str, + turn_key: str, + event_type: str, + phase_key: str, + fencing: object, + payload: Mapping[str, Any], + ) -> dict[str, Any]: + del runtime_root + request = { + "event_type": event_type, + "phase_key": phase_key, + "fencing": { + "todo_id": str(getattr(fencing, "todo_id", "")), + "owner": str(getattr(fencing, "owner", "")), + "idempotency_key": str( + getattr(fencing, "idempotency_key", "") + ), + "token": str(getattr(fencing, "token", "")), + }, + "payload": dict(payload), + } + command = ( + f"{self._prefix} turn journal-append " + f"--goal-id {shlex.quote(goal_id)} " + f"--turn-key {shlex.quote(turn_key)} " + f"--event-json {shlex.quote(json.dumps(request, sort_keys=True, separators=(',', ':')))}" + ) + try: + response = self._bridge.loopx_json(command) + except SkillsBenchTurnBridgeError as exc: + _bridge_authority_error(exc) + raise AssertionError("unreachable") + event = response.get("event") + if response.get("ok") is not True or not isinstance(event, dict): + raise TurnJournalError( + "scored-workspace Turn journal append response was malformed" + ) + return dict(event) + + def _case_cli_prefix(config: SkillsBenchTurnRuntimeConfig) -> str: return benchmark_case_loopx_command_prefix( case_cli_path=config.case_cli_path, @@ -339,6 +614,19 @@ def _callback_payload(payload: Mapping[str, Any]) -> dict[str, Any]: } +def _turn_effect_fence_arguments( + effect: TurnEffectEnvelope, + *, + todo_id: str, +) -> str: + return ( + f" --turn-effect-key {shlex.quote(effect.phase_key)}" + f" --turn-fence-todo-id {shlex.quote(todo_id)}" + f" --turn-fence-idempotency-key {shlex.quote(f'turn:{effect.turn_key}')}" + f" --turn-fencing-token {shlex.quote(effect.fencing_token)}" + ) + + def _validation_baseline( bridge: SkillsBenchTurnBridge, config: SkillsBenchTurnRuntimeConfig, @@ -424,6 +712,41 @@ def run_skillsbench_loopx_turn( if turn_instance_id is not None else _turn_plan(bridge, config) ) + envelope = plan.get("turn_envelope") + if not isinstance(envelope, Mapping): + raise SkillsBenchTurnBridgeError( + "LoopX Turn plan omitted its Turn envelope", + stage="turn_plan", + category="loopx_turn_plan_not_executable", + ) + selected_todo = selected_turn_todo(envelope) + todo_id = str(selected_todo.get("todo_id") or "") + transaction = plan.get("transaction") + turn_key = str( + transaction.get("turn_key") if isinstance(transaction, Mapping) else "" + ) + if not todo_id or not turn_key: + raise SkillsBenchTurnBridgeError( + "LoopX Turn plan omitted its lease lineage", + stage="turn_plan", + category="loopx_turn_plan_not_executable", + ) + try: + write_scopes = selected_turn_todo_write_scopes(selected_todo) + except ValueError as exc: + raise SkillsBenchTurnBridgeError( + "LoopX Turn selected Todo write scopes were malformed", + stage="turn_plan", + category="loopx_turn_plan_not_executable", + ) from exc + lease_controller = SkillsBenchTurnLeaseController( + bridge=bridge, + config=config, + turn_key=turn_key, + todo_id=todo_id, + write_scopes=write_scopes, + ) + journal_store = SkillsBenchTurnJournalStore(bridge=bridge, config=config) validation_baseline = _validation_baseline(bridge, config) prefix = _case_cli_prefix(config) baseline_path = "" @@ -562,7 +885,10 @@ def validator( "exit_code": 0, } - def writeback(result: dict[str, Any]) -> dict[str, Any]: + def writeback( + effect: TurnEffectEnvelope, + result: dict[str, Any], + ) -> dict[str, Any]: command = ( f"{prefix} refresh-state " f"--goal-id {shlex.quote(config.goal_id)} " @@ -575,19 +901,24 @@ def writeback(result: dict[str, Any]) -> dict[str, Any]: "--progress-scope goal " f"--vision-unchanged-reason {shlex.quote(str(result['vision_unchanged_reason']))} " "--no-global-sync" + + _turn_effect_fence_arguments(effect, todo_id=todo_id) ) return _callback_payload(bridge.loopx_json(command)) - def spend() -> dict[str, Any]: + def spend(effect: TurnEffectEnvelope) -> dict[str, Any]: command = ( f"{prefix} quota spend-slot " f"--goal-id {shlex.quote(config.goal_id)} --slots 1 " "--source adapter --execute " f"--agent-id {shlex.quote(config.agent_id)}" + + _turn_effect_fence_arguments(effect, todo_id=todo_id) ) return _callback_payload(bridge.loopx_json(command)) - def scheduler(_spend_payload: dict[str, Any]) -> dict[str, Any]: + def scheduler( + _effect: TurnEffectEnvelope, + _spend_payload: dict[str, Any], + ) -> dict[str, Any]: command = ( f"{prefix} quota should-run " f"--goal-id {shlex.quote(config.goal_id)} " @@ -621,6 +952,8 @@ def scheduler(_spend_payload: dict[str, Any]) -> dict[str, Any]: writeback=writeback, spend=spend, scheduler=scheduler, + lease_controller=lease_controller, + journal_store=journal_store, ) finally: if baseline_path: diff --git a/loopx/cli_commands/project_lifecycle.py b/loopx/cli_commands/project_lifecycle.py index db45a5bb5..dc2160e26 100644 --- a/loopx/cli_commands/project_lifecycle.py +++ b/loopx/cli_commands/project_lifecycle.py @@ -54,6 +54,11 @@ refresh_state_run, render_state_refresh_markdown, ) +from .task_lease import ( + add_turn_effect_fence_arguments, + hold_cli_turn_effect_fence, + turn_effect_key_from_args, +) PrintPayload = Callable[ @@ -362,6 +367,7 @@ def register_project_lifecycle_commands( "sink writes for this refresh. Pending sink digests remain retryable." ), ) + add_turn_effect_fence_arguments(refresh_state_parser) read_only_map_parser = subparsers.add_parser( "read-only-map", @@ -540,35 +546,46 @@ def handle_project_lifecycle_command( print_payload(payload, fmt, render_state_refresh_markdown) return 1 try: - payload = refresh_state_run( - registry_path=registry_path, - runtime_root_override=args.runtime_root, - goal_id=args.goal_id, - project=Path(args.project).expanduser() if args.project else None, - state_file=Path(args.state_file).expanduser() if args.state_file else None, - classification=args.classification, - recommended_action=args.recommended_action, - next_action=args.next_action, - delivery_batch_scale=args.delivery_batch_scale, - delivery_outcome=args.delivery_outcome, - delivery_workspace_path=( - Path(args.delivery_workspace_path).expanduser() - if args.delivery_workspace_path - else None - ), - todo_id=getattr(args, "todo_id", None), - turn_instance_id=getattr(args, "turn_instance_id", None), - agent_id=args.agent_id, - agent_lane=args.agent_lane, - progress_scope=args.progress_scope, - autonomous_replan_recorded=bool(args.autonomous_replan_recorded), - repair_delta_kinds=args.repair_delta_kinds, - agent_vision_packet=agent_vision_packet, - merge_agent_vision_patch=merge_agent_vision_patch, - vision_unchanged_reason=args.vision_unchanged_reason, - dry_run=bool(args.dry_run), - sync_global=not bool(args.no_global_sync), + effect_runtime_root = resolve_runtime_root( + load_registry(registry_path), + args.runtime_root, ) + with hold_cli_turn_effect_fence( + args, + runtime_root=effect_runtime_root, + goal_id=args.goal_id, + owner=args.agent_id, + ): + payload = refresh_state_run( + registry_path=registry_path, + runtime_root_override=args.runtime_root, + goal_id=args.goal_id, + project=Path(args.project).expanduser() if args.project else None, + state_file=Path(args.state_file).expanduser() if args.state_file else None, + classification=args.classification, + recommended_action=args.recommended_action, + next_action=args.next_action, + delivery_batch_scale=args.delivery_batch_scale, + delivery_outcome=args.delivery_outcome, + delivery_workspace_path=( + Path(args.delivery_workspace_path).expanduser() + if args.delivery_workspace_path + else None + ), + todo_id=getattr(args, "todo_id", None), + turn_instance_id=getattr(args, "turn_instance_id", None), + agent_id=args.agent_id, + agent_lane=args.agent_lane, + progress_scope=args.progress_scope, + autonomous_replan_recorded=bool(args.autonomous_replan_recorded), + repair_delta_kinds=args.repair_delta_kinds, + agent_vision_packet=agent_vision_packet, + merge_agent_vision_patch=merge_agent_vision_patch, + vision_unchanged_reason=args.vision_unchanged_reason, + dry_run=bool(args.dry_run), + sync_global=not bool(args.no_global_sync), + turn_effect_key=turn_effect_key_from_args(args), + ) except Exception as exc: payload = { "ok": False, diff --git a/loopx/cli_commands/quota.py b/loopx/cli_commands/quota.py index 8f51e7919..e8624e0b3 100644 --- a/loopx/cli_commands/quota.py +++ b/loopx/cli_commands/quota.py @@ -14,7 +14,6 @@ fail_heartbeat_receipt, find_heartbeat_receipt, heartbeat_receipt_view, - upgrade_identityless_heartbeat_receipt, ) from ..control_plane.quota.live_decision import build_live_quota_should_run_decision from ..control_plane.quota.monitor_poll import find_quota_monitor_poll_turn @@ -72,6 +71,11 @@ validate_quota_command_request, ) from .quota_registration import register_quota_command as register_quota_command +from .task_lease import ( + hold_cli_turn_effect_fence, + turn_effect_fence_requested, + turn_effect_key_from_args, +) PrintPayload = Callable[ [dict[str, object], str, Callable[[dict[str, object]], str]], @@ -194,6 +198,10 @@ def _prepare_quota_command_context( raise ValueError("turn-scoped quota settlement requires --agent-id") if heartbeat_turn_id and command == "should-run" and bool(args.dry_run): raise ValueError("turn-scoped `quota should-run` cannot use --dry-run") + if turn_effect_fence_requested(args) and command != "spend-slot": + raise ValueError( + "Turn effect fencing is only valid with `quota spend-slot`" + ) scan_roots = [Path(item).expanduser() for item in args.scan_path] if not scan_roots: @@ -587,18 +595,37 @@ def handle_quota_command( failure_kind=args.failure_kind, ) elif args.quota_command == "spend-slot": - payload = spend_quota_slot( - status_payload, + with hold_cli_turn_effect_fence( + args, + runtime_root=runtime_root, goal_id=args.goal_id, - slots=args.slots, - execute=bool(args.execute), - source=args.source, - agent_id=args.agent_id, - available_capabilities=args.available_capabilities, - operator_inbox_urgency_projector=operator_inbox_urgency_projector, - todo_id=args.todo_id, - turn_instance_id=heartbeat_turn_id, - ) + owner=args.agent_id, + ): + spend_status = ( + collect_status( + registry_path=registry_path, + runtime_root_override=runtime_root_arg, + scan_roots=scan_roots, + limit=status_limit, + goal_id=status_goal_id, + available_capabilities=args.available_capabilities, + ) + if turn_effect_fence_requested(args) + else status_payload + ) + payload = spend_quota_slot( + spend_status, + goal_id=args.goal_id, + slots=args.slots, + execute=bool(args.execute), + source=args.source, + agent_id=args.agent_id, + available_capabilities=args.available_capabilities, + operator_inbox_urgency_projector=operator_inbox_urgency_projector, + todo_id=args.todo_id, + turn_instance_id=heartbeat_turn_id, + turn_effect_key=turn_effect_key_from_args(args), + ) elif args.quota_command == "void-slot": payload = void_quota_slot( status_payload, diff --git a/loopx/cli_commands/quota_registration.py b/loopx/cli_commands/quota_registration.py index 45ea2ed52..97ec8960f 100644 --- a/loopx/cli_commands/quota_registration.py +++ b/loopx/cli_commands/quota_registration.py @@ -12,6 +12,7 @@ QUOTA_DETAIL_SECTIONS, register_quota_monitor_poll_request_arguments, ) +from .task_lease import add_turn_effect_fence_arguments def _default_public_scan_root() -> str: @@ -163,6 +164,7 @@ def register_quota_command( default=DEFAULT_SLOT_SPEND_SOURCE, help="Source label for `quota spend-slot`.", ) + add_turn_effect_fence_arguments(quota_parser) quota_parser.add_argument("--void-generated-at", help="generated_at timestamp of the quota_slot_spent run to void.") quota_parser.add_argument("--reason-summary", help="Public-safe reason for `quota void-slot`.") register_quota_monitor_poll_request_arguments(quota_parser) diff --git a/loopx/cli_commands/task_lease.py b/loopx/cli_commands/task_lease.py index d92d4e052..e1948f399 100644 --- a/loopx/cli_commands/task_lease.py +++ b/loopx/cli_commands/task_lease.py @@ -1,13 +1,16 @@ from __future__ import annotations import argparse -from collections.abc import Callable +from collections.abc import Callable, Iterator +from contextlib import contextmanager from pathlib import Path +from ..control_plane.turn_effect import normalize_turn_effect_key from ..control_plane.work_items.task_lease import ( TaskLeaseError, acquire_task_lease, inspect_task_lease, + hold_task_lease_fence, release_task_lease, renew_task_lease, runtime_root_from_registry, @@ -23,6 +26,72 @@ ] +def add_turn_effect_fence_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--turn-effect-key", help=argparse.SUPPRESS) + parser.add_argument("--turn-fence-todo-id", help=argparse.SUPPRESS) + parser.add_argument("--turn-fence-idempotency-key", help=argparse.SUPPRESS) + parser.add_argument("--turn-fencing-token", help=argparse.SUPPRESS) + + +def turn_effect_key_from_args(args: argparse.Namespace) -> str | None: + return normalize_turn_effect_key(getattr(args, "turn_effect_key", None)) + + +def turn_effect_fence_requested(args: argparse.Namespace) -> bool: + return any( + str(getattr(args, attr, None) or "").strip() + for attr in ( + "turn_effect_key", + "turn_fence_todo_id", + "turn_fence_idempotency_key", + "turn_fencing_token", + ) + ) + + +@contextmanager +def hold_cli_turn_effect_fence( + args: argparse.Namespace, + *, + runtime_root: Path, + goal_id: str, + owner: str | None, +) -> Iterator[None]: + values = { + "--turn-effect-key": getattr(args, "turn_effect_key", None), + "--turn-fence-todo-id": getattr(args, "turn_fence_todo_id", None), + "--turn-fence-idempotency-key": getattr( + args, + "turn_fence_idempotency_key", + None, + ), + "--turn-fencing-token": getattr(args, "turn_fencing_token", None), + } + supplied = {flag for flag, value in values.items() if str(value or "").strip()} + if not supplied: + yield + return + missing = sorted(set(values) - supplied) + if missing: + raise ValueError( + "Turn effect fencing requires all internal fence arguments; missing: " + + ", ".join(missing) + ) + if not str(owner or "").strip(): + raise ValueError("Turn effect fencing requires --agent-id") + turn_effect_key_from_args(args) + with hold_task_lease_fence( + runtime_root=runtime_root, + goal_id=goal_id, + todo_id=str(values["--turn-fence-todo-id"]), + owner=str(owner), + idempotency_key=str(values["--turn-fence-idempotency-key"]), + fencing_token=str(values["--turn-fencing-token"]), + operation="cli_turn_effect", + ): + yield + + def render_task_lease_markdown(payload: dict[str, object]) -> str: lines = [ "# LoopX Task Lease", diff --git a/loopx/cli_commands/turn.py b/loopx/cli_commands/turn.py index 275edb186..bf3165129 100644 --- a/loopx/cli_commands/turn.py +++ b/loopx/cli_commands/turn.py @@ -16,14 +16,21 @@ from ..control_plane.turn_driver import ( LOOPX_TURN_EXECUTION_SCHEMA_VERSION, LOOPX_TURN_SESSION_BINDING_SCHEMA_VERSION, + TurnEffectEnvelope, + TurnJournalError, + TurnLeaseController, + append_turn_event, build_loopx_turn_command_validator, build_loopx_turn_plan, codex_cli_session_binding, load_loopx_turn_plan_from_journal, + load_turn_events, run_codex_cli_host, run_loopx_turn_once, selected_turn_todo, + selected_turn_todo_write_scopes, ) +from ..control_plane.work_items.task_lease import TaskLeaseError from ..quota import spend_quota_slot from ..state_refresh import refresh_state_run from ..status import AUTONOMOUS_REPLAN_PERIODIC_LOOKBACK, collect_status @@ -173,6 +180,23 @@ def register_turn_commands( run_once.add_argument("--scan-path", action="append", default=[]) run_once.add_argument("--limit", type=int, default=5) + journal_read = command_sub.add_parser( + "journal-read", + help=argparse.SUPPRESS, + ) + add_subcommand_format(journal_read) + journal_read.add_argument("--goal-id", required=True) + journal_read.add_argument("--turn-key", required=True) + + journal_append = command_sub.add_parser( + "journal-append", + help=argparse.SUPPRESS, + ) + add_subcommand_format(journal_append) + journal_append.add_argument("--goal-id", required=True) + journal_append.add_argument("--turn-key", required=True) + journal_append.add_argument("--event-json", required=True) + def _add_turn_decision_arguments( parser: argparse.ArgumentParser, @@ -270,6 +294,114 @@ def _render_loopx_turn_execution_markdown(payload: dict[str, object]) -> str: ) +def _render_loopx_turn_journal_markdown(payload: dict[str, object]) -> str: + return "\n".join( + [ + "# LoopX Turn Journal", + f"- ok: {payload.get('ok')}", + f"- mode: {payload.get('mode')}", + f"- event_count: {payload.get('event_count')}", + f"- error: {payload.get('error')}", + ] + ) + + +def _handle_turn_journal_command( + args: argparse.Namespace, + *, + registry_path: Path, + runtime_root_arg: str | None, + output_format: FormatSelector, + print_payload: PrintPayload, +) -> int: + runtime_root = resolve_status_projection_cache_runtime_root( + registry_path=registry_path, + runtime_root_override=runtime_root_arg, + ) + try: + if args.turn_command == "journal-read": + events = load_turn_events(runtime_root, args.goal_id, args.turn_key) + payload: dict[str, object] = { + "ok": True, + "schema_version": "loopx_turn_journal_cli_v0", + "mode": "journal_read", + "goal_id": args.goal_id, + "turn_key": args.turn_key, + "event_count": len(events), + "events": events, + } + else: + request = json.loads(args.event_json) + if not isinstance(request, dict) or set(request) != { + "event_type", + "phase_key", + "fencing", + "payload", + }: + raise ValueError("Turn journal event JSON fields are invalid") + fencing = request["fencing"] + event_payload = request["payload"] + if not isinstance(fencing, dict) or set(fencing) != { + "todo_id", + "owner", + "idempotency_key", + "token", + }: + raise ValueError("Turn journal fencing fields are invalid") + if not isinstance(event_payload, dict): + raise ValueError("Turn journal payload must be an object") + event = append_turn_event( + runtime_root=runtime_root, + goal_id=args.goal_id, + turn_key=args.turn_key, + event_type=str(request["event_type"]), + phase_key=str(request["phase_key"]), + fencing=fencing, + payload=event_payload, + ) + payload = { + "ok": True, + "schema_version": "loopx_turn_journal_cli_v0", + "mode": "journal_append", + "goal_id": args.goal_id, + "turn_key": args.turn_key, + "event_count": None, + "event": event, + } + except TaskLeaseError as exc: + payload = { + "ok": False, + "schema_version": "loopx_turn_journal_cli_v0", + "mode": args.turn_command.replace("-", "_"), + "goal_id": args.goal_id, + "turn_key": args.turn_key, + "error": str(exc), + "error_code": exc.code, + } + except TurnJournalError as exc: + payload = { + "ok": False, + "schema_version": "loopx_turn_journal_cli_v0", + "mode": args.turn_command.replace("-", "_"), + "goal_id": args.goal_id, + "turn_key": args.turn_key, + "error": str(exc), + "error_code": "journal_invariant_failed", + } + except Exception as exc: + payload = { + "ok": False, + "schema_version": "loopx_turn_journal_cli_v0", + "mode": args.turn_command.replace("-", "_"), + "goal_id": args.goal_id, + "turn_key": args.turn_key, + "error": str(exc), + "error_code": exc.__class__.__name__, + } + print_payload(payload, output_format(args), _render_loopx_turn_journal_markdown) + return 0 if payload.get("ok") else 1 + + def handle_turn_command( args: argparse.Namespace, *, @@ -280,6 +412,14 @@ def handle_turn_command( ) -> int | None: if args.command != "turn": return None + if args.turn_command in {"journal-read", "journal-append"}: + return _handle_turn_journal_command( + args, + registry_path=registry_path, + runtime_root_arg=runtime_root_arg, + output_format=output_format, + print_payload=print_payload, + ) try: scan_roots = [Path(item).expanduser() for item in args.scan_path] if not scan_roots: @@ -430,6 +570,34 @@ def handle_turn_command( task_validator = None envelope = payload.get("turn_envelope") if isinstance(payload.get("turn_envelope"), dict) else {} selected_todo = selected_turn_todo(envelope) + lease_controller = None + if args.execute: + todo_id = str(selected_todo.get("todo_id") or "") + if not todo_id: + raise ValueError( + "executing run-once requires one selected todo for its task lease" + ) + transaction = ( + payload.get("transaction") + if isinstance(payload.get("transaction"), dict) + else {} + ) + turn_key = str(transaction.get("turn_key") or "") + if not turn_key: + raise ValueError( + "executing run-once requires a transaction turn_key" + ) + write_scopes = selected_turn_todo_write_scopes(selected_todo) + lease_controller = TurnLeaseController( + registry_path=registry_path, + runtime_root=runtime_root, + goal_id=args.goal_id, + todo_id=todo_id, + owner=args.agent_id, + idempotency_key=f"turn:{turn_key}", + write_scopes=write_scopes, + terminal_replay_key=turn_key, + ) writeback_contract = ( envelope.get("writeback") if isinstance(envelope.get("writeback"), dict) @@ -443,7 +611,10 @@ def handle_turn_command( else None ) - def writeback(result: dict[str, object]) -> dict[str, object]: + def writeback( + effect: TurnEffectEnvelope, + result: dict[str, object], + ) -> dict[str, object]: # The host workspace is execution context, not state authority. state_project = None result_kind = str(result.get("result_kind") or "") @@ -487,9 +658,13 @@ def writeback(result: dict[str, object]) -> dict[str, object]: ), dry_run=False, sync_global=not bool(args.no_global_sync), + turn_effect_key=effect.phase_key, ) - def completion_writeback(result: dict[str, object]) -> dict[str, object]: + def completion_writeback( + effect: TurnEffectEnvelope, + result: dict[str, object], + ) -> dict[str, object]: todo_id = str(selected_todo.get("todo_id") or "") if not todo_id: raise ValueError( @@ -501,6 +676,9 @@ def completion_writeback(result: dict[str, object]) -> dict[str, object]: todo_id=todo_id, role="agent", completion_turn_key=str(result["turn_key"]), + task_lease_idempotency_key=f"turn:{effect.turn_key}", + task_lease_runtime_root=runtime_root, + release_task_lease_on_commit=False, evidence=( "LoopX Turn validated completion: " + str(result.get("summary") or result["classification"]) @@ -510,7 +688,7 @@ def completion_writeback(result: dict[str, object]) -> dict[str, object]: project=None, dry_run=False, ) - refresh = writeback(result) + refresh = writeback(effect, result) return { "ok": bool(completion.get("ok")) and bool(refresh.get("ok")), # A completed Todo is idempotent under Turn replay: after an @@ -535,7 +713,7 @@ def current_status() -> dict[str, object]: available_capabilities=args.available_capabilities, ) - def spend() -> dict[str, object]: + def spend(effect: TurnEffectEnvelope) -> dict[str, object]: return spend_quota_slot( current_status(), goal_id=args.goal_id, @@ -546,9 +724,13 @@ def spend() -> dict[str, object]: workspace_path=delivery_workspace_path, available_capabilities=args.available_capabilities, operator_inbox_urgency_projector=operator_inbox_urgency_projector, + turn_effect_key=effect.phase_key, ) - def scheduler(_spend_payload: dict[str, object]) -> dict[str, object]: + def scheduler( + _effect: TurnEffectEnvelope, + _spend_payload: dict[str, object], + ) -> dict[str, object]: turn_scheduler_context = ( payload.get("scheduler_execution_context") if isinstance(payload.get("scheduler_execution_context"), dict) @@ -606,6 +788,7 @@ def run_built_in_host(request: dict[str, object]) -> dict[str, object]: completion_writeback=completion_writeback if args.execute else None, spend=spend if args.execute else None, scheduler=scheduler if args.execute else None, + lease_controller=lease_controller, ) else: raise ValueError("turn requires the `plan` or `run-once` subcommand") diff --git a/loopx/control_plane/quota/slot_accounting.py b/loopx/control_plane/quota/slot_accounting.py index eca2b1cf6..bc88dd91a 100644 --- a/loopx/control_plane/quota/slot_accounting.py +++ b/loopx/control_plane/quota/slot_accounting.py @@ -6,10 +6,17 @@ from pathlib import Path from typing import Any +from ...file_lock import exclusive_file_lock from ..agents.workspace_guard import ( build_delivery_workspace_guard, delivery_workspace_repository, ) +from ..turn_effect import ( + find_turn_effect_record, + normalize_turn_effect_key, + require_matching_turn_effect, + turn_effect_input_hash, +) from ..runtime.run_artifacts import ( next_run_artifact_paths, reserve_run_artifact_paths, @@ -1025,7 +1032,7 @@ def record_quota_slot_void_from_preview( return payload -def record_quota_slot_spend_from_preview( +def _record_quota_slot_spend_from_preview( preview: dict[str, Any], status_payload: dict[str, Any], *, @@ -1034,6 +1041,8 @@ def record_quota_slot_spend_from_preview( render_markdown: Callable[[dict[str, Any]], str], execute: bool = False, source: str = DEFAULT_SLOT_SPEND_SOURCE, + _turn_effect_key: str | None = None, + _effect_input_hash: str | None = None, ) -> dict[str, Any]: safe_goal_id = _validate_goal_id_path_segment(str(goal_id or "")) if not preview.get("ok"): @@ -1046,6 +1055,11 @@ def record_quota_slot_spend_from_preview( source=source, generated_at=generated_at, ) + if _turn_effect_key is not None: + if _effect_input_hash is None: + raise ValueError("turn effect input hash is required") + record["turn_effect_key"] = _turn_effect_key + record["effect_input_hash"] = _effect_input_hash raw_runtime_root = status_payload.get("runtime_root") if not raw_runtime_root: raise ValueError("status payload does not include runtime_root") @@ -1064,6 +1078,9 @@ def record_quota_slot_spend_from_preview( "json_path": str(json_path), "markdown_path": str(markdown_path), } + if _turn_effect_key is not None: + index_record["turn_effect_key"] = _turn_effect_key + index_record["effect_input_hash"] = _effect_input_hash if record.get("agent_id"): index_record["agent_id"] = record["agent_id"] for field in ("turn_instance_id", "todo_id", "settlement_identity"): @@ -1089,6 +1106,10 @@ def record_quota_slot_spend_from_preview( f"{record['quota_event']['after']['spent_slots']} slots" ), } + if _turn_effect_key is not None: + payload["turn_effect_key"] = _turn_effect_key + payload["effect_input_hash"] = _effect_input_hash + payload["idempotent"] = False if execute: payload["before"] = record["quota_event"]["before"] payload["after"] = record["quota_event"]["after"] @@ -1098,3 +1119,98 @@ def record_quota_slot_spend_from_preview( with index_path.open("a", encoding="utf-8") as f: f.write(json.dumps(index_record, ensure_ascii=False) + "\n") return payload + + +def record_quota_slot_spend_from_preview( + preview: dict[str, Any], + status_payload: dict[str, Any], + *, + goal_id: str, + self_repair_spend_actions: set[str] | frozenset[str], + render_markdown: Callable[[dict[str, Any]], str], + execute: bool = False, + source: str = DEFAULT_SLOT_SPEND_SOURCE, + turn_effect_key: str | None = None, +) -> dict[str, Any]: + normalized_effect_key = normalize_turn_effect_key(turn_effect_key) + effect_input_hash = ( + turn_effect_input_hash( + { + "request": { + field: preview.get(field) + for field in ( + "goal_id", + "slots", + "agent_id", + "todo_id", + "turn_instance_id", + ) + }, + "goal_id": goal_id, + "source": source, + "execute": execute, + } + ) + if normalized_effect_key is not None + else None + ) + def write_once() -> dict[str, Any]: + return _record_quota_slot_spend_from_preview( + preview, + status_payload, + goal_id=goal_id, + self_repair_spend_actions=self_repair_spend_actions, + render_markdown=render_markdown, + execute=execute, + source=source, + _turn_effect_key=normalized_effect_key, + _effect_input_hash=effect_input_hash, + ) + + if normalized_effect_key is None or not execute: + return write_once() + safe_goal_id = _validate_goal_id_path_segment(str(goal_id or "")) + raw_runtime_root = status_payload.get("runtime_root") + if not raw_runtime_root: + raise ValueError("status payload does not include runtime_root") + runtime_root = Path(str(raw_runtime_root)).expanduser() + index_path = runtime_root / "goals" / safe_goal_id / "runs" / "index.jsonl" + with exclusive_file_lock( + index_path, + agent_id=preview.get("agent_id"), + operation="quota_spend_turn_effect", + ): + existing = find_turn_effect_record(index_path, normalized_effect_key) + if existing is not None: + assert effect_input_hash is not None + require_matching_turn_effect(existing, effect_input_hash) + quota_event = load_quota_event_from_run(existing) + if quota_event is None: + raise ValueError("turn effect durable record is missing quota_event") + before = quota_event.get("before") + after = quota_event.get("after") + if not isinstance(before, dict) or not isinstance(after, dict): + raise ValueError("turn effect durable quota_event is incomplete") + return { + **preview, + "ok": True, + "dry_run": False, + "appended": False, + "idempotent": True, + "idempotent_replay": True, + "registry_mutated": False, + "source": quota_event.get("source"), + "classification": QUOTA_SLOT_SPENT_CLASSIFICATION, + "generated_at": existing.get("generated_at"), + "agent_id": existing.get("agent_id"), + "quota_event": quota_event, + "before": before, + "after": after, + "json_path": existing.get("json_path"), + "markdown_path": existing.get("markdown_path"), + "index_path": str(index_path), + "turn_effect_key": normalized_effect_key, + "effect_input_hash": effect_input_hash, + "reason": "quota slot spend replayed for the same Turn effect", + } + return write_once() diff --git a/loopx/control_plane/quota/task_orchestration_admission.py b/loopx/control_plane/quota/task_orchestration_admission.py index aaf9cd5bc..0cbe7288c 100644 --- a/loopx/control_plane/quota/task_orchestration_admission.py +++ b/loopx/control_plane/quota/task_orchestration_admission.py @@ -139,6 +139,10 @@ def build_adaptive_task_orchestration_contract( return None primary = admitted[0] + primary_todo = { + "todo_id": primary.todo_id, + "required_write_scopes": list(primary.required_write_scopes), + } child_brief_defaults = _child_brief_defaults( parent_goal_id=parent_goal_id, available_capabilities=available_capabilities, @@ -188,6 +192,7 @@ def build_adaptive_task_orchestration_contract( "strategy_owner": "task_coordinator", "max_children": max_children, "primary_todo_id": primary.todo_id, + "primary_todo": primary_todo, "child_brief_defaults": child_brief_defaults, "eligible_child_lanes": selected_children, "blocked_lanes": blocked, diff --git a/loopx/control_plane/quota/turn_envelope.py b/loopx/control_plane/quota/turn_envelope.py index 2df391ce1..8df742f06 100644 --- a/loopx/control_plane/quota/turn_envelope.py +++ b/loopx/control_plane/quota/turn_envelope.py @@ -29,6 +29,8 @@ ACTION_SIGNATURE_SCHEMA_VERSION = "loopx_action_signature_v0" ACTION_SIGNATURE_COVERAGE_V0 = "turn_envelope_action_dimensions_v0" ACTION_SIGNATURE_COVERAGE_V1 = "turn_envelope_action_dimensions_v1" +ACTION_SIGNATURE_COVERAGE_V2 = "turn_envelope_action_dimensions_v2" +ACTION_SIGNATURE_COVERAGE_V3 = "turn_envelope_action_dimensions_v3" ACTION_SIGNATURE_COVERAGE = ACTION_SIGNATURE_COVERAGE_V0 ACTIONABLE_WARNING_FIELDS = ( "state_projection_gap", @@ -256,6 +258,7 @@ def _selected_todo( "task_class", "action_kind", "task_repository", + "required_write_scopes", "continuation_policy", "claimed_by", "bound_agent", @@ -641,6 +644,18 @@ def _action_projection( or payload.get("recommended_action"), limit=480, ) + selected_todo_source: Mapping[str, Any] = payload + task_orchestration = _mapping(payload.get("task_orchestration_contract")) + primary_todo_id = str(task_orchestration.get("primary_todo_id") or "").strip() + primary_todo = _mapping(task_orchestration.get("primary_todo")) + if ( + task_orchestration.get("schema_version") + == "task_orchestration_contract_v2" + and task_orchestration.get("mode") == "adaptive" + and primary_todo_id + and str(primary_todo.get("todo_id") or "") == primary_todo_id + ): + selected_todo_source = {"selected_todo": primary_todo} action = { "recommended_action": recommended_action, "primary_action": _text(agent_channel.get("primary_action"), limit=480), @@ -648,7 +663,7 @@ def _action_projection( "delivery_allowed": bool(agent_channel.get("delivery_allowed")), "quiet_noop_allowed": bool(agent_channel.get("quiet_noop_allowed")), "selected_todo": _selected_todo( - payload, + selected_todo_source, recommended_action=recommended_action, ), } @@ -690,8 +705,7 @@ def _action_projection( scheduler=scheduler, ), } - task_orchestration = payload.get("task_orchestration_contract") - if isinstance(task_orchestration, Mapping): + if task_orchestration: projection["task_orchestration_contract"] = dict(task_orchestration) response_plan = _response_plan(interaction) if response_plan is not None: @@ -717,11 +731,16 @@ def turn_envelope_action_signature_document(envelope: Mapping[str, Any]) -> dict "task_orchestration_contract", ) response_plan = envelope.get("response_plan") - coverage = ( - ACTION_SIGNATURE_COVERAGE_V1 - if isinstance(response_plan, Mapping) - else ACTION_SIGNATURE_COVERAGE_V0 - ) + action = _mapping(envelope.get("action")) + selected_todo = _mapping(action.get("selected_todo")) + has_write_scopes = "required_write_scopes" in selected_todo + has_response_plan = isinstance(response_plan, Mapping) + coverage = { + (False, False): ACTION_SIGNATURE_COVERAGE_V0, + (False, True): ACTION_SIGNATURE_COVERAGE_V1, + (True, False): ACTION_SIGNATURE_COVERAGE_V2, + (True, True): ACTION_SIGNATURE_COVERAGE_V3, + }[(has_write_scopes, has_response_plan)] signature = { "schema_version": ACTION_SIGNATURE_SCHEMA_VERSION, "coverage": coverage, diff --git a/loopx/control_plane/state_refresh_effect.py b/loopx/control_plane/state_refresh_effect.py new file mode 100644 index 000000000..2bc15da75 --- /dev/null +++ b/loopx/control_plane/state_refresh_effect.py @@ -0,0 +1,192 @@ +"""Exactly-once Turn effect wrapper for the state refresh writer.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from ..file_lock import exclusive_file_lock +from .turn_effect import ( + find_turn_effect_record, + normalize_turn_effect_key, + require_matching_turn_effect, + turn_effect_input_hash, +) +from .work_items.delivery_batch_scale import require_delivery_batch_scale +from .work_items.delivery_outcome import require_delivery_outcome + + +def refresh_state_run( + *, + registry_path: Path, + runtime_root_override: str | None, + goal_id: str, + project: Path | None, + state_file: Path | None, + classification: str, + recommended_action: str | None, + next_action: str | None = None, + delivery_batch_scale: str | None = None, + delivery_outcome: str | None = None, + delivery_workspace_path: Path | None = None, + todo_id: str | None = None, + turn_instance_id: str | None = None, + agent_id: str | None = None, + agent_lane: str | None = None, + progress_scope: str | None = None, + autonomous_replan_recorded: bool = False, + repair_delta_kinds: list[str] | None = None, + agent_vision_packet: dict[str, Any] | None = None, + merge_agent_vision_patch: bool = False, + vision_unchanged_reason: str | None = None, + dry_run: bool, + sync_global: bool = True, + turn_effect_key: str | None = None, +) -> dict[str, Any]: + # Imported lazily so loopx.state_refresh can preserve its public function + # while this module wraps the already-initialized core writer. + from .. import state_refresh as state_refresh_core + + normalized_effect_key = normalize_turn_effect_key(turn_effect_key) + effect_input_hash: str | None = None + effect_runtime_root: Path | None = None + safe_effect_goal_id: str | None = None + if normalized_effect_key is not None: + safe_effect_goal_id = state_refresh_core.validate_goal_id_path_segment(goal_id) + effect_registry = state_refresh_core.load_registry(registry_path) + effect_runtime_root = state_refresh_core.resolve_runtime_root( + effect_registry, + runtime_root_override, + ).expanduser().resolve() + _registry_goal, effect_project, effect_state_file = ( + state_refresh_core.resolve_goal_state( + registry=effect_registry, + goal_id=safe_effect_goal_id, + project_override=project, + state_file_override=state_file, + ) + ) + effect_input_hash = turn_effect_input_hash( + { + "registry_path": str(registry_path.expanduser().resolve()), + "runtime_root": str(effect_runtime_root), + "goal_id": safe_effect_goal_id, + "project": ( + str(effect_project.expanduser().resolve()) + if effect_project + else None + ), + "state_file": str(effect_state_file.expanduser().resolve()), + "classification": classification, + "recommended_action": recommended_action, + "next_action": ( + state_refresh_core.normalize_next_action_text(next_action) + if next_action + else None + ), + "delivery_batch_scale": ( + require_delivery_batch_scale(delivery_batch_scale).value + if delivery_batch_scale + else None + ), + "delivery_outcome": ( + require_delivery_outcome(delivery_outcome).value + if delivery_outcome + else None + ), + "delivery_workspace_path": ( + str(delivery_workspace_path.expanduser().resolve()) + if delivery_workspace_path + else None + ), + "todo_id": todo_id, + "turn_instance_id": turn_instance_id, + "agent_id": (agent_id or "").strip() or None, + "agent_lane": (agent_lane or "").strip() or None, + "progress_scope": state_refresh_core.normalize_progress_scope( + progress_scope + ), + "autonomous_replan_recorded": autonomous_replan_recorded, + "repair_delta_kinds": state_refresh_core.normalize_repair_delta_kinds( + repair_delta_kinds + ), + "agent_vision_packet": agent_vision_packet, + "merge_agent_vision_patch": merge_agent_vision_patch, + "vision_unchanged_reason": ( + state_refresh_core.normalize_vision_unchanged_reason( + vision_unchanged_reason + ) + ), + "dry_run": dry_run, + "sync_global": sync_global, + } + ) + + def write_once() -> dict[str, Any]: + return state_refresh_core._refresh_state_run( + registry_path=registry_path, + runtime_root_override=runtime_root_override, + goal_id=goal_id, + project=project, + state_file=state_file, + classification=classification, + recommended_action=recommended_action, + next_action=next_action, + delivery_batch_scale=delivery_batch_scale, + delivery_outcome=delivery_outcome, + delivery_workspace_path=delivery_workspace_path, + todo_id=todo_id, + turn_instance_id=turn_instance_id, + agent_id=agent_id, + agent_lane=agent_lane, + progress_scope=progress_scope, + autonomous_replan_recorded=autonomous_replan_recorded, + repair_delta_kinds=repair_delta_kinds, + agent_vision_packet=agent_vision_packet, + merge_agent_vision_patch=merge_agent_vision_patch, + vision_unchanged_reason=vision_unchanged_reason, + dry_run=dry_run, + sync_global=sync_global, + _turn_effect_key=normalized_effect_key, + _effect_input_hash=effect_input_hash, + ) + + if normalized_effect_key is None or dry_run: + return write_once() + + assert effect_runtime_root is not None + assert safe_effect_goal_id is not None + index_path = ( + effect_runtime_root + / "goals" + / safe_effect_goal_id + / "runs" + / "index.jsonl" + ) + with exclusive_file_lock( + index_path, + agent_id=agent_id, + operation="refresh_state_turn_effect", + ): + existing = find_turn_effect_record(index_path, normalized_effect_key) + if existing is not None: + assert effect_input_hash is not None + require_matching_turn_effect(existing, effect_input_hash) + return { + "ok": existing.get("turn_effect_result_ok") is not False, + "dry_run": False, + "appended": False, + "idempotent": True, + "idempotent_replay": True, + "registry": str(registry_path), + "runtime_root": str(effect_runtime_root), + "goal_id": safe_effect_goal_id, + "classification": existing.get("classification"), + "generated_at": existing.get("generated_at"), + "json_path": existing.get("json_path"), + "markdown_path": existing.get("markdown_path"), + "index_path": str(index_path), + "turn_effect_key": normalized_effect_key, + "effect_input_hash": effect_input_hash, + } + return write_once() diff --git a/loopx/control_plane/state_refresh_recording.py b/loopx/control_plane/state_refresh_recording.py new file mode 100644 index 000000000..30c19f107 --- /dev/null +++ b/loopx/control_plane/state_refresh_recording.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + + +def build_state_refresh_output_projections( + *, + record: dict[str, Any], + registry_path: Path, + runtime_root: Path, + project: Path | None, + json_path: Path, + markdown_path: Path, + index_path: Path, + dry_run: bool, + autonomous_replan_recorded_requested: bool, +) -> tuple[dict[str, Any], dict[str, Any]]: + """Project one refresh record into its compact index and CLI response.""" + + record_state = record.get("state") if isinstance(record.get("state"), dict) else {} + record_frontmatter = record_state.get("frontmatter") or {} + index_record = { + field: record[field] + for field in ( + "generated_at", "goal_id", "classification", "recommended_action", + "recommended_action_source", "health_check", + ) + } + index_record.update({ + "json_path": str(json_path), + "markdown_path": str(markdown_path), + "state": { + "sha256_16": record_state.get("sha256_16"), + "frontmatter": {"updated_at": record_frontmatter.get("updated_at")}, + }, + "runtime_projection_route": record["runtime_projection_route"], + }) + for field in ( + "delivery_batch_scale", + "delivery_outcome", + "delivery_workspace", + "settlement_identity", + "turn_instance_id", + "todo_id", + ): + if field in record: + index_record[field] = record[field] + + replan_ack = record.get("autonomous_replan_ack") or {} + if autonomous_replan_recorded_requested: + index_record["autonomous_replan_ack"] = replan_ack + if replan_ack.get("requested_classification"): + index_record["requested_classification"] = replan_ack["requested_classification"] + + agent_vision = record.get("agent_vision") + if isinstance(agent_vision, dict): + index_record["agent_vision"] = { + field: agent_vision.get(field) + for field in ( + "schema_version", "agent_id", "state", "vision_patch", + "todo_delta", "vision_budget", + ) + } + if isinstance(agent_vision.get("path_delta"), dict): + index_record["agent_vision"]["path_delta"] = agent_vision["path_delta"] + + for field in ("vision_checkpoint", "progress_scope", "agent_id", "agent_lane"): + if field in record: + index_record[field] = record[field] + + payload: dict[str, Any] = { + "ok": True, + "dry_run": dry_run, + "appended": not dry_run, + "registry": str(registry_path), + "runtime_root": str(runtime_root), + "project": str(project) if project else None, + } + payload.update({ + field: record.get(field) + for field in ("goal_id", "classification", "progress_scope", "agent_id", "agent_lane") + }) + payload.update({ + "autonomous_replan_recorded": bool(replan_ack.get("recorded")), + "autonomous_replan_recorded_requested": autonomous_replan_recorded_requested, + "repair_delta_contract": replan_ack.get("delta_contract"), + "json_path": str(json_path), + "markdown_path": str(markdown_path), + "index_path": str(index_path), + }) + payload.update({ + field: record.get(field) + for field in ( + "agent_vision", "vision_checkpoint", "recommended_action", + "recommended_action_source", "active_state_next_action_update", + "generated_at", "health_check", + ) + }) + payload.update(record) + return index_record, payload + + +def append_state_refresh_index( + index_path: Path, + index_record: dict[str, Any], + *, + turn_effect_result_ok: bool | None = None, +) -> None: + if turn_effect_result_ok is not None: + index_record["turn_effect_result_ok"] = turn_effect_result_ok + with index_path.open("a", encoding="utf-8") as index_file: + index_file.write(json.dumps(index_record, ensure_ascii=False) + "\n") diff --git a/loopx/control_plane/testing/cli_output_differential.py b/loopx/control_plane/testing/cli_output_differential.py index 3e0ab56ca..c4c62abbe 100644 --- a/loopx/control_plane/testing/cli_output_differential.py +++ b/loopx/control_plane/testing/cli_output_differential.py @@ -7,6 +7,8 @@ from ..quota.turn_envelope import ( ACTION_SIGNATURE_COVERAGE_V0, ACTION_SIGNATURE_COVERAGE_V1, + ACTION_SIGNATURE_COVERAGE_V2, + ACTION_SIGNATURE_COVERAGE_V3, ) @@ -123,11 +125,23 @@ def _action_signature_migration( ) -> str | None: base_coverages = base.get("action_signature_coverages") candidate_coverages = candidate.get("action_signature_coverages") - if base_coverages != [ACTION_SIGNATURE_COVERAGE_V0]: + if not ( + isinstance(base_coverages, list) + and len(base_coverages) == 1 + and isinstance(candidate_coverages, list) + and len(candidate_coverages) == 1 + ): return None - if candidate_coverages != [ACTION_SIGNATURE_COVERAGE_V1]: + transition = (base_coverages[0], candidate_coverages[0]) + allowed_transitions = { + (ACTION_SIGNATURE_COVERAGE_V0, ACTION_SIGNATURE_COVERAGE_V1), + (ACTION_SIGNATURE_COVERAGE_V0, ACTION_SIGNATURE_COVERAGE_V2), + (ACTION_SIGNATURE_COVERAGE_V1, ACTION_SIGNATURE_COVERAGE_V3), + (ACTION_SIGNATURE_COVERAGE_V2, ACTION_SIGNATURE_COVERAGE_V3), + } + if transition not in allowed_transitions: return None - return f"{ACTION_SIGNATURE_COVERAGE_V0} -> {ACTION_SIGNATURE_COVERAGE_V1}" + return f"{transition[0]} -> {transition[1]}" def _compare_row(base: dict[str, Any], candidate: dict[str, Any]) -> dict[str, Any]: diff --git a/loopx/control_plane/turn_driver/__init__.py b/loopx/control_plane/turn_driver/__init__.py index 7c8b5e955..43e06ded4 100644 --- a/loopx/control_plane/turn_driver/__init__.py +++ b/loopx/control_plane/turn_driver/__init__.py @@ -13,6 +13,7 @@ LoopXTurnRoute, build_loopx_turn_plan, selected_turn_todo, + selected_turn_todo_write_scopes, ) from .executor import ( LOOPX_TURN_HOST_REQUEST_SCHEMA_VERSION, @@ -33,10 +34,29 @@ ValidatedTurnReceipt, decide_loop_disposition, ) +from .journal import ( + LocalTurnJournalStore, + TURN_JOURNAL_EVENT_SCHEMA_VERSION, + TURN_JOURNAL_PROJECTION_SCHEMA_VERSION, + TurnJournalError, + TurnJournalStore, + append_turn_event, + load_turn_events, + rebuild_turn_projection, + turn_journal_path, + turn_projection_path, +) +from .lease import ( + TurnFence, + TurnLeaseAuthority, + TurnLeaseController, + hold_turn_lease_heartbeat, +) from .transaction import ( LOOPX_TURN_EXECUTION_SCHEMA_VERSION, LOOPX_TURN_RESULT_SCHEMA_VERSION, LoopXTurnResultKind, + TurnEffectEnvelope, build_loopx_turn_transaction_plan, loopx_turn_execution_committed, loopx_turn_execution_has_durable_effects, @@ -53,29 +73,45 @@ "LOOPX_TURN_SESSION_BINDING_SCHEMA_VERSION", "LOOPX_TURN_TASK_VALIDATION_SCHEMA_VERSION", "LOOP_CONTROLLER_DISPOSITION_SCHEMA_VERSION", + "TURN_JOURNAL_EVENT_SCHEMA_VERSION", + "TURN_JOURNAL_PROJECTION_SCHEMA_VERSION", "VALIDATED_TURN_RECEIPT_SCHEMA_VERSION", "BoundedTurnBudget", "LoopDisposition", "LoopXTurnResultKind", "LoopXTurnRoute", + "LocalTurnJournalStore", + "TurnJournalError", + "TurnJournalStore", + "TurnFence", + "TurnEffectEnvelope", + "TurnLeaseAuthority", + "TurnLeaseController", "ValidatedTurnReceipt", "build_loopx_turn_command_validator", "build_loopx_turn_host_request", "build_loopx_turn_plan", "build_loopx_turn_transaction_plan", + "append_turn_event", "codex_cli_result_schema", "codex_cli_session_binding", "codex_cli_session_id_from_jsonl", "decide_loop_disposition", "load_codex_cli_session", "load_loopx_turn_plan_from_journal", + "load_turn_events", + "hold_turn_lease_heartbeat", "loopx_turn_execution_committed", "loopx_turn_execution_has_durable_effects", "loopx_turn_execution_recovery_required", "normalize_host_argv", "run_codex_cli_host", "run_loopx_turn_once", + "rebuild_turn_projection", "selected_turn_todo", + "selected_turn_todo_write_scopes", + "turn_journal_path", + "turn_projection_path", "validate_loopx_turn_host_result", "validate_loopx_turn_receipt", ] diff --git a/loopx/control_plane/turn_driver/driver.py b/loopx/control_plane/turn_driver/driver.py index f7bcbe1cd..3a6ea39aa 100644 --- a/loopx/control_plane/turn_driver/driver.py +++ b/loopx/control_plane/turn_driver/driver.py @@ -99,6 +99,8 @@ def _typed_route(envelope: Mapping[str, Any]) -> LoopXTurnRoute: def selected_turn_todo(envelope: Mapping[str, Any]) -> dict[str, Any]: """Resolve the todo that owns one Turn across adaptive bundle execution.""" + action = _mapping(envelope.get("action")) + action_selected_todo = _mapping(action.get("selected_todo")) orchestration = _mapping(envelope.get("task_orchestration_contract")) primary_todo_id = str(orchestration.get("primary_todo_id") or "").strip() if ( @@ -106,12 +108,41 @@ def selected_turn_todo(envelope: Mapping[str, Any]) -> dict[str, Any]: and orchestration.get("mode") == "adaptive" and primary_todo_id ): + primary_todo = _mapping(orchestration.get("primary_todo")) + if str(primary_todo.get("todo_id") or "") == primary_todo_id: + return { + **primary_todo, + "source": "task_orchestration_contract.primary_todo", + } + if str(action_selected_todo.get("todo_id") or "") == primary_todo_id: + return action_selected_todo return { "todo_id": primary_todo_id, "source": "task_orchestration_contract.primary_todo_id", } - action = _mapping(envelope.get("action")) - return _mapping(action.get("selected_todo")) + return action_selected_todo + + +def selected_turn_todo_write_scopes( + selected_todo: Mapping[str, Any], +) -> list[str]: + """Return the selected Todo's normalized lease scope projection.""" + + raw_write_scopes = selected_todo.get("required_write_scopes") + if raw_write_scopes is None: + if ( + selected_todo.get("source") + == "task_orchestration_contract.primary_todo_id" + ): + raise ValueError( + "adaptive primary todo must project required_write_scopes" + ) + return [] + if not isinstance(raw_write_scopes, list) or not all( + isinstance(scope, str) for scope in raw_write_scopes + ): + raise ValueError("selected todo required_write_scopes must be a string array") + return list(raw_write_scopes) def _turn_lineage( diff --git a/loopx/control_plane/turn_driver/executor.py b/loopx/control_plane/turn_driver/executor.py index 2a45406c2..474b37a19 100644 --- a/loopx/control_plane/turn_driver/executor.py +++ b/loopx/control_plane/turn_driver/executor.py @@ -3,31 +3,25 @@ from __future__ import annotations import json -import os import re import subprocess -import tempfile from collections.abc import Callable, Mapping, Sequence from pathlib import Path from typing import Any from ...authority import validate_public_safe_text -from ...file_lock import exclusive_file_lock -from ..effect_program import ( - SettlementStepKind, - interpret_turn_result_packet, - settlement_result_payload, -) from ..goals.goal_vision import normalize_goal_vision_packet from ..work_items.delivery_batch_scale import require_delivery_batch_scale from ..work_items.delivery_outcome import require_delivery_outcome from .driver import selected_turn_todo -from .settlement import execute_turn_driver_settlement +from .lease import TurnLeaseAuthority +from .journal import TurnJournalStore from .transaction import ( LOOPX_TURN_EXECUTION_SCHEMA_VERSION, LOOPX_TURN_RESULT_SCHEMA_VERSION, TRANSACTION_PHASES, LoopXTurnResultKind, + TurnEffectEnvelope, build_loopx_turn_transaction_plan, require_loopx_turn_completion_outcome, validate_loopx_turn_receipt, @@ -76,11 +70,18 @@ "summary", } -Writeback = Callable[[dict[str, Any]], dict[str, Any]] -CompletionWriteback = Callable[[dict[str, Any]], dict[str, Any]] -Spend = Callable[[], dict[str, Any]] -Scheduler = Callable[[dict[str, Any]], dict[str, Any]] +Writeback = Callable[[TurnEffectEnvelope, dict[str, Any]], dict[str, Any]] +CompletionWriteback = Callable[ + [TurnEffectEnvelope, dict[str, Any]], + dict[str, Any], +] +Spend = Callable[[TurnEffectEnvelope], dict[str, Any]] +Scheduler = Callable[ + [TurnEffectEnvelope, dict[str, Any]], + dict[str, Any], +] HostRunner = Callable[[Mapping[str, Any]], dict[str, Any]] +FaultInjector = Callable[[str], None] TaskValidator = Callable[ [Mapping[str, Any], Mapping[str, Any]], Mapping[str, Any], @@ -538,56 +539,19 @@ def validate( return validate -def turn_journal_path(runtime_root: Path, *, goal_id: str, turn_key: str) -> Path: - match = TURN_KEY_RE.fullmatch(turn_key) - if not match: - raise ValueError("turn_key must be a sha256 digest") - return runtime_root / "goals" / goal_id / "turns" / f"{match.group('digest')}.json" - - -def _write_journal(path: Path, journal: Mapping[str, Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - descriptor, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) - temporary = Path(temporary_name) - try: - with os.fdopen(descriptor, "w", encoding="utf-8") as handle: - json.dump(journal, handle, ensure_ascii=False, indent=2, sort_keys=True) - handle.write("\n") - os.replace(temporary, path) - finally: - temporary.unlink(missing_ok=True) - - -def _load_journal(path: Path) -> dict[str, Any] | None: - if not path.exists(): - return None - value = json.loads(path.read_text(encoding="utf-8")) - if not isinstance(value, dict) or value.get("schema_version") != LOOPX_TURN_JOURNAL_SCHEMA_VERSION: - raise ValueError("LoopX Turn journal has an unsupported schema") - return value - - def load_loopx_turn_plan_from_journal( runtime_root: Path, *, goal_id: str, turn_key: str, ) -> dict[str, Any]: - path = turn_journal_path(runtime_root, goal_id=goal_id, turn_key=turn_key) - with exclusive_file_lock(path): - journal = _load_journal(path) - if journal is None: - raise ValueError("LoopX Turn resume journal does not exist") - plan = journal.get("plan") - if not isinstance(plan, dict): - raise TypeError("LoopX Turn resume journal does not contain a plan") - transaction = plan.get("transaction") if isinstance(plan.get("transaction"), dict) else {} - if transaction.get("turn_key") != turn_key or journal.get("turn_key") != turn_key: - raise ValueError("LoopX Turn resume journal has mismatched turn lineage") - envelope = plan.get("turn_envelope") if isinstance(plan.get("turn_envelope"), dict) else {} - if envelope.get("goal_id") != goal_id or journal.get("goal_id") != goal_id: - raise ValueError("LoopX Turn resume journal belongs to another goal") - return dict(plan) + from .fenced_runtime import load_loopx_turn_plan_from_fenced_journal + + return load_loopx_turn_plan_from_fenced_journal( + runtime_root, + goal_id=goal_id, + turn_key=turn_key, + ) def _receipt( @@ -753,205 +717,6 @@ def _execution_payload( } -def _host_result_stage( - plan: Mapping[str, Any], - request: Mapping[str, Any], - *, - host_runner: HostRunner | None, - argv: Sequence[str] | None, - completion_writeback_configured: bool, - project: Path, - timeout_seconds: float, - journal: dict[str, Any], - journal_path: Path, - effects: dict[str, bool], -) -> tuple[dict[str, Any] | None, list[str], dict[str, Any] | None]: - completed_phases = list(journal.get("completed_phases") or []) - result = ( - journal.get("host_result") - if isinstance(journal.get("host_result"), dict) - else None - ) - if "typed_result" not in completed_phases: - host_observation = ( - _run_host_runner(request, runner=host_runner) - if host_runner is not None - else _run_host( - request, - argv=argv or [], - project=project, - timeout_seconds=timeout_seconds, - ) - ) - effects["host_invoked"] = True - if not host_observation.get("ok"): - failure = _host_failure( - plan, - kind=LoopXTurnResultKind.HOST_FAILURE, - completed_phases=[], - failed_phase="host_execute", - reason=str(host_observation.get("reason") or "host execution failed"), - ) - journal.update( - status="failed", - reason=failure["reason"], - receipt=failure["receipt"], - completed_phases=[], - result_kind=LoopXTurnResultKind.HOST_FAILURE.value, - ) - _write_journal(journal_path, journal) - return ( - None, - [], - _execution_payload( - plan, - journal, - execute=True, - replayed=False, - effects=effects, - ), - ) - result = dict(host_observation["value"]) - completed_phases = list(TRANSACTION_PHASES[:2]) - - validation = validate_loopx_turn_host_result( - plan, - result or {}, - completion_writeback_configured=completion_writeback_configured, - ) - if not validation.get("ok"): - failure = _host_failure( - plan, - kind=LoopXTurnResultKind.VALIDATION_FAILED, - completed_phases=list(TRANSACTION_PHASES[:2]), - failed_phase="validation", - reason="; ".join( - validation.get("errors") or ["host result validation failed"] - ), - ) - journal.update( - status="failed", - reason=failure["reason"], - receipt=failure["receipt"], - completed_phases=list(TRANSACTION_PHASES[:2]), - result_kind=LoopXTurnResultKind.VALIDATION_FAILED.value, - validation_stage="host_result_contract", - ) - _write_journal(journal_path, journal) - return ( - None, - list(TRANSACTION_PHASES[:2]), - _execution_payload( - plan, - journal, - execute=True, - replayed=False, - effects=effects, - ), - ) - - normalized = dict(validation["result"]) - if len(completed_phases) < 2: - completed_phases = list(TRANSACTION_PHASES[:2]) - journal.update( - host_result=normalized, - result_kind=normalized.get("result_kind"), - completed_phases=completed_phases, - ) - _write_journal(journal_path, journal) - return normalized, completed_phases, None - - -def _task_validation_stage( - plan: Mapping[str, Any], - result: Mapping[str, Any], - *, - task_validator: TaskValidator | None, - completed_phases: list[str], - journal: dict[str, Any], - journal_path: Path, - effects: dict[str, bool], -) -> tuple[list[str], dict[str, Any] | None]: - turn = interpret_turn_result_packet(result) - kind = LoopXTurnResultKind(turn.observation.decision) - if kind in STOP_HOST_RESULT_KINDS: - completed_phases = list(TRANSACTION_PHASES[:3]) - journal.update( - status="stopped", - completed_phases=completed_phases, - task_validation=_task_validation_receipt( - status="not_required", - validator_kind="stop_result", - summary="task validation is not required for a typed stop result", - ), - receipt=_receipt(plan, result, completed_phases=completed_phases), - scheduler={"disposition": "not_applicable"}, - ) - _write_journal(journal_path, journal) - return completed_phases, _execution_payload( - plan, - journal, - execute=True, - replayed=False, - effects=effects, - ) - - stored_task_validation = ( - journal.get("task_validation") - if isinstance(journal.get("task_validation"), dict) - else None - ) - task_validation = ( - stored_task_validation - if "validation" in completed_phases - and stored_task_validation is not None - and stored_task_validation.get("ok") is True - else _run_task_validator( - plan, - result, - validator=task_validator, - ) - ) - journal["task_validation"] = task_validation - if not task_validation.get("ok"): - reason = str( - task_validation.get("summary") or "independent task validation failed" - ) - failure = _host_failure( - plan, - kind=LoopXTurnResultKind.VALIDATION_FAILED, - completed_phases=list(TRANSACTION_PHASES[:2]), - failed_phase="validation", - reason=reason, - ) - journal.update( - status="failed", - reason=reason, - receipt=failure["receipt"], - completed_phases=list(TRANSACTION_PHASES[:2]), - result_kind=LoopXTurnResultKind.VALIDATION_FAILED.value, - validation_stage="task_postcondition", - ) - _write_journal(journal_path, journal) - return list(TRANSACTION_PHASES[:2]), _execution_payload( - plan, - journal, - execute=True, - replayed=False, - effects=effects, - ) - - if "validation" not in completed_phases: - completed_phases = list(TRANSACTION_PHASES[:3]) - journal.update( - result_kind=result.get("result_kind"), - completed_phases=completed_phases, - validation_stage="task_postcondition", - ) - _write_journal(journal_path, journal) - return completed_phases, None - - def _completion_writeback_outcome( payload: Mapping[str, Any], *, @@ -1014,159 +779,6 @@ def _ensure_turn_settlement_plan( transaction_plan["settlement_plan"] = settlement_plan -def _typed_settlement_stage( - plan: Mapping[str, Any], - result: dict[str, Any], - *, - completed_phases: list[str], - journal: dict[str, Any], - journal_path: Path, - effects: dict[str, bool], - writeback: Writeback, - completion_writeback: CompletionWriteback | None, - spend: Spend, - scheduler: Scheduler, -) -> dict[str, Any]: - transaction_plan = ( - plan.get("transaction") - if isinstance(plan.get("transaction"), Mapping) - else {} - ) - _ensure_turn_settlement_plan(plan, transaction_plan) - - def writeback_effect() -> Mapping[str, Any]: - if result.get("result_kind") == LoopXTurnResultKind.VALIDATED_COMPLETION.value: - if completion_writeback is None: - raise ValueError("validated_completion requires a todo lifecycle adapter") - callback_payload = completion_writeback(result) - completion_outcome = _completion_writeback_outcome( - callback_payload, - plan=plan, - ) - if completion_outcome is None: - return { - "ok": False, - "appended": False, - "reason": "todo lifecycle adapter returned an invalid completion outcome", - } - return { - **callback_payload, - "completion": completion_outcome, - } - return writeback(result) - - def checkpoint( - step_kind: SettlementStepKind, - payload: Mapping[str, Any], - phases: tuple[str, ...], - ) -> None: - if step_kind is SettlementStepKind.DURABLE_WRITEBACK: - effects["state_written"] = True - journal["writeback"] = { - **_compact_callback(payload), - **( - {"completion": payload["completion"]} - if isinstance(payload.get("completion"), dict) - else {} - ), - } - elif step_kind is SettlementStepKind.QUOTA_SPEND: - effects["quota_spent"] = True - journal["quota_spend"] = _compact_callback(payload) - journal["completed_phases"] = list(phases) - _write_journal(journal_path, journal) - - settlement_result = execute_turn_driver_settlement( - transaction_plan, - transaction_phases=TRANSACTION_PHASES, - completed_phases=completed_phases, - writeback_payload=( - journal.get("writeback") - if isinstance(journal.get("writeback"), Mapping) - else None - ), - quota_spend_payload=( - journal.get("quota_spend") - if isinstance(journal.get("quota_spend"), Mapping) - else None - ), - writeback=writeback_effect, - spend=spend, - checkpoint=checkpoint, - ) - journal["settlement_result"] = settlement_result_payload(settlement_result) - if settlement_result.failure is not None: - failure_step = settlement_result.failure.step_kind - result_kind = ( - LoopXTurnResultKind.VALIDATION_FAILED - if failure_step is SettlementStepKind.VALIDATION - else LoopXTurnResultKind.WRITEBACK_FAILED - if failure_step is SettlementStepKind.DURABLE_WRITEBACK - else LoopXTurnResultKind.QUOTA_SPEND_FAILED - ) - completed_phases = list(journal.get("completed_phases") or completed_phases) - failure = _host_failure( - plan, - kind=result_kind, - completed_phases=completed_phases, - failed_phase=failure_step.value, - reason=settlement_result.failure.reason, - ) - journal.update( - status="failed", - result_kind=result_kind.value, - reason=failure["reason"], - receipt=failure["receipt"], - ) - _write_journal(journal_path, journal) - return _execution_payload( - plan, - journal, - execute=True, - replayed=False, - effects=effects, - ) - - settlement_state = settlement_result.value - if settlement_state is None or settlement_state.quota_spend is None: - raise ValueError("typed Turn settlement completed without a quota spend receipt") - completed_phases = list(settlement_state.completed_phases) - spend_payload = dict(settlement_state.quota_spend) - _write_journal(journal_path, journal) - - scheduler_payload = scheduler(spend_payload) - journal["scheduler"] = scheduler_payload - if scheduler_payload.get("completed") is not True: - journal.update( - status="scheduler_action_required", - receipt=_receipt(plan, result, completed_phases=completed_phases), - ) - _write_journal(journal_path, journal) - return _execution_payload( - plan, - journal, - execute=True, - replayed=False, - effects=effects, - ) - - completed_phases = list(TRANSACTION_PHASES) - effects["scheduler_acknowledged"] = bool(scheduler_payload.get("acknowledged")) - journal.update( - status="committed", - completed_phases=completed_phases, - receipt=_receipt(plan, result, completed_phases=completed_phases), - ) - _write_journal(journal_path, journal) - return _execution_payload( - plan, - journal, - execute=True, - replayed=False, - effects=effects, - ) - - def run_loopx_turn_once( plan: Mapping[str, Any], *, @@ -1183,127 +795,28 @@ def run_loopx_turn_once( completion_writeback: CompletionWriteback | None = None, spend: Spend | None = None, scheduler: Scheduler | None = None, + lease_controller: TurnLeaseAuthority | None = None, + journal_store: TurnJournalStore | None = None, + fault_injector: FaultInjector | None = None, ) -> dict[str, Any]: - if host_runner is not None and host_argv is not None: - raise ValueError("run-once accepts either host_argv or host_runner, not both") - if host_runner is None: - argv = normalize_host_argv(host_argv or []) - host_projection = {"executable": Path(argv[0]).name, "argv_count": len(argv)} - else: - argv = None - planned_host = plan.get("host") if isinstance(plan.get("host"), dict) else {} - host_projection = { - "executable": "built-in", - "kind": str(planned_host.get("kind") or "codex-cli"), - } - request = build_loopx_turn_host_request(plan) - empty_effects = { - "host_invoked": False, - "state_written": False, - "quota_spent": False, - "scheduler_acknowledged": False, - } - if not execute: - preview = { - "schema_version": LOOPX_TURN_JOURNAL_SCHEMA_VERSION, - "status": "preview", - "host": host_projection, - "result_kind": None, - "receipt": None, - "scheduler": {"disposition": "not_evaluated"}, - } - return _execution_payload( - plan, - preview, - execute=False, - replayed=False, - effects=empty_effects, - ) - if writeback is None or spend is None or scheduler is None: - raise ValueError("executing run-once requires writeback, spend, and scheduler callbacks") - - turn_key = str(request["turn_key"]) - journal_path = turn_journal_path(runtime_root, goal_id=goal_id, turn_key=turn_key) - with exclusive_file_lock(journal_path): - journal = _load_journal(journal_path) - if journal and ( - journal.get("status") in {"committed", "stopped"} - or journal.get("status") == "failed" and not retry_failed - ): - return _execution_payload( - plan, - journal, - execute=True, - replayed=True, - effects=empty_effects, - ) - if journal and journal.get("status") == "failed": - receipt = journal.get("receipt") if isinstance(journal.get("receipt"), dict) else {} - if receipt.get("failed_phase") == "validation": - if journal.get("validation_stage") != "task_postcondition": - journal.pop("host_result", None) - journal.pop("result_kind", None) - journal["completed_phases"] = ( - list(TRANSACTION_PHASES[:2]) - if isinstance(journal.get("host_result"), dict) - else [] - ) - journal.pop("task_validation", None) - journal.pop("validation_stage", None) - journal.pop("reason", None) - journal.pop("receipt", None) - journal["status"] = "in_progress" - _write_journal(journal_path, journal) - if journal is None: - journal = { - "schema_version": LOOPX_TURN_JOURNAL_SCHEMA_VERSION, - "turn_key": turn_key, - "goal_id": goal_id, - "status": "in_progress", - "host": host_projection, - "completed_phases": [], - "plan": dict(plan), - } - _write_journal(journal_path, journal) + from .fenced_runtime import run_fenced_loopx_turn_once - effects = dict(empty_effects) - result, completed_phases, terminal = _host_result_stage( - plan, - request, - host_runner=host_runner, - argv=argv, - completion_writeback_configured=completion_writeback is not None, - project=project, - timeout_seconds=timeout_seconds, - journal=journal, - journal_path=journal_path, - effects=effects, - ) - if terminal is not None: - return terminal - assert result is not None - - completed_phases, terminal = _task_validation_stage( - plan, - result, - task_validator=task_validator, - completed_phases=completed_phases, - journal=journal, - journal_path=journal_path, - effects=effects, - ) - if terminal is not None: - return terminal - - return _typed_settlement_stage( - plan, - result, - completed_phases=completed_phases, - journal=journal, - journal_path=journal_path, - effects=effects, - writeback=writeback, - completion_writeback=completion_writeback, - spend=spend, - scheduler=scheduler, - ) + return run_fenced_loopx_turn_once( + plan, + host_argv=host_argv, + host_runner=host_runner, + project=project, + runtime_root=runtime_root, + goal_id=goal_id, + timeout_seconds=timeout_seconds, + execute=execute, + retry_failed=retry_failed, + task_validator=task_validator, + writeback=writeback, + completion_writeback=completion_writeback, + spend=spend, + scheduler=scheduler, + lease_controller=lease_controller, + journal_store=journal_store, + fault_injector=fault_injector, + ) diff --git a/loopx/control_plane/turn_driver/fenced_runtime.py b/loopx/control_plane/turn_driver/fenced_runtime.py new file mode 100644 index 000000000..edd0067d5 --- /dev/null +++ b/loopx/control_plane/turn_driver/fenced_runtime.py @@ -0,0 +1,1070 @@ +"""Lease-fenced, append-only runtime for one bounded LoopX Turn.""" + +from __future__ import annotations + +import inspect +from collections.abc import Callable, Mapping, Sequence +from pathlib import Path +from typing import Any + +from ..effect_program import SettlementStepKind, interpret_turn_result_packet, settlement_result_payload +from ..work_items.task_lease import TaskLeaseError +from .executor import ( + LOOPX_TURN_JOURNAL_SCHEMA_VERSION, + STOP_HOST_RESULT_KINDS, + CompletionWriteback, + FaultInjector, + HostRunner, + Scheduler, + Spend, + TaskValidator, + Writeback, + _compact_callback, + _completion_writeback_outcome, + _ensure_turn_settlement_plan, + _execution_payload, + _host_failure, + _receipt, + _run_host, + _run_host_runner, + _run_task_validator, + _task_validation_receipt, + build_loopx_turn_host_request, + normalize_host_argv, + validate_loopx_turn_host_result, +) +from .journal import LocalTurnJournalStore, TurnJournalError, TurnJournalStore +from .lease import TurnFence, TurnLeaseAuthority +from .settlement import execute_turn_driver_settlement +from .transaction import TRANSACTION_PHASES, LoopXTurnResultKind, TurnEffectEnvelope + + +def _state_from_events(events: Sequence[Mapping[str, Any]]) -> dict[str, Any] | None: + state: dict[str, Any] | None = None + for event in events: + payload = event.get("payload") + if not isinstance(payload, Mapping): + continue + candidate = payload.get("state") + if not isinstance(candidate, Mapping): + continue + if candidate.get("schema_version") != LOOPX_TURN_JOURNAL_SCHEMA_VERSION: + raise TurnJournalError("Turn state snapshot has an unsupported schema") + state = dict(candidate) + return state + + +def load_turn_state( + runtime_root: Path, + *, + goal_id: str, + turn_key: str, + journal_store: TurnJournalStore | None = None, +) -> dict[str, Any] | None: + store = journal_store or LocalTurnJournalStore() + events = store.load_events( + runtime_root=runtime_root, + goal_id=goal_id, + turn_key=turn_key, + ) + state = _state_from_events(events) + if state is None: + return None + if state.get("goal_id") != goal_id or state.get("turn_key") != turn_key: + raise TurnJournalError("Turn state snapshot has mismatched lineage") + return state + + +def load_loopx_turn_plan_from_fenced_journal( + runtime_root: Path, + *, + goal_id: str, + turn_key: str, +) -> dict[str, Any]: + state = load_turn_state( + runtime_root, + goal_id=goal_id, + turn_key=turn_key, + ) + if state is None: + raise ValueError("LoopX Turn resume journal does not exist") + plan = state.get("plan") + if not isinstance(plan, dict): + raise TypeError("LoopX Turn resume journal does not contain a plan") + transaction = plan.get("transaction") + if not isinstance(transaction, dict) or transaction.get("turn_key") != turn_key: + raise ValueError("LoopX Turn resume journal has mismatched turn lineage") + envelope = plan.get("turn_envelope") + if not isinstance(envelope, dict) or envelope.get("goal_id") != goal_id: + raise ValueError("LoopX Turn resume journal belongs to another goal") + return dict(plan) + + +def _append_state( + *, + runtime_root: Path, + goal_id: str, + turn_key: str, + event_type: str, + phase_key: str, + phase: str, + fence: TurnFence, + state: dict[str, Any], + journal_store: TurnJournalStore, +) -> None: + state["fencing_token"] = fence.token + journal_store.append_event( + runtime_root=runtime_root, + goal_id=goal_id, + turn_key=turn_key, + event_type=event_type, + phase_key=phase_key, + fencing=fence, + payload={"phase": phase, "state": dict(state)}, + ) + + +def _unique_phase_key( + runtime_root: Path, + *, + goal_id: str, + turn_key: str, + prefix: str, + journal_store: TurnJournalStore, +) -> str: + event_count = len( + journal_store.load_events( + runtime_root=runtime_root, + goal_id=goal_id, + turn_key=turn_key, + ) + ) + return f"{turn_key}:{prefix}:event:{event_count:06d}" + + +def _record_intent( + *, + runtime_root: Path, + goal_id: str, + turn_key: str, + envelope: TurnEffectEnvelope, + fence: TurnFence, + journal_store: TurnJournalStore, +) -> None: + phase_key = f"{envelope.phase_key}:intent" + expected_payload = { + "phase": envelope.phase, + "phase_key": envelope.phase_key, + } + for event in journal_store.load_events( + runtime_root=runtime_root, + goal_id=goal_id, + turn_key=turn_key, + ): + if event.get("phase_key") != phase_key: + continue + if event.get("event_type") != "phase_intent" or event.get( + "payload" + ) != expected_payload: + raise TurnJournalError("turn journal phase intent conflict") + return + journal_store.append_event( + runtime_root=runtime_root, + goal_id=goal_id, + turn_key=turn_key, + event_type="phase_intent", + phase_key=phase_key, + fencing=fence, + payload=expected_payload, + ) + + +def _call_compatible( + callback: Callable[..., dict[str, Any]], + current_args: tuple[Any, ...], + legacy_args: tuple[Any, ...], +) -> dict[str, Any]: + try: + callback_signature = inspect.signature(callback) + except (TypeError, ValueError): + return callback(*current_args) + try: + callback_signature.bind(*current_args) + except TypeError: + callback_signature.bind(*legacy_args) + return callback(*legacy_args) + return callback(*current_args) + + +def _fault(fault_injector: FaultInjector | None, phase: str) -> None: + if fault_injector is not None: + fault_injector(phase) + + +def _public_payload( + plan: Mapping[str, Any], + state: Mapping[str, Any], + *, + execute: bool, + replayed: bool, + effects: Mapping[str, bool], + fence: TurnFence | None, +) -> dict[str, Any]: + payload = _execution_payload( + plan, + state, + execute=execute, + replayed=replayed, + effects=effects, + ) + fencing_token = fence.token if fence is not None else state.get("fencing_token") + if fencing_token: + payload["fencing_token"] = fencing_token + if state.get("reason_code"): + payload["reason_code"] = state["reason_code"] + return payload + + +def _failed_closed_payload( + plan: Mapping[str, Any], + state: Mapping[str, Any] | None, + *, + reason: str, + reason_code: str, + effects: Mapping[str, bool], + fence: TurnFence | None, +) -> dict[str, Any]: + failed_state = dict(state or {}) + transaction = plan.get("transaction") + turn_key = str(transaction.get("turn_key") if isinstance(transaction, Mapping) else "") + completed = list(failed_state.get("completed_phases") or []) + failed_phase = ( + TRANSACTION_PHASES[len(completed)] + if len(completed) < len(TRANSACTION_PHASES) + else None + ) + result = {"turn_key": turn_key, "result_kind": LoopXTurnResultKind.FAILED_CLOSED.value} + failed_state.update( + schema_version=LOOPX_TURN_JOURNAL_SCHEMA_VERSION, + turn_key=turn_key, + status="failed_closed", + result_kind=LoopXTurnResultKind.FAILED_CLOSED.value, + completed_phases=completed, + reason=reason, + reason_code=reason_code, + receipt=_receipt( + plan, + result, + completed_phases=completed, + failure_kind=LoopXTurnResultKind.FAILED_CLOSED, + failed_phase=failed_phase, + ), + ) + return _public_payload( + plan, + failed_state, + execute=True, + replayed=False, + effects=effects, + fence=fence, + ) + + +def _record_failure( + plan: Mapping[str, Any], + state: dict[str, Any], + *, + kind: LoopXTurnResultKind, + failed_phase: str, + reason: str, + runtime_root: Path, + goal_id: str, + turn_key: str, + fence: TurnFence, + journal_store: TurnJournalStore, +) -> None: + completed = list(state.get("completed_phases") or []) + failure = _host_failure( + plan, + kind=kind, + completed_phases=completed, + failed_phase=failed_phase, + reason=reason, + ) + state.update( + status="failed", + result_kind=kind.value, + reason=reason, + receipt=failure["receipt"], + ) + _append_state( + runtime_root=runtime_root, + goal_id=goal_id, + turn_key=turn_key, + event_type="phase_failed", + phase_key=_unique_phase_key( + runtime_root, + goal_id=goal_id, + turn_key=turn_key, + prefix=f"{failed_phase}:failed", + journal_store=journal_store, + ), + phase=failed_phase, + fence=fence, + state=state, + journal_store=journal_store, + ) + + +def _prepare_retry( + state: dict[str, Any], + *, + runtime_root: Path, + goal_id: str, + turn_key: str, + fence: TurnFence, + journal_store: TurnJournalStore, +) -> None: + receipt = state.get("receipt") if isinstance(state.get("receipt"), dict) else {} + if receipt.get("failed_phase") == "validation": + if state.get("validation_stage") != "task_postcondition": + state.pop("host_result", None) + state.pop("result_kind", None) + state["completed_phases"] = ( + list(TRANSACTION_PHASES[:2]) + if isinstance(state.get("host_result"), dict) + else [] + ) + state.pop("task_validation", None) + state.pop("validation_stage", None) + state.pop("reason", None) + state.pop("reason_code", None) + state.pop("receipt", None) + state["status"] = "in_progress" + _append_state( + runtime_root=runtime_root, + goal_id=goal_id, + turn_key=turn_key, + event_type="turn_retry", + phase_key=_unique_phase_key( + runtime_root, + goal_id=goal_id, + turn_key=turn_key, + prefix="retry", + journal_store=journal_store, + ), + phase="retry", + fence=fence, + state=state, + journal_store=journal_store, + ) + + +def _guarded_effect( + *, + controller: TurnLeaseAuthority, + latest: Callable[[], TurnFence], + runtime_root: Path, + goal_id: str, + turn_key: str, + phase: str, + invoke: Callable[[TurnEffectEnvelope], dict[str, Any]], + journal_store: TurnJournalStore, +) -> dict[str, Any]: + fence = latest() + envelope = TurnEffectEnvelope( + turn_key=turn_key, + phase=phase, + phase_key=f"{turn_key}:{phase}", + fencing_token=fence.token, + ) + controller.require_current(fence) + _record_intent( + runtime_root=runtime_root, + goal_id=goal_id, + turn_key=turn_key, + envelope=envelope, + fence=fence, + journal_store=journal_store, + ) + fence = latest() + controller.require_current(fence) + with controller.effect_guard(fence): + payload = invoke(envelope) + controller.require_current(latest()) + return payload + + +def _execute_fenced( + plan: Mapping[str, Any], + *, + request: Mapping[str, Any], + host_projection: Mapping[str, Any], + argv: Sequence[str] | None, + host_runner: HostRunner | None, + project: Path, + runtime_root: Path, + goal_id: str, + timeout_seconds: float, + retry_failed: bool, + task_validator: TaskValidator | None, + writeback: Writeback, + completion_writeback: CompletionWriteback | None, + spend: Spend, + scheduler: Scheduler, + controller: TurnLeaseAuthority, + fence: TurnFence, + latest: Callable[[], TurnFence], + fault_injector: FaultInjector | None, + effects: dict[str, bool], + journal_store: TurnJournalStore, +) -> dict[str, Any]: + turn_key = str(request["turn_key"]) + state = load_turn_state( + runtime_root, + goal_id=goal_id, + turn_key=turn_key, + journal_store=journal_store, + ) + if state is not None and ( + state.get("status") in {"committed", "stopped"} + or state.get("status") == "failed" and not retry_failed + ): + return _public_payload( + plan, + state, + execute=True, + replayed=True, + effects=effects, + fence=fence, + ) + if state is None: + state = { + "schema_version": LOOPX_TURN_JOURNAL_SCHEMA_VERSION, + "turn_key": turn_key, + "goal_id": goal_id, + "status": "in_progress", + "host": dict(host_projection), + "completed_phases": [], + "plan": dict(plan), + } + if state.get("fencing_token") != fence.token: + _append_state( + runtime_root=runtime_root, + goal_id=goal_id, + turn_key=turn_key, + event_type="turn_owned", + phase_key=f"{turn_key}:ownership:{fence.generation}", + phase="ownership", + fence=fence, + state=state, + journal_store=journal_store, + ) + if state.get("status") == "failed" and retry_failed: + _prepare_retry( + state, + runtime_root=runtime_root, + goal_id=goal_id, + turn_key=turn_key, + fence=latest(), + journal_store=journal_store, + ) + + completed = list(state.get("completed_phases") or []) + result = state.get("host_result") if isinstance(state.get("host_result"), dict) else None + if "typed_result" not in completed: + observation = ( + _run_host_runner(request, runner=host_runner) + if host_runner is not None + else _run_host( + request, + argv=argv or [], + project=project, + timeout_seconds=timeout_seconds, + ) + ) + effects["host_invoked"] = True + if not observation.get("ok"): + _record_failure( + plan, + state, + kind=LoopXTurnResultKind.HOST_FAILURE, + failed_phase="host_execute", + reason=str(observation.get("reason") or "host execution failed"), + runtime_root=runtime_root, + goal_id=goal_id, + turn_key=turn_key, + fence=latest(), + journal_store=journal_store, + ) + return _public_payload( + plan, + state, + execute=True, + replayed=False, + effects=effects, + fence=latest(), + ) + candidate = dict(observation["value"]) + validation = validate_loopx_turn_host_result( + plan, + candidate, + completion_writeback_configured=completion_writeback is not None, + ) + if not validation.get("ok"): + state["completed_phases"] = list(TRANSACTION_PHASES[:2]) + state["validation_stage"] = "host_result_contract" + _record_failure( + plan, + state, + kind=LoopXTurnResultKind.VALIDATION_FAILED, + failed_phase="validation", + reason="; ".join( + validation.get("errors") or ["host result validation failed"] + ), + runtime_root=runtime_root, + goal_id=goal_id, + turn_key=turn_key, + fence=latest(), + journal_store=journal_store, + ) + return _public_payload( + plan, + state, + execute=True, + replayed=False, + effects=effects, + fence=latest(), + ) + result = dict(validation["result"]) + completed = list(TRANSACTION_PHASES[:2]) + state.update( + host_result=result, + result_kind=result.get("result_kind"), + completed_phases=completed, + status="in_progress", + ) + _append_state( + runtime_root=runtime_root, + goal_id=goal_id, + turn_key=turn_key, + event_type="phase_completed", + phase_key=f"{turn_key}:typed_result:completed", + phase="typed_result", + fence=latest(), + state=state, + journal_store=journal_store, + ) + _fault(fault_injector, "after_host") + assert result is not None + + turn = interpret_turn_result_packet(result) + kind = LoopXTurnResultKind(turn.observation.decision) + completed = list(state.get("completed_phases") or completed) + if kind in STOP_HOST_RESULT_KINDS: + completed = list(TRANSACTION_PHASES[:3]) + state.update( + status="stopped", + completed_phases=completed, + task_validation=_task_validation_receipt( + status="not_required", + validator_kind="stop_result", + summary="task validation is not required for a typed stop result", + ), + receipt=_receipt(plan, result, completed_phases=completed), + scheduler={"disposition": "not_applicable"}, + ) + _append_state( + runtime_root=runtime_root, + goal_id=goal_id, + turn_key=turn_key, + event_type="phase_completed", + phase_key=f"{turn_key}:validation:completed", + phase="validation", + fence=latest(), + state=state, + journal_store=journal_store, + ) + _fault(fault_injector, "after_validation") + return _public_payload( + plan, + state, + execute=True, + replayed=False, + effects=effects, + fence=latest(), + ) + + stored_validation = ( + state.get("task_validation") + if isinstance(state.get("task_validation"), dict) + else None + ) + task_validation = ( + stored_validation + if "validation" in completed + and stored_validation is not None + and stored_validation.get("ok") is True + else _run_task_validator(plan, result, validator=task_validator) + ) + state["task_validation"] = task_validation + if not task_validation.get("ok"): + state["completed_phases"] = list(TRANSACTION_PHASES[:2]) + state["validation_stage"] = "task_postcondition" + _record_failure( + plan, + state, + kind=LoopXTurnResultKind.VALIDATION_FAILED, + failed_phase="validation", + reason=str( + task_validation.get("summary") + or "independent task validation failed" + ), + runtime_root=runtime_root, + goal_id=goal_id, + turn_key=turn_key, + fence=latest(), + journal_store=journal_store, + ) + return _public_payload( + plan, + state, + execute=True, + replayed=False, + effects=effects, + fence=latest(), + ) + if "validation" not in completed: + completed = list(TRANSACTION_PHASES[:3]) + state.update( + result_kind=result.get("result_kind"), + completed_phases=completed, + validation_stage="task_postcondition", + ) + _append_state( + runtime_root=runtime_root, + goal_id=goal_id, + turn_key=turn_key, + event_type="phase_completed", + phase_key=f"{turn_key}:validation:completed", + phase="validation", + fence=latest(), + state=state, + journal_store=journal_store, + ) + _fault(fault_injector, "after_validation") + + transaction_plan = ( + plan.get("transaction") if isinstance(plan.get("transaction"), dict) else {} + ) + _ensure_turn_settlement_plan(plan, transaction_plan) + + def writeback_effect() -> Mapping[str, Any]: + def invoke(envelope: TurnEffectEnvelope) -> dict[str, Any]: + callback = ( + completion_writeback + if result.get("result_kind") + == LoopXTurnResultKind.VALIDATED_COMPLETION.value + else writeback + ) + if callback is None: + raise ValueError( + "validated_completion requires a todo lifecycle adapter" + ) + callback_payload = _call_compatible( + callback, + (envelope, result), + (result,), + ) + if callback is completion_writeback: + completion_outcome = _completion_writeback_outcome( + callback_payload, + plan=plan, + ) + if completion_outcome is None: + return { + "ok": False, + "appended": False, + "reason": "todo lifecycle adapter returned an invalid completion outcome", + } + return {**callback_payload, "completion": completion_outcome} + return callback_payload + + return _guarded_effect( + controller=controller, + latest=latest, + runtime_root=runtime_root, + goal_id=goal_id, + turn_key=turn_key, + phase="durable_writeback", + invoke=invoke, + journal_store=journal_store, + ) + + def spend_effect() -> Mapping[str, Any]: + return _guarded_effect( + controller=controller, + latest=latest, + runtime_root=runtime_root, + goal_id=goal_id, + turn_key=turn_key, + phase="quota_spend", + invoke=lambda envelope: _call_compatible( + spend, + (envelope,), + (), + ), + journal_store=journal_store, + ) + + def checkpoint( + step_kind: SettlementStepKind, + payload: Mapping[str, Any], + phases: tuple[str, ...], + ) -> None: + if step_kind is SettlementStepKind.DURABLE_WRITEBACK: + effects["state_written"] = True + state["writeback"] = { + **_compact_callback(payload), + **( + {"completion": payload["completion"]} + if isinstance(payload.get("completion"), dict) + else {} + ), + } + else: + effects["quota_spent"] = True + state["quota_spend"] = _compact_callback(payload) + state["completed_phases"] = list(phases) + _append_state( + runtime_root=runtime_root, + goal_id=goal_id, + turn_key=turn_key, + event_type="phase_completed", + phase_key=f"{turn_key}:{step_kind.value}:completed", + phase=step_kind.value, + fence=latest(), + state=state, + journal_store=journal_store, + ) + fault_phase = ( + "after_writeback" + if step_kind is SettlementStepKind.DURABLE_WRITEBACK + else "after_spend" + ) + _fault(fault_injector, fault_phase) + + completed = list(state.get("completed_phases") or completed) + settlement_result = execute_turn_driver_settlement( + transaction_plan, + transaction_phases=TRANSACTION_PHASES, + completed_phases=completed, + writeback_payload=( + state.get("writeback") + if isinstance(state.get("writeback"), Mapping) + else None + ), + quota_spend_payload=( + state.get("quota_spend") + if isinstance(state.get("quota_spend"), Mapping) + else None + ), + writeback=writeback_effect, + spend=spend_effect, + checkpoint=checkpoint, + ) + state["settlement_result"] = settlement_result_payload(settlement_result) + if settlement_result.failure is not None: + failure_step = settlement_result.failure.step_kind + result_kind = ( + LoopXTurnResultKind.VALIDATION_FAILED + if failure_step is SettlementStepKind.VALIDATION + else LoopXTurnResultKind.WRITEBACK_FAILED + if failure_step is SettlementStepKind.DURABLE_WRITEBACK + else LoopXTurnResultKind.QUOTA_SPEND_FAILED + ) + _record_failure( + plan, + state, + kind=result_kind, + failed_phase=failure_step.value, + reason=settlement_result.failure.reason, + runtime_root=runtime_root, + goal_id=goal_id, + turn_key=turn_key, + fence=latest(), + journal_store=journal_store, + ) + return _public_payload( + plan, + state, + execute=True, + replayed=False, + effects=effects, + fence=latest(), + ) + settlement_state = settlement_result.value + if settlement_state is None or settlement_state.quota_spend is None: + raise ValueError("typed Turn settlement completed without a quota spend receipt") + completed = list(settlement_state.completed_phases) + state["completed_phases"] = completed + if not state.get("settlement_recorded"): + state["settlement_recorded"] = True + _append_state( + runtime_root=runtime_root, + goal_id=goal_id, + turn_key=turn_key, + event_type="settlement_completed", + phase_key=f"{turn_key}:settlement:completed", + phase="quota_spend", + fence=latest(), + state=state, + journal_store=journal_store, + ) + + scheduler_payload = ( + state.get("scheduler") + if "scheduler_apply" in completed and isinstance(state.get("scheduler"), dict) + else None + ) + if scheduler_payload is None: + scheduler_payload = _guarded_effect( + controller=controller, + latest=latest, + runtime_root=runtime_root, + goal_id=goal_id, + turn_key=turn_key, + phase="scheduler_apply", + invoke=lambda envelope: _call_compatible( + scheduler, + (envelope, dict(settlement_state.quota_spend)), + (dict(settlement_state.quota_spend),), + ), + journal_store=journal_store, + ) + state["scheduler"] = scheduler_payload + if scheduler_payload.get("completed") is not True: + state.update( + status="scheduler_action_required", + receipt=_receipt(plan, result, completed_phases=completed), + ) + _append_state( + runtime_root=runtime_root, + goal_id=goal_id, + turn_key=turn_key, + event_type="phase_pending", + phase_key=_unique_phase_key( + runtime_root, + goal_id=goal_id, + turn_key=turn_key, + prefix="scheduler_apply:pending", + journal_store=journal_store, + ), + phase="scheduler_apply", + fence=latest(), + state=state, + journal_store=journal_store, + ) + return _public_payload( + plan, + state, + execute=True, + replayed=False, + effects=effects, + fence=latest(), + ) + completed = list(TRANSACTION_PHASES[:6]) + state["completed_phases"] = completed + effects["scheduler_acknowledged"] = bool( + scheduler_payload.get("acknowledged") + ) + _append_state( + runtime_root=runtime_root, + goal_id=goal_id, + turn_key=turn_key, + event_type="phase_completed", + phase_key=f"{turn_key}:scheduler_apply:completed", + phase="scheduler_apply", + fence=latest(), + state=state, + journal_store=journal_store, + ) + _fault(fault_injector, "after_scheduler_apply") + + completed = list(TRANSACTION_PHASES) + state.update( + status="committed", + completed_phases=completed, + receipt=_receipt(plan, result, completed_phases=completed), + ) + _append_state( + runtime_root=runtime_root, + goal_id=goal_id, + turn_key=turn_key, + event_type="phase_completed", + phase_key=f"{turn_key}:scheduler_ack:completed", + phase="scheduler_ack", + fence=latest(), + state=state, + journal_store=journal_store, + ) + return _public_payload( + plan, + state, + execute=True, + replayed=False, + effects=effects, + fence=latest(), + ) + + +def run_fenced_loopx_turn_once( + plan: Mapping[str, Any], + *, + host_argv: Sequence[str] | None = None, + host_runner: HostRunner | None = None, + project: Path, + runtime_root: Path, + goal_id: str, + timeout_seconds: float, + execute: bool, + retry_failed: bool = False, + task_validator: TaskValidator | None = None, + writeback: Writeback | None = None, + completion_writeback: CompletionWriteback | None = None, + spend: Spend | None = None, + scheduler: Scheduler | None = None, + lease_controller: TurnLeaseAuthority | None = None, + journal_store: TurnJournalStore | None = None, + fault_injector: FaultInjector | None = None, +) -> dict[str, Any]: + if host_runner is not None and host_argv is not None: + raise ValueError("run-once accepts either host_argv or host_runner, not both") + if host_runner is None: + argv = normalize_host_argv(host_argv or []) + host_projection = {"executable": Path(argv[0]).name, "argv_count": len(argv)} + else: + argv = None + planned_host = plan.get("host") if isinstance(plan.get("host"), dict) else {} + host_projection = { + "executable": "built-in", + "kind": str(planned_host.get("kind") or "codex-cli"), + } + request = build_loopx_turn_host_request(plan) + effects = { + "host_invoked": False, + "state_written": False, + "quota_spent": False, + "scheduler_acknowledged": False, + } + if not execute: + preview = { + "schema_version": LOOPX_TURN_JOURNAL_SCHEMA_VERSION, + "status": "preview", + "host": host_projection, + "result_kind": None, + "receipt": None, + "scheduler": {"disposition": "not_evaluated"}, + } + return _public_payload( + plan, + preview, + execute=False, + replayed=False, + effects=effects, + fence=None, + ) + if writeback is None or spend is None or scheduler is None: + raise ValueError( + "executing run-once requires writeback, spend, and scheduler callbacks" + ) + turn_key = str(request["turn_key"]) + store = journal_store or LocalTurnJournalStore() + try: + prior = load_turn_state( + runtime_root, + goal_id=goal_id, + turn_key=turn_key, + journal_store=store, + ) + except TurnJournalError as exc: + return _failed_closed_payload( + plan, + None, + reason=str(exc), + reason_code="journal_invariant_failed", + effects=effects, + fence=None, + ) + if prior is not None and ( + prior.get("status") in {"committed", "stopped"} + or prior.get("status") == "failed" and not retry_failed + ): + return _public_payload( + plan, + prior, + execute=True, + replayed=True, + effects=effects, + fence=None, + ) + if lease_controller is None: + raise ValueError("executing run-once requires a Turn lease controller") + fence: TurnFence | None = None + final_fence: TurnFence | None = None + try: + fence = lease_controller.acquire() + with lease_controller.heartbeat(fence) as latest: + payload = _execute_fenced( + plan, + request=request, + host_projection=host_projection, + argv=argv, + host_runner=host_runner, + project=project, + runtime_root=runtime_root, + goal_id=goal_id, + timeout_seconds=timeout_seconds, + retry_failed=retry_failed, + task_validator=task_validator, + writeback=writeback, + completion_writeback=completion_writeback, + spend=spend, + scheduler=scheduler, + controller=lease_controller, + fence=fence, + latest=latest, + fault_injector=fault_injector, + effects=effects, + journal_store=store, + ) + final_fence = latest() + except TaskLeaseError as exc: + return _failed_closed_payload( + plan, + prior, + reason=str(exc), + reason_code=exc.code, + effects=effects, + fence=fence, + ) + except TurnJournalError as exc: + return _failed_closed_payload( + plan, + prior, + reason=str(exc), + reason_code="journal_invariant_failed", + effects=effects, + fence=fence, + ) + if payload.get("status") in {"committed", "stopped"} and final_fence is not None: + try: + lease_controller.release(final_fence) + except TaskLeaseError as exc: + payload["lease_release"] = { + "released": False, + "reason_code": exc.code, + } + else: + payload["lease_release"] = {"released": True} + return payload diff --git a/loopx/control_plane/turn_driver/journal.py b/loopx/control_plane/turn_driver/journal.py new file mode 100644 index 000000000..e620f276e --- /dev/null +++ b/loopx/control_plane/turn_driver/journal.py @@ -0,0 +1,459 @@ +"""Append-only, lease-fenced persistence for one LoopX Turn.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +from collections.abc import Mapping +from pathlib import Path +from typing import Any, Protocol + +from ...file_lock import exclusive_file_lock +from ..work_items.task_lease import ( + _require_task_lease_fence_unlocked, + normalize_goal_id, + normalize_idempotency_key, + normalize_lease_todo_id, + normalize_owner, + read_lease, + task_lease_lock_path, + task_lease_path, +) + + +TURN_JOURNAL_EVENT_SCHEMA_VERSION = "loopx_turn_journal_event_v1" +TURN_JOURNAL_PROJECTION_SCHEMA_VERSION = "loopx_turn_journal_projection_v1" +TURN_KEY_RE = re.compile(r"^sha256:[0-9a-f]{64}$") +PUBLIC_TOKEN_RE = re.compile(r"^[A-Za-z0-9_.:@/-]{1,512}$") +MAX_EVENT_BYTES = 128 * 1024 +MAX_EVENT_COUNT = 100_000 +EVENT_FIELDS = { + "schema_version", + "turn_key", + "goal_id", + "event_type", + "phase_key", + "fencing_token", + "payload", + "event_hash", +} + + +class TurnJournalError(ValueError): + """The authoritative Turn journal is malformed or internally inconsistent.""" + + +class TurnJournalStore(Protocol): + """Persistence interface for a Turn journal at its lease-authority seam.""" + + def load_events( + self, + *, + runtime_root: Path, + goal_id: str, + turn_key: str, + ) -> list[dict[str, Any]]: ... + + def append_event( + self, + *, + runtime_root: Path, + goal_id: str, + turn_key: str, + event_type: str, + phase_key: str, + fencing: object, + payload: Mapping[str, Any], + ) -> dict[str, Any]: ... + + +def _require_turn_key(value: object) -> str: + turn_key = str(value or "") + if not TURN_KEY_RE.fullmatch(turn_key): + raise TurnJournalError("turn_key must be a sha256 digest") + return turn_key + + +def _require_public_token(value: object, *, field: str) -> str: + token = str(value or "") + if not PUBLIC_TOKEN_RE.fullmatch(token): + raise TurnJournalError(f"{field} must be a public-safe token") + return token + + +def _canonical_json(value: Mapping[str, Any]) -> bytes: + try: + return json.dumps( + value, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + except (TypeError, ValueError) as exc: + raise TurnJournalError("turn event must be JSON serializable") from exc + + +def _canonical_hash(value: Mapping[str, Any]) -> str: + return "sha256:" + hashlib.sha256(_canonical_json(value)).hexdigest() + + +def _turn_digest(turn_key: str) -> str: + return _require_turn_key(turn_key).removeprefix("sha256:") + + +def turn_journal_path(runtime_root: Path, goal_id: str, turn_key: str) -> Path: + return ( + runtime_root + / "goals" + / normalize_goal_id(goal_id) + / "turn-journals" + / f"{_turn_digest(turn_key)}.jsonl" + ) + + +def turn_projection_path(runtime_root: Path, goal_id: str, turn_key: str) -> Path: + return turn_journal_path(runtime_root, goal_id, turn_key).with_suffix(".json") + + +def build_turn_event( + *, + turn_key: str, + goal_id: str, + event_type: str, + phase_key: str, + fencing_token: str, + payload: Mapping[str, Any], +) -> dict[str, Any]: + normalized_payload = dict(payload) + body = { + "schema_version": TURN_JOURNAL_EVENT_SCHEMA_VERSION, + "turn_key": _require_turn_key(turn_key), + "goal_id": normalize_goal_id(goal_id), + "event_type": _require_public_token(event_type, field="event_type"), + "phase_key": _require_public_token(phase_key, field="phase_key"), + "fencing_token": _require_public_token( + fencing_token, + field="fencing_token", + ), + "payload": normalized_payload, + } + if len(_canonical_json(body)) > MAX_EVENT_BYTES: + raise TurnJournalError("turn event exceeds the size limit") + body["event_hash"] = _canonical_hash(body) + return body + + +def _validate_event( + value: object, + *, + goal_id: str, + turn_key: str, +) -> dict[str, Any]: + if not isinstance(value, dict): + raise TurnJournalError("turn journal line must be one JSON object") + unknown = sorted(set(value) - EVENT_FIELDS) + missing = sorted(EVENT_FIELDS - set(value)) + if unknown or missing: + raise TurnJournalError("turn journal event fields are invalid") + if value.get("schema_version") != TURN_JOURNAL_EVENT_SCHEMA_VERSION: + raise TurnJournalError("turn journal schema is unsupported") + if value.get("goal_id") != goal_id or value.get("turn_key") != turn_key: + raise TurnJournalError("turn journal identity does not match its path") + if not isinstance(value.get("payload"), dict): + raise TurnJournalError("turn journal payload must be an object") + _require_public_token(value.get("event_type"), field="event_type") + _require_public_token(value.get("phase_key"), field="phase_key") + _require_public_token(value.get("fencing_token"), field="fencing_token") + without_hash = {key: value[key] for key in EVENT_FIELDS if key != "event_hash"} + if value.get("event_hash") != _canonical_hash(without_hash): + raise TurnJournalError("turn journal event hash does not match") + return dict(value) + + +def _load_turn_events_unlocked( + path: Path, + *, + goal_id: str, + turn_key: str, +) -> list[dict[str, Any]]: + if not path.exists(): + return [] + raw = path.read_bytes() + if raw and not raw.endswith(b"\n"): + raise TurnJournalError("turn journal contains a truncated line") + events: list[dict[str, Any]] = [] + seen: dict[str, dict[str, Any]] = {} + for line_number, raw_line in enumerate(raw.splitlines(), start=1): + if not raw_line or len(raw_line) > MAX_EVENT_BYTES: + raise TurnJournalError(f"turn journal line {line_number} is invalid") + try: + decoded = json.loads(raw_line) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise TurnJournalError( + f"turn journal line {line_number} is not valid JSON" + ) from exc + event = _validate_event(decoded, goal_id=goal_id, turn_key=turn_key) + phase_key = str(event["phase_key"]) + if prior := seen.get(phase_key): + if prior != event: + raise TurnJournalError("turn journal contains a phase key conflict") + raise TurnJournalError("turn journal contains a duplicate phase key") + seen[phase_key] = event + events.append(event) + if len(events) > MAX_EVENT_COUNT: + raise TurnJournalError("turn journal exceeds the event count limit") + return events + + +def load_turn_events( + runtime_root: Path, + goal_id: str, + turn_key: str, +) -> list[dict[str, Any]]: + normalized_goal_id = normalize_goal_id(goal_id) + normalized_turn_key = _require_turn_key(turn_key) + path = turn_journal_path(runtime_root, normalized_goal_id, normalized_turn_key) + with exclusive_file_lock( + path, + operation="turn_journal_read", + ): + return _load_turn_events_unlocked( + path, + goal_id=normalized_goal_id, + turn_key=normalized_turn_key, + ) + + +def _projection( + *, + goal_id: str, + turn_key: str, + events: list[dict[str, Any]], +) -> dict[str, Any]: + last = events[-1] if events else None + payload = last.get("payload") if isinstance(last, dict) else {} + last_phase = payload.get("phase") if isinstance(payload, dict) else None + return { + "schema_version": TURN_JOURNAL_PROJECTION_SCHEMA_VERSION, + "goal_id": goal_id, + "turn_key": turn_key, + "event_count": len(events), + "phase_keys": [event["phase_key"] for event in events], + "last_phase": last_phase, + "last_event_type": last.get("event_type") if last else None, + "last_event_hash": last.get("event_hash") if last else None, + "fencing_token": last.get("fencing_token") if last else None, + } + + +def _atomic_write_projection(path: Path, value: Mapping[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + encoded = _canonical_json(value) + b"\n" + temporary = path.with_name(f".{path.name}.{os.getpid()}.{id(value)}.tmp") + try: + descriptor = os.open( + temporary, + os.O_CREAT | os.O_EXCL | os.O_WRONLY, + 0o600, + ) + with os.fdopen(descriptor, "wb") as handle: + os.fchmod(handle.fileno(), 0o600) + handle.write(encoded) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + if os.name != "nt": + directory = os.open( + path.parent, + os.O_RDONLY | getattr(os, "O_DIRECTORY", 0), + ) + try: + os.fsync(directory) + finally: + os.close(directory) + finally: + temporary.unlink(missing_ok=True) + + +def _write_projection( + projection_path: Path, + *, + goal_id: str, + turn_key: str, + events: list[dict[str, Any]], +) -> dict[str, Any]: + projection = _projection(goal_id=goal_id, turn_key=turn_key, events=events) + _atomic_write_projection(projection_path, projection) + return projection + + +def rebuild_turn_projection( + runtime_root: Path, + goal_id: str, + turn_key: str, +) -> dict[str, Any]: + normalized_goal_id = normalize_goal_id(goal_id) + normalized_turn_key = _require_turn_key(turn_key) + journal_path = turn_journal_path( + runtime_root, + normalized_goal_id, + normalized_turn_key, + ) + projection_path = turn_projection_path( + runtime_root, + normalized_goal_id, + normalized_turn_key, + ) + with exclusive_file_lock(journal_path, operation="turn_projection_rebuild"): + events = _load_turn_events_unlocked( + journal_path, + goal_id=normalized_goal_id, + turn_key=normalized_turn_key, + ) + return _write_projection( + projection_path, + goal_id=normalized_goal_id, + turn_key=normalized_turn_key, + events=events, + ) + + +def _fencing_value(fencing: object, name: str) -> object: + if isinstance(fencing, Mapping): + return fencing.get(name) + return getattr(fencing, name, None) + + +def append_turn_event( + *, + runtime_root: Path, + goal_id: str, + turn_key: str, + event_type: str, + phase_key: str, + fencing: object, + payload: Mapping[str, Any], +) -> dict[str, Any]: + normalized_goal_id = normalize_goal_id(goal_id) + normalized_turn_key = _require_turn_key(turn_key) + todo_id = normalize_lease_todo_id(_fencing_value(fencing, "todo_id")) + owner = normalize_owner(_fencing_value(fencing, "owner")) + idempotency_key = normalize_idempotency_key( + _fencing_value(fencing, "idempotency_key") + ) + fencing_token = str(_fencing_value(fencing, "token") or "") + event = build_turn_event( + turn_key=normalized_turn_key, + goal_id=normalized_goal_id, + event_type=event_type, + phase_key=phase_key, + fencing_token=fencing_token, + payload=payload, + ) + lease_lock = task_lease_lock_path( + runtime_root=runtime_root, + goal_id=normalized_goal_id, + ) + lease_path = task_lease_path( + runtime_root=runtime_root, + goal_id=normalized_goal_id, + todo_id=todo_id, + ) + journal_path = turn_journal_path( + runtime_root, + normalized_goal_id, + normalized_turn_key, + ) + projection_path = turn_projection_path( + runtime_root, + normalized_goal_id, + normalized_turn_key, + ) + with exclusive_file_lock( + lease_lock, + agent_id=owner, + operation="turn_journal_lease_fence", + ): + _require_task_lease_fence_unlocked( + read_lease(lease_path), + owner=owner, + idempotency_key=idempotency_key, + fencing_token=fencing_token, + ) + with exclusive_file_lock( + journal_path, + agent_id=owner, + operation="turn_journal_append", + ): + events = _load_turn_events_unlocked( + journal_path, + goal_id=normalized_goal_id, + turn_key=normalized_turn_key, + ) + for existing in events: + if existing["phase_key"] != event["phase_key"]: + continue + if existing == event: + _write_projection( + projection_path, + goal_id=normalized_goal_id, + turn_key=normalized_turn_key, + events=events, + ) + return existing + raise TurnJournalError("turn journal phase key conflict") + journal_path.parent.mkdir(parents=True, exist_ok=True) + descriptor = os.open( + journal_path, + os.O_APPEND | os.O_CREAT | os.O_WRONLY, + 0o600, + ) + with os.fdopen(descriptor, "ab") as handle: + os.fchmod(handle.fileno(), 0o600) + handle.write(_canonical_json(event) + b"\n") + handle.flush() + os.fsync(handle.fileno()) + events.append(event) + _write_projection( + projection_path, + goal_id=normalized_goal_id, + turn_key=normalized_turn_key, + events=events, + ) + return event + + +class LocalTurnJournalStore: + """Local filesystem adapter for the canonical fenced Turn journal.""" + + def load_events( + self, + *, + runtime_root: Path, + goal_id: str, + turn_key: str, + ) -> list[dict[str, Any]]: + return load_turn_events(runtime_root, goal_id, turn_key) + + def append_event( + self, + *, + runtime_root: Path, + goal_id: str, + turn_key: str, + event_type: str, + phase_key: str, + fencing: object, + payload: Mapping[str, Any], + ) -> dict[str, Any]: + return append_turn_event( + runtime_root=runtime_root, + goal_id=goal_id, + turn_key=turn_key, + event_type=event_type, + phase_key=phase_key, + fencing=fencing, + payload=payload, + ) diff --git a/loopx/control_plane/turn_driver/lease.py b/loopx/control_plane/turn_driver/lease.py new file mode 100644 index 000000000..330e2bdee --- /dev/null +++ b/loopx/control_plane/turn_driver/lease.py @@ -0,0 +1,217 @@ +"""Turn-scoped ownership adapter over the canonical task lease.""" + +from __future__ import annotations + +import threading +from collections.abc import Callable, Iterator +from contextlib import AbstractContextManager, contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Protocol + +from ..work_items.task_lease import ( + DEFAULT_TASK_LEASE_TTL_SECONDS, + acquire_task_lease, + hold_task_lease_fence, + release_task_lease, + renew_task_lease, + require_task_lease_fence, + task_lease_fencing_generation, + task_lease_fencing_token, +) + + +@dataclass(frozen=True, slots=True) +class TurnFence: + goal_id: str + todo_id: str + owner: str + idempotency_key: str + token: str + generation: int + version: int + + +class TurnLeaseAuthority(Protocol): + """Ownership interface shared by local and remote Turn lease adapters.""" + + def acquire(self) -> TurnFence: ... + + def renew(self, fence: TurnFence) -> TurnFence: ... + + def require_current(self, fence: TurnFence) -> None: ... + + def release(self, fence: TurnFence) -> None: ... + + def heartbeat( + self, + fence: TurnFence, + ) -> AbstractContextManager[Callable[[], TurnFence]]: ... + + def effect_guard(self, fence: TurnFence) -> AbstractContextManager[None]: ... + + +def _turn_fence(lease: dict[str, Any]) -> TurnFence: + fencing_token = task_lease_fencing_token(lease) + return TurnFence( + goal_id=str(lease["goal_id"]), + todo_id=str(lease["todo_id"]), + owner=str(lease["owner"]), + idempotency_key=str(lease["idempotency_key"]), + generation=task_lease_fencing_generation(lease), + version=int(lease["version"]), + **{"token": fencing_token}, + ) + + +@contextmanager +def hold_turn_lease_heartbeat( + fence: TurnFence, + *, + renew: Callable[[TurnFence], TurnFence], + interval_seconds: float, +) -> Iterator[Callable[[], TurnFence]]: + """Renew one lease and expose the latest fence to either adapter.""" + + current = fence + renewal_error: BaseException | None = None + state_lock = threading.Lock() + stop = threading.Event() + + def latest() -> TurnFence: + with state_lock: + if renewal_error is not None: + raise renewal_error + return current + + def renew_until_stopped() -> None: + nonlocal current, renewal_error + while not stop.wait(interval_seconds): + try: + renewed = renew(latest()) + except BaseException as exc: # noqa: BLE001 - propagated by latest() + with state_lock: + renewal_error = exc + stop.set() + return + with state_lock: + current = renewed + + worker = threading.Thread( + target=renew_until_stopped, + name=f"loopx-turn-heartbeat-{fence.generation}", + daemon=True, + ) + worker.start() + try: + yield latest + finally: + stop.set() + worker.join(timeout=max(1.0, interval_seconds + 0.1)) + + +class TurnLeaseController: + """Own one task lease for a bounded Turn execution.""" + + def __init__( + self, + *, + registry_path: Path, + runtime_root: Path, + goal_id: str, + todo_id: str, + owner: str, + idempotency_key: str, + write_scopes: list[str] | None = None, + ttl_seconds: int | None = None, + heartbeat_interval_seconds: float | None = None, + terminal_replay_key: str | None = None, + ) -> None: + self._registry_path = registry_path + self._runtime_root = runtime_root + self._goal_id = goal_id + self._todo_id = todo_id + self._owner = owner + self._idempotency_key = idempotency_key + self._write_scopes = list(write_scopes or []) + self._ttl_seconds = ttl_seconds + self._terminal_replay_key = terminal_replay_key + ttl = ttl_seconds or DEFAULT_TASK_LEASE_TTL_SECONDS + self._heartbeat_interval_seconds = ( + max(0.001, heartbeat_interval_seconds) + if heartbeat_interval_seconds is not None + else ttl / 3 + ) + + def acquire(self) -> TurnFence: + outcome = acquire_task_lease( + registry_path=self._registry_path, + runtime_root=self._runtime_root, + goal_id=self._goal_id, + todo_id=self._todo_id, + owner=self._owner, + idempotency_key=self._idempotency_key, + write_scopes=self._write_scopes, + ttl_seconds=self._ttl_seconds, + terminal_replay_key=self._terminal_replay_key, + ) + return _turn_fence(dict(outcome["lease"])) + + def renew(self, fence: TurnFence) -> TurnFence: + outcome = renew_task_lease( + registry_path=self._registry_path, + runtime_root=self._runtime_root, + goal_id=fence.goal_id, + todo_id=fence.todo_id, + owner=fence.owner, + idempotency_key=fence.idempotency_key, + ttl_seconds=self._ttl_seconds, + expected_version=fence.version, + terminal_replay_key=self._terminal_replay_key, + ) + return _turn_fence(dict(outcome["lease"])) + + def require_current(self, fence: TurnFence) -> None: + require_task_lease_fence( + runtime_root=self._runtime_root, + goal_id=fence.goal_id, + todo_id=fence.todo_id, + owner=fence.owner, + idempotency_key=fence.idempotency_key, + fencing_token=fence.token, + ) + + def release(self, fence: TurnFence) -> None: + release_task_lease( + runtime_root=self._runtime_root, + goal_id=fence.goal_id, + todo_id=fence.todo_id, + owner=fence.owner, + idempotency_key=fence.idempotency_key, + expected_version=fence.version, + ) + + @contextmanager + def heartbeat( + self, + fence: TurnFence, + ) -> Iterator[Callable[[], TurnFence]]: + with hold_turn_lease_heartbeat( + fence, + renew=self.renew, + interval_seconds=self._heartbeat_interval_seconds, + ) as latest: + yield latest + + @contextmanager + def effect_guard(self, fence: TurnFence) -> Iterator[None]: + with hold_task_lease_fence( + runtime_root=self._runtime_root, + goal_id=fence.goal_id, + todo_id=fence.todo_id, + owner=fence.owner, + idempotency_key=fence.idempotency_key, + fencing_token=fence.token, + operation="turn_effect_guard", + ): + yield diff --git a/loopx/control_plane/turn_driver/settlement.py b/loopx/control_plane/turn_driver/settlement.py index b71806f62..14f4b84b4 100644 --- a/loopx/control_plane/turn_driver/settlement.py +++ b/loopx/control_plane/turn_driver/settlement.py @@ -92,7 +92,10 @@ def _committed_payload(value: Mapping[str, Any] | None) -> bool: return bool( isinstance(value, Mapping) and value.get("ok") is True - and value.get("appended") is True + and ( + value.get("appended") is True + or value.get("idempotent") is True + ) ) diff --git a/loopx/control_plane/turn_driver/transaction.py b/loopx/control_plane/turn_driver/transaction.py index 92470fc92..c0f52fb97 100644 --- a/loopx/control_plane/turn_driver/transaction.py +++ b/loopx/control_plane/turn_driver/transaction.py @@ -3,7 +3,9 @@ from __future__ import annotations import json +import re from collections.abc import Mapping +from dataclasses import dataclass from enum import Enum from hashlib import sha256 from typing import Any @@ -30,6 +32,27 @@ "scheduler_apply", "scheduler_ack", ) +TURN_EFFECT_PHASES = {"durable_writeback", "quota_spend", "scheduler_apply"} +TURN_KEY_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$") +FENCING_TOKEN_PATTERN = re.compile(r"^fence:[0-9a-f]{64}$") + + +@dataclass(frozen=True, slots=True) +class TurnEffectEnvelope: + turn_key: str + phase: str + phase_key: str + fencing_token: str + + def __post_init__(self) -> None: + if not TURN_KEY_PATTERN.fullmatch(self.turn_key): + raise ValueError("Turn effect turn_key must be a sha256 digest") + if self.phase not in TURN_EFFECT_PHASES: + raise ValueError("Turn effect phase is not irreversible") + if self.phase_key != f"{self.turn_key}:{self.phase}": + raise ValueError("Turn effect phase_key is not canonical") + if not FENCING_TOKEN_PATTERN.fullmatch(self.fencing_token): + raise ValueError("Turn effect fencing_token is invalid") class LoopXTurnResultKind(str, Enum): @@ -43,6 +66,7 @@ class LoopXTurnResultKind(str, Enum): VALIDATION_FAILED = "validation_failed" WRITEBACK_FAILED = "writeback_failed" QUOTA_SPEND_FAILED = "quota_spend_failed" + FAILED_CLOSED = "failed_closed" MATERIAL_RESULT_KINDS = { @@ -58,6 +82,7 @@ class LoopXTurnResultKind(str, Enum): LoopXTurnResultKind.VALIDATION_FAILED, LoopXTurnResultKind.WRITEBACK_FAILED, LoopXTurnResultKind.QUOTA_SPEND_FAILED, + LoopXTurnResultKind.FAILED_CLOSED, } STOP_RESULT_KINDS = { LoopXTurnResultKind.USER_ACTION_REQUIRED, @@ -304,8 +329,14 @@ def validate_loopx_turn_receipt( if failed_phase and failed_phase != expected_next: errors.append("failed_phase must be the next uncompleted transaction phase") - if failed_phase and kind not in FAILURE_PHASES: + if ( + failed_phase + and kind not in FAILURE_PHASES + and kind is not LoopXTurnResultKind.FAILED_CLOSED + ): errors.append("failed_phase is only valid for a typed failure result") + if kind is LoopXTurnResultKind.FAILED_CLOSED and failed_phase is None: + errors.append("failed_closed must declare the next uncompleted phase") if kind in FAILURE_PHASES and failed_phase != FAILURE_PHASES[kind]: errors.append(f"{kind.value} must declare failed_phase={FAILURE_PHASES[kind]}") if kind in MATERIAL_RESULT_KINDS and "validation" not in completed: @@ -315,7 +346,13 @@ def validate_loopx_turn_receipt( ok = not errors fully_committed = completed == list(TRANSACTION_PHASES) - failed = kind in FAILURE_PHASES if kind is not None else False + failed = bool( + kind is not None + and ( + kind in FAILURE_PHASES + or kind is LoopXTurnResultKind.FAILED_CLOSED + ) + ) stopped = kind in STOP_RESULT_KINDS if kind is not None else False status = ( "invalid" diff --git a/loopx/control_plane/turn_effect.py b/loopx/control_plane/turn_effect.py new file mode 100644 index 000000000..b9e08f3d2 --- /dev/null +++ b/loopx/control_plane/turn_effect.py @@ -0,0 +1,68 @@ +"""Stable idempotency identities for irreversible Turn callbacks.""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections.abc import Mapping +from pathlib import Path +from typing import Any + + +# Mirrors task_lease without importing its history-dependent module here. +TURN_EFFECT_KEY_PATTERN = re.compile(r"^[A-Za-z0-9_.:@/-]{1,160}$") + + +def normalize_turn_effect_key(value: str | None) -> str | None: + key = str(value or "").strip() + if not key: + return None + if not TURN_EFFECT_KEY_PATTERN.fullmatch(key): + raise ValueError("turn_effect_key must be a public-safe token") + return key + + +def turn_effect_input_hash(value: Mapping[str, Any]) -> str: + encoded = json.dumps( + value, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return "sha256:" + hashlib.sha256(encoded).hexdigest() + + +def find_turn_effect_record( + index_path: Path, + turn_effect_key: str, +) -> dict[str, Any] | None: + if not index_path.exists(): + return None + found: dict[str, Any] | None = None + for line_number, line in enumerate( + index_path.read_text(encoding="utf-8").splitlines(), + start=1, + ): + if not line.strip(): + continue + try: + value = json.loads(line) + except json.JSONDecodeError as exc: + raise ValueError(f"run index line {line_number} is not valid JSON") from exc + if not isinstance(value, dict): + raise ValueError(f"run index line {line_number} must be an object") + if value.get("turn_effect_key") != turn_effect_key: + continue + if found is not None and found != value: + raise ValueError("turn effect key conflict") + found = value + return found + + +def require_matching_turn_effect( + existing: Mapping[str, Any], + effect_input_hash: str, +) -> None: + if existing.get("effect_input_hash") != effect_input_hash: + raise ValueError("turn effect key conflict") diff --git a/loopx/control_plane/work_items/task_lease.py b/loopx/control_plane/work_items/task_lease.py index ce8101cca..6de8d631c 100644 --- a/loopx/control_plane/work_items/task_lease.py +++ b/loopx/control_plane/work_items/task_lease.py @@ -1,8 +1,11 @@ from __future__ import annotations import fnmatch +import hashlib import json +import os import re +import threading from collections.abc import Callable, Iterator from contextlib import contextmanager from datetime import datetime, timedelta @@ -30,6 +33,7 @@ DEFAULT_TASK_LEASE_TTL_SECONDS = 45 * 60 MAX_TASK_LEASE_TTL_SECONDS = 24 * 60 * 60 IDEMPOTENCY_KEY_PATTERN = re.compile(r"^[A-Za-z0-9_.:@/-]{1,160}$") +_TASK_LEASE_LOCK_STATE = threading.local() class TaskLeaseError(ValueError): @@ -109,7 +113,64 @@ class _VerifiedTaskLeaseFence(dict): the payload stays JSON-serializable for every consumer at all times. """ - release_hook: Callable[[], None] | None = None + release_hook: Callable[[], dict[str, Any]] | None = None + + +@contextmanager +def hold_task_lease_lock( + *, + runtime_root: Path, + goal_id: str, + agent_id: str | None, + operation: str, +) -> Iterator[Path]: + """Hold the canonical per-goal lease lock, reusing it on this thread.""" + + lock_target = task_lease_lock_path(runtime_root=runtime_root, goal_id=goal_id) + lock_key = str(lock_target.expanduser().resolve(strict=False)) + held = getattr(_TASK_LEASE_LOCK_STATE, "paths", None) + if held is None: + held = set() + _TASK_LEASE_LOCK_STATE.paths = held + if lock_key in held: + yield lock_target + return + with exclusive_file_lock( + lock_target, + agent_id=agent_id, + operation=operation, + ): + held.add(lock_key) + try: + yield lock_target + finally: + held.remove(lock_key) + + +@contextmanager +def hold_task_lease_mutation_locks( + *, + registry_path: Path, + goal_id: str, + state_file: Path, + agent_id: str | None, + operation: str, + lease_runtime_root: Path | None = None, +) -> Iterator[Path]: + """Hold lease then state-file locks for one todo lifecycle mutation.""" + + runtime_root = lease_runtime_root or runtime_root_from_registry(registry_path, None) + with hold_task_lease_lock( + runtime_root=runtime_root, + goal_id=goal_id, + agent_id=agent_id, + operation=f"{operation}_lease_fence", + ), exclusive_file_lock( + state_file, + agent_id=agent_id, + operation=operation, + ): + yield runtime_root @contextmanager @@ -123,6 +184,7 @@ def hold_task_lease_mutation_fence( idempotency_key: str | None, expected_version: int | None = None, require_active_when_key_supplied: bool = True, + lease_runtime_root: Path | None = None, ) -> Iterator[dict[str, Any]]: """Hold the per-goal lease lock while one todo lifecycle write commits. @@ -140,18 +202,15 @@ def hold_task_lease_mutation_fence( if idempotency_key is not None else None ) - runtime_root = runtime_root_from_registry(registry_path, None) + runtime_root = lease_runtime_root or runtime_root_from_registry(registry_path, None) lease_path = task_lease_path( runtime_root=runtime_root, goal_id=normalized_goal_id, todo_id=normalized_todo_id, ) - lock_target = task_lease_lock_path( + with hold_task_lease_lock( runtime_root=runtime_root, goal_id=normalized_goal_id, - ) - with exclusive_file_lock( - lock_target, agent_id=actor_agent_id, operation="task_lease_mutation_fence", ): @@ -230,7 +289,11 @@ def hold_task_lease_mutation_fence( "execution_instance_verified": True, } ) - fence.release_hook = lambda: remove_lease(lease_path) + fence.release_hook = lambda: _write_released_lease( + lease_path, + lease, + released_at=now_utc(), + ) yield fence @@ -245,14 +308,15 @@ def release_verified_task_lease_fence( the per-goal lease lock it acquired is still held; the lease therefore cannot have been renewed, transferred, or re-acquired since the fence verified the owner and idempotency key. The release reuses the CLI release - semantics (the lease file is removed; no new lifecycle state). + semantics and atomically writes a released tombstone so future acquisitions + retain a monotonic fencing generation. The private release hook rides on the fence object's attribute, not inside the payload mapping, and is disarmed here on every call. Only a committed, key-verified fence releases the lease; non-verified fences carry no hook and are never touched. A release failure never unwinds the committed lifecycle write: it is surfaced additively as fence["released"] = False - and the lease file is left for an explicit `loopx task-lease release` or + and the active lease is left for an explicit `loopx task-lease release` or TTL expiry. """ @@ -370,6 +434,7 @@ def require_task_lease_owner_allowed( goal_id: str, todo_id: str, owner: str, + terminal_replay_key: str | None = None, ) -> dict[str, Any]: owner = require_registered_task_lease_owner( registry_path=registry_path, @@ -387,6 +452,26 @@ def require_task_lease_owner_allowed( owner=owner, registered_agents=registered_agent_ids_from_registry(registry_path, goal_id), ) + normalized_replay_key = str(terminal_replay_key or "").strip() + if ( + constraint.get("reason") == "todo_not_open" + and isinstance(todo, dict) + and str(todo.get("status") or "").strip().lower() == "done" + and normalized_replay_key + and str(todo.get("completion_turn_key") or "").strip() + == normalized_replay_key + ): + replay_todo = {**todo, "status": "open"} + replay_constraint = task_lease_owner_constraint( + replay_todo, + owner=owner, + registered_agents=registered_agent_ids_from_registry( + registry_path, + goal_id, + ), + ) + if replay_constraint.get("effective") is True: + constraint = {"effective": True, "terminal_replay": True} if constraint.get("effective") is not True: reason = str(constraint.get("reason") or "owner_not_allowed") if reason == "todo_not_found": @@ -440,18 +525,197 @@ def read_lease(path: Path) -> dict[str, Any] | None: return payload +def _fsync_parent_directory(path: Path) -> None: + if os.name == "nt": # Windows does not expose a portable directory fsync. + return + directory = os.open(path.parent, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + try: + os.fsync(directory) + finally: + os.close(directory) + + def write_lease(path: Path, payload: dict[str, Any]) -> None: path.parent.mkdir(parents=True, exist_ok=True) - temp_path = path.with_name(f".{path.name}.{id(payload)}.tmp") - temp_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") - temp_path.replace(path) + encoded = (json.dumps(payload, ensure_ascii=False, indent=2) + "\n").encode( + "utf-8" + ) + temp_path = path.with_name(f".{path.name}.{os.getpid()}.{id(payload)}.tmp") + try: + descriptor = os.open( + temp_path, + os.O_CREAT | os.O_EXCL | os.O_WRONLY, + 0o600, + ) + with os.fdopen(descriptor, "wb") as handle: + os.fchmod(handle.fileno(), 0o600) + handle.write(encoded) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temp_path, path) + _fsync_parent_directory(path) + finally: + temp_path.unlink(missing_ok=True) + + +def _write_released_lease( + path: Path, + lease: dict[str, Any], + *, + released_at: datetime, +) -> dict[str, Any]: + released_lease = dict(lease) + released_lease["status"] = "released" + released_lease["version"] = int(lease.get("version") or 0) + 1 + released_lease["fencing_generation"] = task_lease_fencing_generation(lease) + released_lease["released_at"] = isoformat(released_at) + released_lease["updated_at"] = isoformat(released_at) + write_lease(path, released_lease) + return released_lease + + +def task_lease_fencing_generation(lease: dict[str, Any]) -> int: + if lease.get("schema_version") != TASK_LEASE_SCHEMA_VERSION: + raise TaskLeaseError("lease schema is unsupported", code="corrupt_lease") + try: + generation = int(lease.get("fencing_generation") or lease.get("version") or 0) + except (TypeError, ValueError) as exc: + raise TaskLeaseError( + "lease fencing generation is invalid", + code="corrupt_lease", + ) from exc + if generation <= 0: + raise TaskLeaseError( + "lease fencing generation is invalid", + code="corrupt_lease", + ) + return generation + + +def task_lease_fencing_token(lease: dict[str, Any]) -> str: + generation = task_lease_fencing_generation(lease) + material = "\0".join( + [ + str(lease.get("goal_id") or ""), + str(lease.get("todo_id") or ""), + str(lease.get("owner") or ""), + str(lease.get("idempotency_key") or ""), + str(lease.get("acquired_at") or ""), + str(generation), + ] + ) + return "fence:" + hashlib.sha256(material.encode("utf-8")).hexdigest() -def remove_lease(path: Path) -> None: - try: - path.unlink() - except FileNotFoundError: - pass +def _require_task_lease_fence_unlocked( + lease: dict[str, Any] | None, + *, + owner: str, + idempotency_key: str, + fencing_token: str, + at: datetime | None = None, +) -> dict[str, Any]: + if not lease_is_active(lease, at=at): + raise TaskLeaseError("lease is missing or expired", code="lease_not_active") + assert lease is not None + if task_lease_fencing_token(lease) != fencing_token: + raise TaskLeaseError( + "worker fencing token is stale", + code="stale_fencing_token", + ) + if lease.get("owner") != owner or lease.get("idempotency_key") != idempotency_key: + raise TaskLeaseError( + "lease owner or idempotency key mismatch", + code="lease_cas_mismatch", + ) + return dict(lease) + + +def require_task_lease_fence_value( + lease: dict[str, Any] | None, + *, + owner: str, + idempotency_key: str, + fencing_token: str, + at: datetime | None = None, +) -> dict[str, Any]: + """Validate an already-read lease, including remote CLI projections.""" + + return _require_task_lease_fence_unlocked( + lease, + owner=normalize_owner(owner), + idempotency_key=normalize_idempotency_key(idempotency_key), + fencing_token=str(fencing_token or ""), + at=at, + ) + + +def require_task_lease_fence( + *, + runtime_root: Path, + goal_id: str, + todo_id: str, + owner: str, + idempotency_key: str, + fencing_token: str, +) -> dict[str, Any]: + goal_id = normalize_goal_id(goal_id) + todo_id = normalize_lease_todo_id(todo_id) + owner = normalize_owner(owner) + idempotency_key = normalize_idempotency_key(idempotency_key) + lock_target = task_lease_lock_path(runtime_root=runtime_root, goal_id=goal_id) + path = task_lease_path( + runtime_root=runtime_root, + goal_id=goal_id, + todo_id=todo_id, + ) + with exclusive_file_lock( + lock_target, + agent_id=owner, + operation="task_lease_fence", + ): + return _require_task_lease_fence_unlocked( + read_lease(path), + owner=owner, + idempotency_key=idempotency_key, + fencing_token=str(fencing_token or ""), + ) + + +@contextmanager +def hold_task_lease_fence( + *, + runtime_root: Path, + goal_id: str, + todo_id: str, + owner: str, + idempotency_key: str, + fencing_token: str, + operation: str, +) -> Iterator[dict[str, Any]]: + """Hold the lease lock while one fenced durable effect executes.""" + + normalized_goal_id = normalize_goal_id(goal_id) + normalized_todo_id = normalize_lease_todo_id(todo_id) + normalized_owner = normalize_owner(owner) + normalized_idempotency_key = normalize_idempotency_key(idempotency_key) + path = task_lease_path( + runtime_root=runtime_root, + goal_id=normalized_goal_id, + todo_id=normalized_todo_id, + ) + with hold_task_lease_lock( + runtime_root=runtime_root, + goal_id=normalized_goal_id, + agent_id=normalized_owner, + operation=operation, + ): + yield _require_task_lease_fence_unlocked( + read_lease(path), + owner=normalized_owner, + idempotency_key=normalized_idempotency_key, + fencing_token=str(fencing_token or ""), + ) def lease_expires_at(lease: dict[str, Any] | None) -> datetime | None: @@ -602,6 +866,7 @@ def build_lease( write_scopes: list[str], acquire_ttl_seconds: int, version: int, + fencing_generation: int, acquired_at: str, updated_at: str, expires_at: str, @@ -615,6 +880,7 @@ def build_lease( "write_scopes": write_scopes, "acquire_ttl_seconds": acquire_ttl_seconds, "version": version, + "fencing_generation": fencing_generation, "acquired_at": acquired_at, "updated_at": updated_at, "expires_at": expires_at, @@ -633,6 +899,7 @@ def acquire_task_lease( ttl_seconds: int | None = None, write_scopes: list[str] | None = None, expected_version: int | None = None, + terminal_replay_key: str | None = None, ) -> dict[str, Any]: goal_id = normalize_goal_id(goal_id) todo_id = normalize_lease_todo_id(todo_id) @@ -654,6 +921,7 @@ def acquire_task_lease( goal_id=goal_id, todo_id=todo_id, owner=owner, + terminal_replay_key=terminal_replay_key, ) existing = read_lease(lease_path) assert_expected_version(existing, expected_version) @@ -727,6 +995,9 @@ def acquire_task_lease( write_scopes=normalized_write_scopes, acquire_ttl_seconds=ttl, version=int((existing or {}).get("version") or 0) + 1, + fencing_generation=( + task_lease_fencing_generation(existing) + 1 if existing else 1 + ), acquired_at=updated_at, updated_at=updated_at, expires_at=expires_at, @@ -754,6 +1025,7 @@ def renew_task_lease( idempotency_key: str, ttl_seconds: int | None = None, expected_version: int | None = None, + terminal_replay_key: str | None = None, ) -> dict[str, Any]: goal_id = normalize_goal_id(goal_id) todo_id = normalize_lease_todo_id(todo_id) @@ -773,6 +1045,7 @@ def renew_task_lease( goal_id=goal_id, todo_id=todo_id, owner=owner, + terminal_replay_key=terminal_replay_key, ) lease = read_lease(lease_path) assert_expected_version(lease, expected_version) @@ -781,6 +1054,7 @@ def renew_task_lease( if lease.get("owner") != owner or lease.get("idempotency_key") != idempotency_key: raise TaskLeaseError("lease owner or idempotency key mismatch", code="lease_cas_mismatch") lease = dict(lease) + lease["fencing_generation"] = task_lease_fencing_generation(lease) lease["version"] = int(lease.get("version") or 0) + 1 lease["updated_at"] = isoformat(at) lease["expires_at"] = isoformat(at + timedelta(seconds=ttl)) @@ -844,6 +1118,7 @@ def transfer_task_lease( lease = dict(lease) lease["owner"] = new_owner lease["idempotency_key"] = new_idempotency_key + lease["fencing_generation"] = task_lease_fencing_generation(lease) + 1 lease["version"] = int(lease.get("version") or 0) + 1 lease["updated_at"] = isoformat(at) lease["expires_at"] = isoformat(at + timedelta(seconds=ttl)) @@ -890,15 +1165,23 @@ def release_task_lease( "missing": True, "lease_path": str(lease_path), } - if lease_is_active(lease, at=at) and ( - lease.get("owner") != owner or lease.get("idempotency_key") != idempotency_key - ): + if lease.get("owner") != owner or lease.get("idempotency_key") != idempotency_key: raise TaskLeaseError("lease owner or idempotency key mismatch", code="lease_cas_mismatch") - remove_lease(lease_path) - released_lease = dict(lease) - released_lease["status"] = "released" - released_lease["released_at"] = isoformat(at) - released_lease["updated_at"] = isoformat(at) + if lease.get("status") == "released": + return { + "ok": True, + "schema_version": TASK_LEASE_SCHEMA_VERSION, + "action": "release", + "released": False, + "idempotent": True, + "lease": lease, + "lease_path": str(lease_path), + } + released_lease = _write_released_lease( + lease_path, + lease, + released_at=at, + ) return { "ok": True, "schema_version": TASK_LEASE_SCHEMA_VERSION, diff --git a/loopx/quota.py b/loopx/quota.py index 12d12d73c..6eda6495d 100644 --- a/loopx/quota.py +++ b/loopx/quota.py @@ -1045,6 +1045,7 @@ def spend_quota_slot( operator_inbox_urgency_projector: Callable[..., dict[str, Any]] | None = None, todo_id: str | None = None, turn_instance_id: str | None = None, + turn_effect_key: str | None = None, ) -> dict[str, Any]: safe_goal_id = _validate_goal_id_path_segment(str(goal_id or "")) if turn_instance_id and source != DEFAULT_SLOT_SPEND_SOURCE: @@ -1173,4 +1174,5 @@ def spend_quota_slot( render_markdown=_render_quota_slot_preview_markdown, execute=execute, source=source, + turn_effect_key=turn_effect_key, ) diff --git a/loopx/state_refresh.py b/loopx/state_refresh.py index 287913381..d7729b230 100644 --- a/loopx/state_refresh.py +++ b/loopx/state_refresh.py @@ -7,6 +7,10 @@ from typing import Any from .control_plane.runtime.time import now_local_iso +from .control_plane.state_refresh_recording import ( + append_state_refresh_index, + build_state_refresh_output_projections as _build_state_refresh_output_projections, +) from .control_plane.work_items.delivery_batch_scale import ( DELIVERY_BATCH_SCALE_CHOICES as DELIVERY_BATCH_SCALE_CHOICES, require_delivery_batch_scale, @@ -665,103 +669,6 @@ def build_state_refresh_record( return record -def _build_state_refresh_output_projections( - *, - record: dict[str, Any], - registry_path: Path, - runtime_root: Path, - project: Path | None, - json_path: Path, - markdown_path: Path, - index_path: Path, - dry_run: bool, - autonomous_replan_recorded_requested: bool, -) -> tuple[dict[str, Any], dict[str, Any]]: - """Project one refresh record into its compact index and CLI response.""" - - record_state = record.get("state") if isinstance(record.get("state"), dict) else {} - record_frontmatter = record_state.get("frontmatter") or {} - index_record = { - field: record[field] - for field in ( - "generated_at", "goal_id", "classification", "recommended_action", - "recommended_action_source", "health_check", - ) - } - index_record.update({ - "json_path": str(json_path), - "markdown_path": str(markdown_path), - "state": { - "sha256_16": record_state.get("sha256_16"), - "frontmatter": {"updated_at": record_frontmatter.get("updated_at")}, - }, - "runtime_projection_route": record["runtime_projection_route"], - }) - for field in ( - "delivery_batch_scale", - "delivery_outcome", - "delivery_workspace", - "settlement_identity", - "turn_instance_id", - "todo_id", - ): - if field in record: - index_record[field] = record[field] - - replan_ack = record.get("autonomous_replan_ack") or {} - if autonomous_replan_recorded_requested: - index_record["autonomous_replan_ack"] = replan_ack - if replan_ack.get("requested_classification"): - index_record["requested_classification"] = replan_ack["requested_classification"] - - agent_vision = record.get("agent_vision") - if isinstance(agent_vision, dict): - index_record["agent_vision"] = { - field: agent_vision.get(field) - for field in ( - "schema_version", "agent_id", "state", "vision_patch", - "todo_delta", "vision_budget", - ) - } - if isinstance(agent_vision.get("path_delta"), dict): - index_record["agent_vision"]["path_delta"] = agent_vision["path_delta"] - - for field in ("vision_checkpoint", "progress_scope", "agent_id", "agent_lane"): - if field in record: - index_record[field] = record[field] - - payload: dict[str, Any] = { - "ok": True, - "dry_run": dry_run, - "appended": not dry_run, - "registry": str(registry_path), - "runtime_root": str(runtime_root), - "project": str(project) if project else None, - } - payload.update({ - field: record.get(field) - for field in ("goal_id", "classification", "progress_scope", "agent_id", "agent_lane") - }) - payload.update({ - "autonomous_replan_recorded": bool(replan_ack.get("recorded")), - "autonomous_replan_recorded_requested": autonomous_replan_recorded_requested, - "repair_delta_contract": replan_ack.get("delta_contract"), - "json_path": str(json_path), - "markdown_path": str(markdown_path), - "index_path": str(index_path), - }) - payload.update({ - field: record.get(field) - for field in ( - "agent_vision", "vision_checkpoint", "recommended_action", - "recommended_action_source", "active_state_next_action_update", - "generated_at", "health_check", - ) - }) - payload.update(record) - return index_record, payload - - def render_state_refresh_markdown(payload: dict[str, Any]) -> str: state = payload.get("state") if isinstance(payload.get("state"), dict) else {} frontmatter = state.get("frontmatter") if isinstance(state.get("frontmatter"), dict) else {} @@ -940,7 +847,7 @@ def render_state_refresh_markdown(payload: dict[str, Any]) -> str: return "\n".join(lines) -def refresh_state_run( +def _refresh_state_run( *, registry_path: Path, runtime_root_override: str | None, @@ -965,6 +872,8 @@ def refresh_state_run( vision_unchanged_reason: str | None = None, dry_run: bool, sync_global: bool = True, + _turn_effect_key: str | None = None, + _effect_input_hash: str | None = None, ) -> dict[str, Any]: safe_goal_id = validate_goal_id_path_segment(goal_id) validate_public_safe_text("classification", classification) @@ -1341,7 +1250,11 @@ def refresh_state_run( compact_route["projection_enabled"] = bool(sync_global) compact_route["projection_marker_field"] = "shared_runtime_projection" record["runtime_projection_route"] = compact_route - + if _turn_effect_key is not None and _effect_input_hash is None: + raise ValueError("turn effect input hash is required") + if _turn_effect_key is not None: + record["turn_effect_key"] = _turn_effect_key + record["effect_input_hash"] = _effect_input_hash runs_dir = runtime_root / "goals" / safe_goal_id / "runs" json_path, markdown_path = unique_run_paths(runs_dir, generated_at) index_path = runs_dir / "index.jsonl" @@ -1356,6 +1269,12 @@ def refresh_state_run( dry_run=dry_run, autonomous_replan_recorded_requested=bool(autonomous_replan_recorded), ) + if _turn_effect_key is not None: + index_record["turn_effect_key"] = _turn_effect_key + index_record["effect_input_hash"] = _effect_input_hash + payload["turn_effect_key"] = _turn_effect_key + payload["effect_input_hash"] = _effect_input_hash + payload["idempotent"] = False if dry_run: expected_write_scopes = ["runtime_history"] if active_state_next_action_update and active_state_next_action_update.get("would_update"): @@ -1411,8 +1330,8 @@ def refresh_state_run( encoding="utf-8", ) markdown_path.write_text(render_state_refresh_markdown(payload) + "\n", encoding="utf-8") - with index_path.open("a", encoding="utf-8") as f: - f.write(json.dumps(index_record, ensure_ascii=False) + "\n") + if _turn_effect_key is None: + append_state_refresh_index(index_path, index_record) if sync_global and route_status in {"missing", "ambiguous"}: payload["ok"] = False payload["partial_write"] = not dry_run @@ -1495,4 +1414,12 @@ def refresh_state_run( "raw_artifacts_copied": False, "recommended_action_copied": False, } + if not dry_run and _turn_effect_key is not None: + append_state_refresh_index( + index_path, + index_record, + turn_effect_result_ok=payload.get("ok") is True, + ) return payload + +from .control_plane.state_refresh_effect import refresh_state_run as refresh_state_run # noqa: E402 diff --git a/loopx/todos.py b/loopx/todos.py index 711421749..242364d75 100644 --- a/loopx/todos.py +++ b/loopx/todos.py @@ -107,6 +107,7 @@ ) from .control_plane.work_items.task_lease import ( hold_task_lease_mutation_fence, + hold_task_lease_mutation_locks, release_verified_task_lease_fence, ) @@ -1511,6 +1512,8 @@ def complete_goal_todo( completion_turn_key: str | None = None, task_lease_idempotency_key: str | None = None, task_lease_expected_version: int | None = None, + task_lease_runtime_root: Path | None = None, + release_task_lease_on_commit: bool = True, note: str | None = None, no_followup: bool = False, successor_todo_ids: list[str] | None = None, @@ -1546,11 +1549,11 @@ def complete_goal_todo( project=project, state_file=state_file, ) - with exclusive_file_lock( - resolved_state_file, - agent_id=agent_id or claimed_by, - operation="todo_complete", - ), ExitStack() as lease_fence_stack: + with hold_task_lease_mutation_locks( + registry_path=registry_path, lease_runtime_root=task_lease_runtime_root, + state_file=resolved_state_file, goal_id=goal_id, + agent_id=agent_id or claimed_by, operation="todo_complete", + ) as lease_runtime_root, ExitStack() as lease_fence_stack: original = resolved_state_file.read_text(encoding="utf-8") lines = original.splitlines() updated_at = now_local() @@ -1635,6 +1638,7 @@ def complete_goal_todo( task_lease_idempotency_key is not None or task_lease_expected_version is not None ), + lease_runtime_root=lease_runtime_root, ) ) normalized_successor_todo_ids = normalize_todo_id_list(successor_todo_ids) @@ -1722,10 +1726,11 @@ def complete_goal_todo( event_result["linked_successor_id"] = completion_policy.linked_successor_id event_result["mutation_authority"] = mutation_authority event_result["task_lease_fence"] = task_lease_fence - release_verified_task_lease_fence( - task_lease_fence, - committed=bool(event_result.get("changed")) and not dry_run, - ) + if release_task_lease_on_commit: + release_verified_task_lease_fence( + task_lease_fence, + committed=bool(event_result.get("changed")) and not dry_run, + ) return event_result update_result = apply_todo_update_to_lines( lines, @@ -1837,10 +1842,11 @@ def complete_goal_todo( new_text = replace_updated_at(new_text, updated_at) if changed and not dry_run: resolved_state_file.write_text(new_text, encoding="utf-8") - release_verified_task_lease_fence( - task_lease_fence, - committed=changed and not dry_run, - ) + if release_task_lease_on_commit: + release_verified_task_lease_fence( + task_lease_fence, + committed=changed and not dry_run, + ) result = { "ok": True, "dry_run": dry_run, diff --git a/tests/control_plane/test_cli_output_differential.py b/tests/control_plane/test_cli_output_differential.py index dfa79da56..17456afc6 100644 --- a/tests/control_plane/test_cli_output_differential.py +++ b/tests/control_plane/test_cli_output_differential.py @@ -140,26 +140,60 @@ def test_smaller_candidate_still_fails_when_semantics_are_removed( assert any(failure_fragment in failure for failure in result["rows"][0]["failures"]) -def test_declared_action_signature_coverage_migration_requires_review() -> None: +@pytest.mark.parametrize( + ("base_coverage", "candidate_coverage"), + [ + ( + "turn_envelope_action_dimensions_v0", + "turn_envelope_action_dimensions_v1", + ), + ( + "turn_envelope_action_dimensions_v0", + "turn_envelope_action_dimensions_v2", + ), + ( + "turn_envelope_action_dimensions_v1", + "turn_envelope_action_dimensions_v3", + ), + ( + "turn_envelope_action_dimensions_v2", + "turn_envelope_action_dimensions_v3", + ), + ], +) +def test_declared_action_signature_coverage_migration_requires_review( + base_coverage: str, + candidate_coverage: str, +) -> None: + base = _row(action_signature_coverages=[base_coverage]) candidate = _row( action_signature_sha256="versioned-semantic-signature", - action_signature_coverages=["turn_envelope_action_dimensions_v1"], + action_signature_coverages=[candidate_coverage], ) - result = compare_cli_output_receipts(_receipt(_row()), _receipt(candidate)) + result = compare_cli_output_receipts(_receipt(base), _receipt(candidate)) assert result["ok"] is True assert result["review_required"] is True assert result["rows"][0]["review_signals"] == [ "action_signature coverage migrated: " - "turn_envelope_action_dimensions_v0 -> turn_envelope_action_dimensions_v1" + f"{base_coverage} -> {candidate_coverage}" ] -def test_unknown_action_signature_coverage_migration_fails_closed() -> None: +@pytest.mark.parametrize( + "candidate_coverage", + [ + "turn_envelope_action_dimensions_v3", + "turn_envelope_action_dimensions_v99", + ], +) +def test_unknown_action_signature_coverage_migration_fails_closed( + candidate_coverage: str, +) -> None: candidate = _row( action_signature_sha256="unknown-semantic-signature", - action_signature_coverages=["turn_envelope_action_dimensions_v2"], + action_signature_coverages=[candidate_coverage], ) result = compare_cli_output_receipts(_receipt(_row()), _receipt(candidate)) diff --git a/tests/control_plane/test_quota_slot_accounting.py b/tests/control_plane/test_quota_slot_accounting.py index 755444aa5..b0e884618 100644 --- a/tests/control_plane/test_quota_slot_accounting.py +++ b/tests/control_plane/test_quota_slot_accounting.py @@ -9,6 +9,7 @@ from loopx.control_plane.quota.slot_accounting import ( build_quota_slot_preview_for_decision, build_quota_slot_spend_event, + record_quota_slot_spend_from_preview, ) from loopx.quota import spend_quota_slot from loopx.rollout_event_log import rollout_event_log_path @@ -151,6 +152,185 @@ def test_unchanged_monitor_poll_is_not_accountable_delivery(tmp_path: Path) -> N assert _preview(runtime)["ok"] is False +def test_quota_spend_deduplicates_same_turn_effect_key(tmp_path: Path) -> None: + runtime = tmp_path / "runtime" + preview = _preview( + runtime, + before_overrides={ + "should_run": True, + "effective_action": "normal_run", + }, + ) + effect_key = "sha256:" + "b" * 64 + ":quota_spend" + kwargs = { + "goal_id": GOAL_ID, + "self_repair_spend_actions": frozenset(), + "render_markdown": lambda _payload: "quota fixture", + "execute": True, + "source": "adapter", + "turn_effect_key": effect_key, + } + + first = record_quota_slot_spend_from_preview( + preview, + {"runtime_root": str(runtime)}, + **kwargs, + ) + replay = record_quota_slot_spend_from_preview( + preview, + {"runtime_root": str(runtime)}, + **kwargs, + ) + index_path = runtime / "goals" / GOAL_ID / "runs" / "index.jsonl" + index_record = json.loads(index_path.read_text(encoding="utf-8")) + run_record = json.loads( + Path(str(first["json_path"])).read_text(encoding="utf-8") + ) + + assert first["appended"] is True + assert replay["appended"] is False + assert replay["idempotent"] is True + assert replay["json_path"] == first["json_path"] + assert replay["markdown_path"] == first["markdown_path"] + assert run_record["turn_effect_key"] == effect_key + assert index_record["turn_effect_key"] == effect_key + assert run_record["effect_input_hash"] == first["effect_input_hash"] + assert index_record["effect_input_hash"] == first["effect_input_hash"] + assert first["before"]["spent_slots"] == 0 + assert first["after"]["spent_slots"] == 1 + assert replay["before"]["spent_slots"] == 0 + assert replay["after"]["spent_slots"] == 1 + assert replay["quota_event"] == first["quota_event"] + assert len(index_path.read_text(encoding="utf-8").splitlines()) == 1 + + +def test_quota_spend_replays_after_accounting_projection_advances( + tmp_path: Path, +) -> None: + runtime = tmp_path / "runtime" + preview = _preview( + runtime, + before_overrides={ + "should_run": True, + "effective_action": "normal_run", + }, + ) + preview.update( + { + "delivery_run_generated_at": "2026-08-11T00:00:00+00:00", + "delivery_run_classification": "fixture_delivery", + "delivery_run_agent_id": AGENT_A, + } + ) + reprojected = json.loads(json.dumps(preview)) + reprojected["before"]["quota"]["spent_slots"] = 1 + reprojected["after"]["quota"]["spent_slots"] = 2 + reprojected["delivery_run_generated_at"] = None + reprojected["delivery_run_classification"] = None + reprojected["delivery_run_agent_id"] = None + effect_key = "sha256:" + "e" * 64 + ":quota_spend" + kwargs = { + "goal_id": GOAL_ID, + "self_repair_spend_actions": frozenset(), + "render_markdown": lambda _payload: "quota fixture", + "execute": True, + "source": "adapter", + "turn_effect_key": effect_key, + } + + first = record_quota_slot_spend_from_preview( + preview, + {"runtime_root": str(runtime)}, + **kwargs, + ) + replay = record_quota_slot_spend_from_preview( + reprojected, + {"runtime_root": str(runtime)}, + **kwargs, + ) + + assert replay["appended"] is False + assert replay["idempotent_replay"] is True + assert replay["effect_input_hash"] == first["effect_input_hash"] + assert replay["before"]["spent_slots"] == 0 + assert replay["after"]["spent_slots"] == 1 + + +def test_quota_spend_rejects_turn_effect_key_content_drift(tmp_path: Path) -> None: + runtime = tmp_path / "runtime" + preview = _preview( + runtime, + before_overrides={ + "should_run": True, + "effective_action": "normal_run", + }, + ) + effect_key = "sha256:" + "c" * 64 + ":quota_spend" + kwargs = { + "goal_id": GOAL_ID, + "self_repair_spend_actions": frozenset(), + "render_markdown": lambda _payload: "quota fixture", + "execute": True, + "source": "adapter", + "turn_effect_key": effect_key, + } + record_quota_slot_spend_from_preview( + preview, + {"runtime_root": str(runtime)}, + **kwargs, + ) + changed = json.loads(json.dumps(preview)) + changed["slots"] = 2 + changed["after"]["quota"]["spent_slots"] = 2 + + with pytest.raises(ValueError, match="turn effect key conflict"): + record_quota_slot_spend_from_preview( + changed, + {"runtime_root": str(runtime)}, + **kwargs, + ) + + +def test_spend_quota_slot_propagates_turn_effect_key_to_durable_writer( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + runtime = tmp_path / "runtime" + preview = _preview( + runtime, + before_overrides={ + "should_run": True, + "effective_action": "normal_run", + }, + ) + monkeypatch.setattr( + "loopx.quota.build_quota_slot_preview", + lambda *_args, **_kwargs: preview, + ) + effect_key = "sha256:" + "d" * 64 + ":quota_spend" + + first = spend_quota_slot( + {"runtime_root": str(runtime)}, + goal_id=GOAL_ID, + execute=True, + source="adapter", + turn_effect_key=effect_key, + ) + replay = spend_quota_slot( + {"runtime_root": str(runtime)}, + goal_id=GOAL_ID, + execute=True, + source="adapter", + turn_effect_key=effect_key, + ) + index_path = runtime / "goals" / GOAL_ID / "runs" / "index.jsonl" + + assert first["appended"] is True + assert replay["appended"] is False + assert replay["idempotent"] is True + assert len(index_path.read_text(encoding="utf-8").splitlines()) == 1 + + @pytest.mark.parametrize("before_overrides", SAFE_BYPASS_CASES) def test_safe_bypass_rejects_no_accountable_writeback( tmp_path: Path, diff --git a/tests/control_plane/test_task_lease.py b/tests/control_plane/test_task_lease.py index 8d7a1d425..adcb00265 100644 --- a/tests/control_plane/test_task_lease.py +++ b/tests/control_plane/test_task_lease.py @@ -12,10 +12,14 @@ TaskLeaseError, acquire_task_lease, assert_expected_version, + inspect_task_lease, normalize_idempotency_key, normalize_ttl_seconds, release_task_lease, renew_task_lease, + require_task_lease_fence, + task_lease_fencing_token, + task_lease_path, task_lease_owner_constraint, transfer_task_lease, write_scopes_overlap, @@ -151,9 +155,11 @@ def test_task_lease_lifecycle_preserves_idempotency_and_versions( acquired = acquire_task_lease(**arguments) assert acquired["acquired"] is True assert acquired["lease"]["version"] == 1 + assert acquired["lease"]["fencing_generation"] == 1 assert acquired["lease"]["expires_at"] == ( now + timedelta(seconds=120) ).isoformat().replace("+00:00", "Z") + acquired_fence = task_lease_fencing_token(acquired["lease"]) repeated = acquire_task_lease(**arguments) assert repeated["idempotent"] is True @@ -173,6 +179,8 @@ def test_task_lease_lifecycle_preserves_idempotency_and_versions( expected_version=1, ) assert renewed["lease"]["version"] == 2 + assert renewed["lease"]["fencing_generation"] == 1 + assert task_lease_fencing_token(renewed["lease"]) == acquired_fence transferred = transfer_task_lease( registry_path=registry_path, @@ -187,6 +195,8 @@ def test_task_lease_lifecycle_preserves_idempotency_and_versions( ) assert transferred["lease"]["owner"] == "agent-b" assert transferred["lease"]["version"] == 3 + assert transferred["lease"]["fencing_generation"] == 2 + assert task_lease_fencing_token(transferred["lease"]) != acquired_fence released = release_task_lease( runtime_root=runtime_root, @@ -200,5 +210,207 @@ def test_task_lease_lifecycle_preserves_idempotency_and_versions( assert released["lease"]["status"] == "released" assert released["lease"]["released_at"] == now.isoformat().replace("+00:00", "Z") assert released["lease"]["updated_at"] == released["lease"]["released_at"] + assert released["lease"]["version"] == 4 + assert released["lease"]["fencing_generation"] == 2 assert task_lease.lease_is_active(released["lease"], at=now) is False - assert not Path(str(released["lease_path"])).exists() + assert Path(str(released["lease_path"])).exists() + assert task_lease.read_lease(Path(str(released["lease_path"]))) == released["lease"] + repeated_release = release_task_lease( + runtime_root=runtime_root, + goal_id="goal-a", + todo_id="todo_leasea", + owner="agent-b", + idempotency_key="turn-2", + expected_version=4, + ) + assert repeated_release["idempotent"] is True + assert repeated_release["lease"] == released["lease"] + inspected = inspect_task_lease( + registry_path=registry_path, + runtime_root=runtime_root, + goal_id="goal-a", + todo_id="todo_leasea", + ) + assert inspected["active"] is False + assert inspected["lease"] == released["lease"] + + +def test_stale_task_lease_fence_is_rejected_after_reacquire( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + now = datetime(2026, 7, 13, tzinfo=timezone.utc) + monkeypatch.setattr(task_lease, "now_utc", lambda: now) + monkeypatch.setattr(task_lease, "require_task_lease_owner_allowed", lambda **_: {}) + monkeypatch.setattr(task_lease, "active_conflicts", lambda **_: []) + registry_path = tmp_path / "registry.json" + runtime_root = tmp_path / "runtime" + arguments = { + "registry_path": registry_path, + "runtime_root": runtime_root, + "goal_id": "goal-a", + "todo_id": "todo_leasea", + "owner": "agent-a", + "idempotency_key": "turn-1", + "ttl_seconds": 30, + "write_scopes": ["loopx/**"], + } + + acquired = acquire_task_lease(**arguments) + stale_token = task_lease_fencing_token(acquired["lease"]) + now += timedelta(seconds=31) + reacquired = acquire_task_lease( + **arguments, + expected_version=acquired["lease"]["version"], + ) + + assert reacquired["lease"]["fencing_generation"] == 2 + with pytest.raises(TaskLeaseError) as error: + require_task_lease_fence( + runtime_root=runtime_root, + goal_id="goal-a", + todo_id="todo_leasea", + owner="agent-a", + idempotency_key="turn-1", + fencing_token=stale_token, + ) + + assert error.value.code == "stale_fencing_token" + + +def test_task_lease_file_is_private_and_release_tombstone_reacquires_higher_generation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + now = datetime(2026, 7, 13, tzinfo=timezone.utc) + monkeypatch.setattr(task_lease, "now_utc", lambda: now) + monkeypatch.setattr(task_lease, "require_task_lease_owner_allowed", lambda **_: {}) + monkeypatch.setattr(task_lease, "active_conflicts", lambda **_: []) + registry_path = tmp_path / "registry.json" + runtime_root = tmp_path / "runtime" + arguments = { + "registry_path": registry_path, + "runtime_root": runtime_root, + "goal_id": "goal-a", + "todo_id": "todo_leasea", + "owner": "agent-a", + "idempotency_key": "turn-1", + "ttl_seconds": 120, + "write_scopes": ["loopx/**"], + } + path = task_lease_path( + runtime_root=runtime_root, + goal_id="goal-a", + todo_id="todo_leasea", + ) + + acquired = acquire_task_lease(**arguments) + assert path.stat().st_mode & 0o777 == 0o600 + released = release_task_lease( + runtime_root=runtime_root, + goal_id="goal-a", + todo_id="todo_leasea", + owner="agent-a", + idempotency_key="turn-1", + expected_version=acquired["lease"]["version"], + ) + reacquired = acquire_task_lease( + **{**arguments, "idempotency_key": "turn-2"}, + expected_version=released["lease"]["version"], + ) + + assert reacquired["lease"]["fencing_generation"] == 2 + assert reacquired["lease"]["version"] == released["lease"]["version"] + 1 + + +def test_legacy_active_lease_uses_current_version_as_stable_generation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + now = datetime(2026, 7, 13, tzinfo=timezone.utc) + monkeypatch.setattr(task_lease, "now_utc", lambda: now) + monkeypatch.setattr(task_lease, "require_task_lease_owner_allowed", lambda **_: {}) + runtime_root = tmp_path / "runtime" + path = task_lease_path( + runtime_root=runtime_root, + goal_id="goal-a", + todo_id="todo_leasea", + ) + legacy = { + "schema_version": "task_lease_v0", + "goal_id": "goal-a", + "todo_id": "todo_leasea", + "owner": "agent-a", + "idempotency_key": "turn-1", + "write_scopes": ["loopx/**"], + "acquire_ttl_seconds": 120, + "version": 7, + "acquired_at": now.isoformat().replace("+00:00", "Z"), + "updated_at": now.isoformat().replace("+00:00", "Z"), + "expires_at": (now + timedelta(seconds=120)).isoformat().replace( + "+00:00", "Z" + ), + "status": "active", + } + task_lease.write_lease(path, legacy) + token = task_lease_fencing_token(legacy) + + renewed = renew_task_lease( + registry_path=tmp_path / "registry.json", + runtime_root=runtime_root, + goal_id="goal-a", + todo_id="todo_leasea", + owner="agent-a", + idempotency_key="turn-1", + expected_version=7, + ) + + assert renewed["lease"]["version"] == 8 + assert renewed["lease"]["fencing_generation"] == 7 + assert task_lease_fencing_token(renewed["lease"]) == token + + +def test_write_lease_failure_before_replace_preserves_complete_old_record( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + path = tmp_path / "lease.json" + old = {"schema_version": "task_lease_v0", "version": 1} + new = {"schema_version": "task_lease_v0", "version": 2} + task_lease.write_lease(path, old) + + def fail_replace(_source: Path, _target: Path) -> None: + raise OSError("injected before replace") + + monkeypatch.setattr(task_lease.os, "replace", fail_replace) + with pytest.raises(OSError, match="injected before replace"): + task_lease.write_lease(path, new) + + assert task_lease.read_lease(path) == old + assert list(tmp_path.glob(".lease.json.*.tmp")) == [] + + +def test_write_lease_directory_fsync_failure_leaves_complete_new_record( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + path = tmp_path / "lease.json" + old = {"schema_version": "task_lease_v0", "version": 1} + new = {"schema_version": "task_lease_v0", "version": 2} + task_lease.write_lease(path, old) + real_fsync = task_lease.os.fsync + calls = 0 + + def fail_directory_fsync(descriptor: int) -> None: + nonlocal calls + calls += 1 + if calls == 2: + raise OSError("injected directory fsync") + real_fsync(descriptor) + + monkeypatch.setattr(task_lease.os, "fsync", fail_directory_fsync) + with pytest.raises(OSError, match="injected directory fsync"): + task_lease.write_lease(path, new) + + assert task_lease.read_lease(path) == new + assert list(tmp_path.glob(".lease.json.*.tmp")) == [] diff --git a/tests/control_plane/test_task_orchestration_admission.py b/tests/control_plane/test_task_orchestration_admission.py index 225a57468..21e413bda 100644 --- a/tests/control_plane/test_task_orchestration_admission.py +++ b/tests/control_plane/test_task_orchestration_admission.py @@ -458,7 +458,10 @@ def test_claimed_primary_agent_coordinates_unclaimed_child_work() -> None: summary = { "items": [ { - **_todo("todo_primary"), + **_todo( + "todo_primary", + required_write_scopes=["loopx/**"], + ), "claimed_by": AGENT_ID, }, _todo("todo_child"), @@ -490,6 +493,10 @@ def test_claimed_primary_agent_coordinates_unclaimed_child_work() -> None: assert len(contracts) == 1 assert contracts[0]["coordinator_agent_id"] == AGENT_ID assert contracts[0]["primary_todo_id"] == "todo_primary" + assert contracts[0]["primary_todo"] == { + "todo_id": "todo_primary", + "required_write_scopes": ["loopx/**"], + } assert contracts[0]["eligible_child_lanes"][0]["todo_id"] == "todo_child" diff --git a/tests/control_plane/test_todo_mutation_authority.py b/tests/control_plane/test_todo_mutation_authority.py index aa5904213..485bba351 100644 --- a/tests/control_plane/test_todo_mutation_authority.py +++ b/tests/control_plane/test_todo_mutation_authority.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +from datetime import datetime, timedelta, timezone from pathlib import Path import pytest @@ -339,7 +340,12 @@ def test_active_task_lease_fences_same_agent_completion_instance( "execution_instance_verified": True, "released": True, } - assert not _lease_path(tmp_path, todo["todo_id"]).exists() + released_lease = json.loads( + _lease_path(tmp_path, todo["todo_id"]).read_text(encoding="utf-8") + ) + assert released_lease["status"] == "released" + assert released_lease["version"] == 2 + assert released_lease["fencing_generation"] == 1 replayed = complete_goal_todo( registry_path=registry, @@ -457,7 +463,12 @@ def test_event_projected_completion_reports_task_lease_fence( "execution_instance_verified": True, "released": True, } - assert not _lease_path(tmp_path, todo_id).exists() + released_lease = json.loads( + _lease_path(tmp_path, todo_id).read_text(encoding="utf-8") + ) + assert released_lease["status"] == "released" + assert released_lease["version"] == 2 + assert released_lease["fencing_generation"] == 1 def test_unfenced_completion_leaves_no_lease_artifacts(tmp_path: Path) -> None: @@ -562,6 +573,100 @@ def test_dry_run_completion_does_not_release_lease(tmp_path: Path) -> None: assert state.read_text(encoding="utf-8") == before +def test_completion_can_defer_release_until_outer_turn_commit(tmp_path: Path) -> None: + registry, _state = _write_fixture(tmp_path, multi_agent=False) + todo = _add_agent_todo(registry) + lease_key = "outer-turn-instance" + acquired = acquire_task_lease( + registry_path=registry, + runtime_root=tmp_path / "runtime", + goal_id=GOAL_ID, + todo_id=todo["todo_id"], + owner=AUTHOR_AGENT, + idempotency_key=lease_key, + ttl_seconds=600, + ) + + completed = complete_goal_todo( + registry_path=registry, + goal_id=GOAL_ID, + todo_id=todo["todo_id"], + claimed_by=AUTHOR_AGENT, + task_lease_idempotency_key=lease_key, + release_task_lease_on_commit=False, + evidence="outer Turn still has fenced effects to settle", + no_followup=True, + ) + + assert completed["task_lease_fence"]["execution_instance_verified"] is True + assert "released" not in completed["task_lease_fence"] + lease_path = _lease_path(tmp_path, todo["todo_id"]) + assert json.loads(lease_path.read_text(encoding="utf-8"))["status"] == "active" + + released = release_task_lease( + runtime_root=tmp_path / "runtime", + goal_id=GOAL_ID, + todo_id=todo["todo_id"], + owner=AUTHOR_AGENT, + idempotency_key=lease_key, + expected_version=acquired["lease"]["version"], + ) + assert released["lease"]["status"] == "released" + assert released["lease"]["fencing_generation"] == 1 + + +def test_terminal_turn_replay_reacquires_only_matching_completed_todo( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + current = datetime(2026, 8, 11, tzinfo=timezone.utc) + monkeypatch.setattr(task_lease_module, "now_utc", lambda: current) + registry, _state = _write_fixture(tmp_path, multi_agent=False) + todo = _add_agent_todo(registry) + turn_key = "sha256:" + "a" * 64 + lease_key = f"turn:{turn_key}" + arguments = { + "registry_path": registry, + "runtime_root": tmp_path / "runtime", + "goal_id": GOAL_ID, + "todo_id": todo["todo_id"], + "owner": AUTHOR_AGENT, + "idempotency_key": lease_key, + "ttl_seconds": 1, + } + acquired = acquire_task_lease(**arguments) + stale_token = task_lease_module.task_lease_fencing_token(acquired["lease"]) + complete_goal_todo( + registry_path=registry, + goal_id=GOAL_ID, + todo_id=todo["todo_id"], + claimed_by=AUTHOR_AGENT, + completion_turn_key=turn_key, + task_lease_idempotency_key=lease_key, + release_task_lease_on_commit=False, + evidence="Turn completion persisted before outer settlement", + no_followup=True, + ) + current += timedelta(seconds=2) + + with pytest.raises(TaskLeaseError) as ordinary_reacquire: + acquire_task_lease(**arguments) + assert ordinary_reacquire.value.code == "todo_not_open" + with pytest.raises(TaskLeaseError) as wrong_turn: + acquire_task_lease(**arguments, terminal_replay_key="sha256:other") + assert wrong_turn.value.code == "todo_not_open" + + recovered = acquire_task_lease( + **arguments, + terminal_replay_key=turn_key, + ) + assert recovered["lease"]["fencing_generation"] == 2 + assert recovered["lease"]["version"] == 2 + assert task_lease_module.task_lease_fencing_token( + recovered["lease"] + ) != stale_token + + def test_release_failure_after_commit_keeps_completion_ok( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -579,10 +684,10 @@ def test_release_failure_after_commit_keeps_completion_ok( ttl_seconds=600, ) - def _fail_remove(path: Path) -> None: - raise OSError("simulated unlink failure") + def _fail_write(path: Path, payload: dict[str, object]) -> None: + raise OSError("simulated tombstone write failure") - monkeypatch.setattr(task_lease_module, "remove_lease", _fail_remove) + monkeypatch.setattr(task_lease_module, "write_lease", _fail_write) completed = complete_goal_todo( registry_path=registry, goal_id=GOAL_ID, @@ -604,17 +709,7 @@ def _fail_remove(path: Path) -> None: def test_reopened_todo_completes_unfenced_after_lease_release( tmp_path: Path, ) -> None: - """Pin the accepted reopen-window divergence of release-on-completion. - - Before leases were released on completion, the leftover lease file - resurrected as the execution-instance fence if a done todo was manually - reopened inside the lease TTL, so a second completion without the key - was rejected with lease_fence_required. With the lease released at the - first committed completion, the reopened todo completes unfenced. This - matches the end state of the disciplined baseline flow (complete followed - by an explicit `loopx task-lease release`); no production path reopens - done todos. - """ + """A released tombstone stays inactive if a Todo is manually reopened.""" registry, _state = _write_fixture(tmp_path, multi_agent=False) todo = _add_agent_todo(registry) @@ -639,7 +734,9 @@ def test_reopened_todo_completes_unfenced_after_lease_release( no_followup=True, ) assert completed["task_lease_fence"]["released"] is True - assert not _lease_path(tmp_path, todo["todo_id"]).exists() + assert json.loads( + _lease_path(tmp_path, todo["todo_id"]).read_text(encoding="utf-8") + )["status"] == "released" reopened = update_goal_todo( registry_path=registry, @@ -667,13 +764,8 @@ def test_reopened_todo_completes_unfenced_after_lease_release( } -def test_cli_release_after_auto_release_reports_missing(tmp_path: Path) -> None: - """A post-completion `loopx task-lease release` now sees no lease file. - - Baseline removed the then-stale lease and reported released=True; after - release-on-completion the same call reports released=False, missing=True - with ok=True (the pre-existing double-release shape). - """ +def test_cli_release_after_auto_release_is_idempotent(tmp_path: Path) -> None: + """A post-completion release replays the persisted tombstone.""" registry, _state = _write_fixture(tmp_path, multi_agent=False) todo = _add_agent_todo(registry) @@ -707,7 +799,9 @@ def test_cli_release_after_auto_release_reports_missing(tmp_path: Path) -> None: assert released["ok"] is True assert released["released"] is False - assert released["missing"] is True + assert released["idempotent"] is True + assert released["lease"]["status"] == "released" + assert released["lease"]["fencing_generation"] == 1 def test_exception_after_verified_fence_leaves_lease_intact( diff --git a/tests/test_loopx_turn_driver.py b/tests/test_loopx_turn_driver.py index bca95b097..34e240d8b 100644 --- a/tests/test_loopx_turn_driver.py +++ b/tests/test_loopx_turn_driver.py @@ -14,14 +14,24 @@ from loopx.control_plane.turn_driver import ( LOOPX_TURN_SESSION_BINDING_SCHEMA_VERSION, LoopXTurnRoute, + TurnLeaseController, build_loopx_turn_host_request, build_loopx_turn_plan, codex_cli_session_binding, + load_turn_events, loopx_turn_execution_committed, run_loopx_turn_once, + selected_turn_todo_write_scopes, ) from loopx.control_plane.turn_driver.codex_cli import _store_codex_cli_session from loopx.control_plane.quota.live_decision import bind_scheduler_followup_cli_routes +from loopx.control_plane.work_items.task_lease import ( + acquire_task_lease, + read_lease, + release_task_lease, + task_lease_fencing_token, + task_lease_path, +) from loopx.todos import complete_goal_todo @@ -126,6 +136,11 @@ def _adaptive_envelope() -> dict[str, object]: "schema_version": "task_orchestration_contract_v2", "mode": "adaptive", "coordinator_agent_id": "codex-fixture", + "primary_todo_id": "todo_fixture0001", + "primary_todo": { + "todo_id": "todo_fixture0001", + "required_write_scopes": [], + }, "child_brief_defaults": { "schema_version": "subagent_control_plane_handoff_v0", "parent_goal_id": "fixture-goal", @@ -189,6 +204,10 @@ def _signed_adaptive_envelope( "task_orchestration_contract": { **_adaptive_envelope()["task_orchestration_contract"], "primary_todo_id": "todo_primary", + "primary_todo": { + "todo_id": "todo_primary", + "required_write_scopes": ["src/**"], + }, }, } return build_turn_envelope(decision) @@ -257,7 +276,7 @@ def test_turn_plan_uses_adaptive_primary_todo_for_bundle_lineage() -> None: ) assert ( first_envelope["action_signature"]["source_hash"] - != second_envelope["action_signature"]["source_hash"] + == second_envelope["action_signature"]["source_hash"] ) payload = build_loopx_turn_plan( @@ -270,7 +289,8 @@ def test_turn_plan_uses_adaptive_primary_todo_for_bundle_lineage() -> None: assert payload["route"]["kind"] == LoopXTurnRoute.READY_FOR_HOST.value assert payload["route"]["selected_todo"] == { "todo_id": "todo_primary", - "source": "task_orchestration_contract.primary_todo_id", + "source": "task_orchestration_contract.primary_todo", + "required_write_scopes": ["src/**"], } same_primary_plan = build_loopx_turn_plan( second_envelope, @@ -295,6 +315,19 @@ def test_turn_plan_uses_adaptive_primary_todo_for_bundle_lineage() -> None: ) +def test_adaptive_primary_todo_without_scope_projection_fails_closed() -> None: + with pytest.raises( + ValueError, + match="adaptive primary todo must project required_write_scopes", + ): + selected_turn_todo_write_scopes( + { + "todo_id": "todo_primary", + "source": "task_orchestration_contract.primary_todo_id", + } + ) + + def test_turn_plan_exposes_only_qualified_claude_child_contexts() -> None: envelope = _adaptive_envelope() payload = build_loopx_turn_plan( @@ -342,6 +375,10 @@ def test_codex_session_binding_uses_adaptive_primary_todo( "text": "A stale pre-orchestration selection", } envelope["task_orchestration_contract"]["primary_todo_id"] = "todo_primary" + envelope["task_orchestration_contract"]["primary_todo"] = { + "todo_id": "todo_primary", + "required_write_scopes": ["src/**"], + } lineage = { "goal_id": "fixture-goal", "agent_id": "codex-fixture", @@ -664,7 +701,7 @@ def _write_live_fixture(root: Path) -> tuple[Path, Path, Path]: "## Agent Todo", "", "- [ ] [P0] Advance one public fixture.", - " ", + " ", "", ] ), @@ -1096,6 +1133,7 @@ def test_turn_run_once_cli_commits_validated_result_and_one_quota_slot( "quota_spent": True, "scheduler_acknowledged": False, } + assert payload["lease_release"] == {"released": True} state_path = ( project / ".codex" @@ -1110,6 +1148,23 @@ def test_turn_run_once_cli_commits_validated_result_and_one_quota_slot( "fixture_progress", "quota_slot_spent", ] + turn_key = payload["resume_turn_key"] + assert [row["turn_effect_key"] for row in rows] == [ + f"{turn_key}:durable_writeback", + f"{turn_key}:quota_spend", + ] + lease = read_lease( + task_lease_path( + runtime_root=runtime, + goal_id="loopx-turn-fixture", + todo_id="todo_fixture0001", + ) + ) + assert lease is not None + assert lease["status"] == "released" + assert lease["owner"] == "codex-fixture" + assert lease["idempotency_key"] == f"turn:{turn_key}" + assert lease["write_scopes"] == ["docs/**"] resumed_output = io.StringIO() with contextlib.redirect_stdout(resumed_output): @@ -1228,10 +1283,19 @@ def test_turn_run_once_cli_completes_selected_todo_after_validation( assert payload["status"] == "committed" assert payload["effects"]["state_written"] is True assert payload["effects"]["quota_spent"] is True - journal_path = next( - (runtime / "goals" / "loopx-turn-fixture" / "turns").glob("*.json") + events = load_turn_events( + runtime, + "loopx-turn-fixture", + payload["resume_turn_key"], ) - journal = json.loads(journal_path.read_text(encoding="utf-8")) + states = [ + event["payload"]["state"] + for event in events + if isinstance(event.get("payload"), dict) + and isinstance(event["payload"].get("state"), dict) + ] + assert states + journal = states[-1] assert journal["writeback"]["completion"] == { "todo_id": "todo_fixture0001", "continuation": "active_goal", @@ -1275,14 +1339,373 @@ def test_turn_run_once_cli_completes_selected_todo_after_validation( assert not any(replayed["effects"].values()) +def test_turn_run_once_cli_fails_closed_before_host_on_lease_conflict( + tmp_path: Path, +) -> None: + project, runtime, registry = _write_live_fixture(tmp_path) + host_project = tmp_path / "isolated-host-workspace" + host_project.mkdir() + acquire_task_lease( + registry_path=registry, + runtime_root=runtime, + goal_id="loopx-turn-fixture", + todo_id="todo_fixture0001", + owner="codex-fixture", + idempotency_key="turn:competing-worker", + write_scopes=["docs/**"], + ) + host_script = """ +import json +import pathlib +import sys +request = json.load(sys.stdin) +pathlib.Path("host-invoked.txt").write_text("invoked", encoding="utf-8") +json.dump({ + "schema_version": "loopx_turn_result_v0", + "turn_key": request["turn_key"], + "result_kind": "wait", + "completed_phases": ["host_execute", "typed_result"], + "classification": "fixture_wait", + "recommended_action": "Wait", + "next_action": "Wait", + "delivery_batch_scale": "single_surface", + "delivery_outcome": "outcome_noop", + "vision_unchanged_reason": "The fixture is unchanged.", + "summary": "No work was attempted." +}, sys.stdout) +""" + output = io.StringIO() + + with contextlib.redirect_stdout(output): + exit_code = cli_main( + [ + "--registry", + str(registry), + "--runtime-root", + str(runtime), + "--format", + "json", + "turn", + "run-once", + "--goal-id", + "loopx-turn-fixture", + "--agent-id", + "codex-fixture", + "--project", + str(host_project), + "--host-adapter-command-json", + json.dumps([sys.executable, "-c", host_script]), + "--scan-root", + str(project), + "--no-global-sync", + "--execute", + ] + ) + + payload = json.loads(output.getvalue()) + assert exit_code == 1, payload + assert payload["status"] == "failed_closed" + assert payload["reason_code"] == "todo_lease_conflict" + assert payload["effects"]["host_invoked"] is False + assert not (host_project / "host-invoked.txt").exists() + lease = read_lease( + task_lease_path( + runtime_root=runtime, + goal_id="loopx-turn-fixture", + todo_id="todo_fixture0001", + ) + ) + assert lease is not None + assert lease["status"] == "active" + assert lease["idempotency_key"] == "turn:competing-worker" + + +def test_remote_turn_cli_journal_and_effects_share_one_lease_fence( + tmp_path: Path, +) -> None: + project, runtime, registry = _write_live_fixture(tmp_path) + envelope = _envelope() + envelope["goal_id"] = "loopx-turn-fixture" + plan = build_loopx_turn_plan( + envelope, + host="generic-cli", + execution_mode="isolated-headless", + ) + transaction = plan["transaction"] + assert isinstance(transaction, dict) + turn_key = str(transaction["turn_key"]) + idempotency_key = f"turn:{turn_key}" + acquired = acquire_task_lease( + registry_path=registry, + runtime_root=runtime, + goal_id="loopx-turn-fixture", + todo_id="todo_fixture0001", + owner="codex-fixture", + idempotency_key=idempotency_key, + write_scopes=["docs/**"], + ) + lease = acquired["lease"] + assert isinstance(lease, dict) + fencing_token = task_lease_fencing_token(lease) + event_request = { + "event_type": "turn_owned", + "phase_key": f"{turn_key}:ownership:1", + "fencing": { + "todo_id": "todo_fixture0001", + "owner": "codex-fixture", + "idempotency_key": idempotency_key, + "token": fencing_token, + }, + "payload": {"phase": "ownership"}, + } + + append_output = io.StringIO() + with contextlib.redirect_stdout(append_output): + append_exit = cli_main( + [ + "--registry", + str(registry), + "--runtime-root", + str(runtime), + "--format", + "json", + "turn", + "journal-append", + "--goal-id", + "loopx-turn-fixture", + "--turn-key", + turn_key, + "--event-json", + json.dumps(event_request), + ] + ) + appended = json.loads(append_output.getvalue()) + assert append_exit == 0, appended + assert appended["event"]["phase_key"] == event_request["phase_key"] + + read_output = io.StringIO() + with contextlib.redirect_stdout(read_output): + read_exit = cli_main( + [ + "--registry", + str(registry), + "--runtime-root", + str(runtime), + "--format", + "json", + "turn", + "journal-read", + "--goal-id", + "loopx-turn-fixture", + "--turn-key", + turn_key, + ] + ) + readback = json.loads(read_output.getvalue()) + assert read_exit == 0, readback + assert readback["event_count"] == 1 + + fence_args = [ + "--turn-fence-todo-id", + "todo_fixture0001", + "--turn-fence-idempotency-key", + idempotency_key, + "--turn-fencing-token", + fencing_token, + ] + writeback_key = f"{turn_key}:durable_writeback" + refresh_argv = [ + "--registry", + str(registry), + "--runtime-root", + str(runtime), + "--format", + "json", + "refresh-state", + "--goal-id", + "loopx-turn-fixture", + "--classification", + "fixture_remote_turn_progress", + "--recommended-action", + "Continue the fixture.", + "--next-action", + "Run the next bounded fixture Turn.", + "--delivery-batch-scale", + "single_surface", + "--delivery-outcome", + "outcome_progress", + "--agent-id", + "codex-fixture", + "--progress-scope", + "goal", + "--vision-unchanged-reason", + "The fixture objective remains open.", + "--no-global-sync", + "--turn-effect-key", + writeback_key, + *fence_args, + ] + refresh_payloads: list[dict[str, object]] = [] + for _ in range(2): + output = io.StringIO() + with contextlib.redirect_stdout(output): + exit_code = cli_main(refresh_argv) + payload = json.loads(output.getvalue()) + assert exit_code == 0, payload + refresh_payloads.append(payload) + assert refresh_payloads[0]["appended"] is True + assert refresh_payloads[1]["appended"] is False + assert refresh_payloads[1]["idempotent_replay"] is True + + spend_key = f"{turn_key}:quota_spend" + spend_argv = [ + "--registry", + str(registry), + "--runtime-root", + str(runtime), + "--format", + "json", + "quota", + "spend-slot", + "--goal-id", + "loopx-turn-fixture", + "--agent-id", + "codex-fixture", + "--source", + "adapter", + "--execute", + "--scan-root", + str(project), + "--turn-effect-key", + spend_key, + *fence_args, + ] + spend_payloads: list[dict[str, object]] = [] + for _ in range(2): + output = io.StringIO() + with contextlib.redirect_stdout(output): + exit_code = cli_main(spend_argv) + payload = json.loads(output.getvalue()) + assert exit_code == 0, payload + spend_payloads.append(payload) + assert spend_payloads[0]["appended"] is True + assert spend_payloads[1]["appended"] is False + assert spend_payloads[1]["idempotent_replay"] is True + + release_task_lease( + runtime_root=runtime, + goal_id="loopx-turn-fixture", + todo_id="todo_fixture0001", + owner="codex-fixture", + idempotency_key=idempotency_key, + expected_version=int(lease["version"]), + ) + acquire_task_lease( + registry_path=registry, + runtime_root=runtime, + goal_id="loopx-turn-fixture", + todo_id="todo_fixture0001", + owner="codex-fixture", + idempotency_key="turn:replacement-worker", + write_scopes=["docs/**"], + ) + stale_output = io.StringIO() + stale_argv = [ + *refresh_argv, + "--classification", + "fixture_stale_write_must_not_append", + "--turn-effect-key", + f"{turn_key}:stale_writeback", + ] + with contextlib.redirect_stdout(stale_output): + stale_exit = cli_main(stale_argv) + stale = json.loads(stale_output.getvalue()) + assert stale_exit == 1, stale + assert stale["appended"] is False + assert "stale" in str(stale["error"]).lower() + + +def test_turn_run_once_cli_rejects_execute_without_selected_todo( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from loopx.cli_commands.turn import build_turn_envelope as real_build_turn_envelope + + project, runtime, registry = _write_live_fixture(tmp_path) + host_project = tmp_path / "isolated-host-workspace" + host_project.mkdir() + + def envelope_without_selected_todo( + *args: object, + **kwargs: object, + ) -> dict[str, object]: + envelope = real_build_turn_envelope(*args, **kwargs) + action = envelope.get("action") + assert isinstance(action, dict) + action.pop("selected_todo", None) + return envelope + + monkeypatch.setattr( + "loopx.cli_commands.turn.build_turn_envelope", + envelope_without_selected_todo, + ) + output = io.StringIO() + with contextlib.redirect_stdout(output): + exit_code = cli_main( + [ + "--registry", + str(registry), + "--runtime-root", + str(runtime), + "--format", + "json", + "turn", + "run-once", + "--goal-id", + "loopx-turn-fixture", + "--agent-id", + "codex-fixture", + "--project", + str(host_project), + "--host-adapter-command-json", + json.dumps([sys.executable, "-c", "raise SystemExit(9)"]), + "--scan-root", + str(project), + "--no-global-sync", + "--execute", + ] + ) + + payload = json.loads(output.getvalue()) + assert exit_code == 1, payload + assert payload["effects"]["host_invoked"] is False + assert "requires one selected todo" in payload["error"] + assert not (runtime / "goals" / "loopx-turn-fixture" / "task-leases").exists() + + def test_turn_run_once_commits_independently_validated_progress( tmp_path: Path, ) -> None: + project, runtime, registry = _write_live_fixture(tmp_path) + envelope = _envelope() + envelope["goal_id"] = "loopx-turn-fixture" plan = build_loopx_turn_plan( - _envelope(), + envelope, host="generic-cli", execution_mode="isolated-headless", ) + transaction = plan["transaction"] + assert isinstance(transaction, dict) + turn_key = str(transaction["turn_key"]) + lease_controller = TurnLeaseController( + registry_path=registry, + runtime_root=runtime, + goal_id="loopx-turn-fixture", + todo_id="todo_fixture0001", + owner="codex-fixture", + idempotency_key=f"turn:{turn_key}", + write_scopes=["docs/**"], + ) def host_runner(request: dict[str, object]) -> dict[str, object]: return { @@ -1302,9 +1725,9 @@ def host_runner(request: dict[str, object]) -> dict[str, object]: execution = run_loopx_turn_once( plan, host_runner=host_runner, - project=tmp_path, - runtime_root=tmp_path / "runtime", - goal_id="fixture-goal", + project=project, + runtime_root=runtime, + goal_id="loopx-turn-fixture", timeout_seconds=10, execute=True, task_validator=lambda _plan, _result: { @@ -1313,14 +1736,15 @@ def host_runner(request: dict[str, object]) -> dict[str, object]: "summary": "intermediate fixture progress is independently valid", "exit_code": 10, }, - writeback=lambda _result: {"ok": True, "appended": True}, - spend=lambda: {"ok": True, "appended": True, "slots": 1}, - scheduler=lambda _spend: { + writeback=lambda _effect, _result: {"ok": True, "appended": True}, + spend=lambda _effect: {"ok": True, "appended": True, "slots": 1}, + scheduler=lambda _effect, _spend: { "disposition": "outer_controller_owned", "completed": True, "acknowledged": False, "apply_needed": False, }, + lease_controller=lease_controller, ) assert execution["status"] == "committed" @@ -1413,6 +1837,17 @@ def test_turn_run_once_cli_rejects_unproven_host_claim_before_writeback( assert payload["effects"]["quota_spent"] is False assert state_path.read_text(encoding="utf-8") == before_state assert not (runtime / "goals" / "loopx-turn-fixture" / "runs").exists() + lease = read_lease( + task_lease_path( + runtime_root=runtime, + goal_id="loopx-turn-fixture", + todo_id="todo_fixture0001", + ) + ) + assert lease is not None + assert lease["status"] == "active" + assert lease["owner"] == "codex-fixture" + assert lease["idempotency_key"] == f"turn:{payload['resume_turn_key']}" @pytest.mark.parametrize( @@ -1471,6 +1906,7 @@ def test_turn_run_once_cli_uses_built_in_codex_host_and_typed_writeback( def adaptive_turn_envelope(*args: object, **kwargs: object) -> dict[str, object]: envelope = real_build_turn_envelope(*args, **kwargs) + primary_todo = dict(envelope["action"]["selected_todo"]) envelope["action"]["selected_todo"] = { "todo_id": "todo_stale_selection", "text": "A stale pre-orchestration selection", @@ -1479,6 +1915,10 @@ def adaptive_turn_envelope(*args: object, **kwargs: object) -> dict[str, object] "schema_version": "task_orchestration_contract_v2", "mode": "adaptive", "primary_todo_id": "todo_fixture0001", + "primary_todo": { + **primary_todo, + "source": "task_orchestration_contract.primary_todo", + }, "eligible_child_lanes": [], } return envelope @@ -1572,6 +2012,15 @@ def fake_codex_host(request: dict[str, object], **_kwargs: object) -> dict[str, assert updated_todo_ids == ( [] if result_kind == "validated_progress" else ["todo_fixture0001"] ) + lease = read_lease( + task_lease_path( + runtime_root=runtime, + goal_id="loopx-turn-fixture", + todo_id="todo_fixture0001", + ) + ) + assert lease is not None + assert lease["write_scopes"] == ["docs/**"] state = ( project / ".codex" diff --git a/tests/test_loopx_turn_executor.py b/tests/test_loopx_turn_executor.py index 1e2d481ee..be667c4c9 100644 --- a/tests/test_loopx_turn_executor.py +++ b/tests/test_loopx_turn_executor.py @@ -2,21 +2,26 @@ import json import sys +import threading +import time from pathlib import Path import pytest from loopx.control_plane.turn_driver import ( LOOPX_TURN_RESULT_SCHEMA_VERSION, + TurnEffectEnvelope, + TurnLeaseController, build_loopx_turn_plan, + load_turn_events, load_loopx_turn_plan_from_journal, - run_loopx_turn_once, + rebuild_turn_projection, + run_loopx_turn_once as _run_loopx_turn_once, validate_loopx_turn_host_result, ) -from loopx.control_plane.turn_driver.executor import ( - BuiltInHostError, - _task_validation_stage, -) +from loopx.control_plane.work_items.task_lease import inspect_task_lease +from loopx.control_plane.work_items.task_lease import transfer_task_lease +from loopx.control_plane.turn_driver.executor import BuiltInHostError from loopx.control_plane.turn_driver.settlement import execute_turn_driver_settlement from loopx.control_plane.turn_driver.transaction import TRANSACTION_PHASES @@ -109,32 +114,6 @@ def _host_result(plan: dict[str, object], *, kind: str = "validated_progress") - return result -def test_task_validation_stage_reads_result_kind_through_effect_turn( - tmp_path: Path, -) -> None: - plan = _plan() - result = _host_result(plan, kind="wait") - journal = { - "status": "in_progress", - "completed_phases": list(TRANSACTION_PHASES[:2]), - } - - completed, payload = _task_validation_stage( - plan, - result, - task_validator=None, - completed_phases=list(TRANSACTION_PHASES[:2]), - journal=journal, - journal_path=tmp_path / "journal.json", - effects={}, - ) - - assert completed == list(TRANSACTION_PHASES[:3]) - assert journal["status"] == "stopped" - assert payload is not None - assert payload["status"] == "stopped" - - def test_typed_settlement_fails_closed_when_journal_receipt_payload_is_missing() -> None: transaction = _plan()["transaction"] assert isinstance(transaction, dict) @@ -228,10 +207,19 @@ def scheduler(_spend: dict[str, object]) -> dict[str, object]: def _journal(runtime_root: Path) -> dict[str, object]: journal_paths = list( - (runtime_root / "goals" / "fixture-goal" / "turns").glob("*.json") + (runtime_root / "goals" / "fixture-goal" / "turn-journals").glob("*.jsonl") ) assert len(journal_paths) == 1 - return json.loads(journal_paths[0].read_text(encoding="utf-8")) + turn_key = f"sha256:{journal_paths[0].stem}" + events = load_turn_events(runtime_root, "fixture-goal", turn_key) + states = [ + event["payload"]["state"] + for event in events + if isinstance(event.get("payload"), dict) + and isinstance(event["payload"].get("state"), dict) + ] + assert states + return states[-1] def _passing_validator( @@ -245,6 +233,424 @@ def _passing_validator( } +def _write_lease_registry(tmp_path: Path) -> Path: + state = tmp_path / "ACTIVE_GOAL_STATE.md" + state.write_text( + "\n".join( + [ + "---", + "status: active", + "updated_at: 2026-01-01T00:00:00+00:00", + "---", + "", + "# Fixture Goal", + "", + "## Agent Todo", + "", + "- [ ] [P0] Advance one public fixture.", + " ", + "", + ] + ), + encoding="utf-8", + ) + registry = tmp_path / "registry.json" + registry.write_text( + json.dumps( + { + "schema_version": 1, + "common_runtime_root": str(tmp_path / "runtime"), + "goals": [ + { + "id": "fixture-goal", + "status": "active", + "repo": str(tmp_path), + "state_file": state.name, + "coordination": { + "registered_agents": ["codex-fixture"], + }, + } + ], + } + ) + + "\n", + encoding="utf-8", + ) + return registry + + +_TURN_CONTROLLERS: dict[tuple[str, str], TurnLeaseController] = {} + + +def run_loopx_turn_once( + plan: dict[str, object], + **kwargs: object, +) -> dict[str, object]: + if kwargs.get("execute") is True and "lease_controller" not in kwargs: + runtime_root = kwargs.get("runtime_root") + assert isinstance(runtime_root, Path) + transaction = plan.get("transaction") + assert isinstance(transaction, dict) + turn_key = str(transaction["turn_key"]) + key = (str(runtime_root), turn_key) + controller = _TURN_CONTROLLERS.get(key) + if controller is None: + registry = _write_lease_registry(runtime_root.parent) + controller = TurnLeaseController( + registry_path=registry, + runtime_root=runtime_root, + goal_id="fixture-goal", + todo_id="todo_fixture0001", + owner="codex-fixture", + idempotency_key=f"turn:{turn_key}", + write_scopes=["docs/**"], + ttl_seconds=120, + ) + _TURN_CONTROLLERS[key] = controller + kwargs["lease_controller"] = controller + return _run_loopx_turn_once(plan, **kwargs) + + +def test_turn_lease_controller_preserves_fence_across_renew_and_release( + tmp_path: Path, +) -> None: + registry = _write_lease_registry(tmp_path) + controller = TurnLeaseController( + registry_path=registry, + runtime_root=tmp_path / "runtime", + goal_id="fixture-goal", + todo_id="todo_fixture0001", + owner="codex-fixture", + idempotency_key="turn:fixture", + write_scopes=["docs/**"], + ttl_seconds=120, + ) + + acquired = controller.acquire() + renewed = controller.renew(acquired) + controller.require_current(renewed) + controller.release(renewed) + + assert renewed.token == acquired.token + assert renewed.generation == acquired.generation + assert renewed.version > acquired.version + inspected = inspect_task_lease( + registry_path=registry, + runtime_root=tmp_path / "runtime", + goal_id="fixture-goal", + todo_id="todo_fixture0001", + ) + assert inspected["active"] is False + assert inspected["lease"]["status"] == "released" + + +def test_turn_lease_heartbeat_renews_without_changing_fencing_token( + tmp_path: Path, +) -> None: + registry = _write_lease_registry(tmp_path) + controller = TurnLeaseController( + registry_path=registry, + runtime_root=tmp_path / "runtime", + goal_id="fixture-goal", + todo_id="todo_fixture0001", + owner="codex-fixture", + idempotency_key="turn:fixture", + write_scopes=["docs/**"], + ttl_seconds=120, + heartbeat_interval_seconds=0.01, + ) + acquired = controller.acquire() + + with controller.heartbeat(acquired) as current: + deadline = time.monotonic() + 1 + while current().version == acquired.version and time.monotonic() < deadline: + time.sleep(0.005) + renewed = current() + + assert renewed.version > acquired.version + assert renewed.token == acquired.token + assert renewed.generation == acquired.generation + controller.release(renewed) + + +def test_turn_effect_guard_serializes_lease_transfer_with_effect_commit( + tmp_path: Path, +) -> None: + registry = _write_lease_registry(tmp_path) + controller = TurnLeaseController( + registry_path=registry, + runtime_root=tmp_path / "runtime", + goal_id="fixture-goal", + todo_id="todo_fixture0001", + owner="codex-fixture", + idempotency_key="turn:fixture:a", + write_scopes=["docs/**"], + ttl_seconds=120, + ) + acquired = controller.acquire() + effect_entered = threading.Event() + allow_effect_exit = threading.Event() + transfer_finished = threading.Event() + + def hold_effect() -> None: + with controller.effect_guard(acquired): + effect_entered.set() + assert allow_effect_exit.wait(timeout=2) + + def transfer() -> None: + transfer_task_lease( + registry_path=registry, + runtime_root=tmp_path / "runtime", + goal_id="fixture-goal", + todo_id="todo_fixture0001", + owner="codex-fixture", + idempotency_key="turn:fixture:a", + new_owner="codex-fixture", + new_idempotency_key="turn:fixture:b", + expected_version=acquired.version, + ) + transfer_finished.set() + + effect_thread = threading.Thread(target=hold_effect) + effect_thread.start() + assert effect_entered.wait(timeout=1) + transfer_thread = threading.Thread(target=transfer) + transfer_thread.start() + time.sleep(0.05) + + assert transfer_finished.is_set() is False + allow_effect_exit.set() + effect_thread.join(timeout=2) + transfer_thread.join(timeout=2) + assert transfer_finished.is_set() is True + + +def test_run_once_rejects_stale_worker_after_lease_transfer( + tmp_path: Path, +) -> None: + plan = _plan() + transaction = plan["transaction"] + assert isinstance(transaction, dict) + turn_key = str(transaction["turn_key"]) + registry = _write_lease_registry(tmp_path) + runtime_root = tmp_path / "runtime" + controller_a = TurnLeaseController( + registry_path=registry, + runtime_root=runtime_root, + goal_id="fixture-goal", + todo_id="todo_fixture0001", + owner="codex-fixture", + idempotency_key=f"turn:{turn_key}:a", + write_scopes=["docs/**"], + ttl_seconds=120, + ) + controller_b = TurnLeaseController( + registry_path=registry, + runtime_root=runtime_root, + goal_id="fixture-goal", + todo_id="todo_fixture0001", + owner="codex-fixture", + idempotency_key=f"turn:{turn_key}:b", + write_scopes=["docs/**"], + ttl_seconds=120, + ) + host_a_entered = threading.Event() + release_host_a = threading.Event() + b_after_scheduler_apply = threading.Event() + release_worker_b = threading.Event() + calls = {"writeback": 0, "spend": 0, "scheduler": 0} + results: dict[str, dict[str, object]] = {} + + def host_a(_request: dict[str, object]) -> dict[str, object]: + host_a_entered.set() + assert release_host_a.wait(timeout=3) + result = _host_result(plan) + result["summary"] = "Worker A stale result must never be journaled." + return result + + def host_b(_request: dict[str, object]) -> dict[str, object]: + result = _host_result(plan) + result["summary"] = "Worker B owns the durable result." + return result + + def writeback( + envelope: TurnEffectEnvelope, + _result: dict[str, object], + ) -> dict[str, object]: + assert envelope.phase_key == f"{turn_key}:durable_writeback" + calls["writeback"] += 1 + return {"ok": True, "appended": True} + + def spend(envelope: TurnEffectEnvelope) -> dict[str, object]: + assert envelope.phase_key == f"{turn_key}:quota_spend" + calls["spend"] += 1 + return {"ok": True, "appended": True} + + def scheduler( + envelope: TurnEffectEnvelope, + _spend: dict[str, object], + ) -> dict[str, object]: + assert envelope.phase_key == f"{turn_key}:scheduler_apply" + calls["scheduler"] += 1 + return {"completed": True, "acknowledged": True} + + common = { + "project": tmp_path, + "runtime_root": runtime_root, + "goal_id": "fixture-goal", + "timeout_seconds": 5, + "execute": True, + "task_validator": _passing_validator, + "writeback": writeback, + "spend": spend, + "scheduler": scheduler, + } + + def run_a() -> None: + results["a"] = run_loopx_turn_once( + plan, + host_runner=host_a, + lease_controller=controller_a, + **common, + ) + + def pause_b(phase: str) -> None: + if phase == "after_scheduler_apply": + b_after_scheduler_apply.set() + assert release_worker_b.wait(timeout=3) + + def run_b() -> None: + results["b"] = run_loopx_turn_once( + plan, + host_runner=host_b, + lease_controller=controller_b, + fault_injector=pause_b, + **common, + ) + + worker_a = threading.Thread(target=run_a) + worker_a.start() + assert host_a_entered.wait(timeout=2) + active = inspect_task_lease( + registry_path=registry, + runtime_root=runtime_root, + goal_id="fixture-goal", + todo_id="todo_fixture0001", + )["lease"] + assert isinstance(active, dict) + transfer_task_lease( + registry_path=registry, + runtime_root=runtime_root, + goal_id="fixture-goal", + todo_id="todo_fixture0001", + owner="codex-fixture", + idempotency_key=f"turn:{turn_key}:a", + new_owner="codex-fixture", + new_idempotency_key=f"turn:{turn_key}:b", + expected_version=int(active["version"]), + ) + worker_b = threading.Thread(target=run_b) + worker_b.start() + assert b_after_scheduler_apply.wait(timeout=2) + release_host_a.set() + worker_a.join(timeout=2) + + assert results["a"]["status"] == "failed_closed" + assert results["a"]["reason_code"] == "stale_fencing_token" + release_worker_b.set() + worker_b.join(timeout=2) + assert results["b"]["status"] == "committed" + assert calls == {"writeback": 1, "spend": 1, "scheduler": 1} + events = load_turn_events(runtime_root, "fixture-goal", turn_key) + assert "Worker A stale result" not in json.dumps(events) + projection = rebuild_turn_projection(runtime_root, "fixture-goal", turn_key) + assert projection["fencing_token"] == results["b"]["fencing_token"] + + +@pytest.mark.parametrize( + "crash_phase", + [ + "after_host", + "after_validation", + "after_writeback", + "after_spend", + "after_scheduler_apply", + ], +) +def test_run_once_resumes_after_each_durable_phase_without_duplicate_effects( + tmp_path: Path, + crash_phase: str, +) -> None: + plan = _plan() + transaction = plan["transaction"] + assert isinstance(transaction, dict) + turn_key = str(transaction["turn_key"]) + registry = _write_lease_registry(tmp_path) + controller = TurnLeaseController( + registry_path=registry, + runtime_root=tmp_path / "runtime", + goal_id="fixture-goal", + todo_id="todo_fixture0001", + owner="codex-fixture", + idempotency_key=f"turn:{turn_key}", + write_scopes=["docs/**"], + ttl_seconds=120, + ) + calls = {"host": 0, "writeback": 0, "spend": 0, "scheduler": 0} + + def host(_request: dict[str, object]) -> dict[str, object]: + calls["host"] += 1 + return _host_result(plan) + + def writeback( + _envelope: TurnEffectEnvelope, + _result: dict[str, object], + ) -> dict[str, object]: + calls["writeback"] += 1 + return {"ok": True, "appended": True} + + def spend(_envelope: TurnEffectEnvelope) -> dict[str, object]: + calls["spend"] += 1 + return {"ok": True, "appended": True} + + def scheduler( + _envelope: TurnEffectEnvelope, + _spend: dict[str, object], + ) -> dict[str, object]: + calls["scheduler"] += 1 + return {"completed": True, "acknowledged": True} + + def crash_after_persisted_phase(phase: str) -> None: + if phase == crash_phase: + raise SystemExit(91) + + common = { + "host_runner": host, + "project": tmp_path, + "runtime_root": tmp_path / "runtime", + "goal_id": "fixture-goal", + "timeout_seconds": 5, + "execute": True, + "task_validator": _passing_validator, + "writeback": writeback, + "spend": spend, + "scheduler": scheduler, + "lease_controller": controller, + } + with pytest.raises(SystemExit, match="91"): + run_loopx_turn_once( + plan, + fault_injector=crash_after_persisted_phase, + **common, + ) + + recovered = run_loopx_turn_once(plan, **common) + + assert recovered["status"] == "committed" + assert calls == {"host": 1, "writeback": 1, "spend": 1, "scheduler": 1} + + def test_host_result_requires_bounded_public_material_fields() -> None: plan = _plan() result = _host_result(plan) @@ -1134,6 +1540,15 @@ def test_run_once_stops_without_writeback_or_spend(tmp_path: Path) -> None: assert payload["ok"] is True assert payload["status"] == "stopped" assert payload["receipt"]["status"] == "stopped" + assert payload["lease_release"] == {"released": True} + inspected = inspect_task_lease( + registry_path=tmp_path / "registry.json", + runtime_root=tmp_path / "runtime", + goal_id="fixture-goal", + todo_id="todo_fixture0001", + ) + assert inspected["active"] is False + assert inspected["lease"]["status"] == "released" assert calls == {"writeback": 0, "spend": 0, "scheduler": 0} diff --git a/tests/test_loopx_turn_journal.py b/tests/test_loopx_turn_journal.py new file mode 100644 index 000000000..4ed631773 --- /dev/null +++ b/tests/test_loopx_turn_journal.py @@ -0,0 +1,184 @@ +from __future__ import annotations + +import json +from contextlib import contextmanager +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +from loopx.control_plane.turn_driver.journal import ( + TurnJournalError, + append_turn_event, + load_turn_events, + rebuild_turn_projection, + turn_journal_path, + turn_projection_path, +) +from loopx.control_plane.turn_driver import journal +from loopx.control_plane.work_items import task_lease +from loopx.control_plane.work_items.task_lease import ( + TaskLeaseError, + acquire_task_lease, + task_lease_fencing_token, +) + + +TURN_KEY = "sha256:" + "a" * 64 + + +@pytest.fixture +def active_fence( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> dict[str, object]: + now = datetime(2026, 8, 10, tzinfo=timezone.utc) + monkeypatch.setattr(task_lease, "now_utc", lambda: now) + monkeypatch.setattr(task_lease, "require_task_lease_owner_allowed", lambda **_: {}) + monkeypatch.setattr(task_lease, "active_conflicts", lambda **_: []) + acquired = acquire_task_lease( + registry_path=tmp_path / "registry.json", + runtime_root=tmp_path, + goal_id="fixture-goal", + todo_id="todo_fixture", + owner="codex-a", + idempotency_key="turn:fixture:a", + ttl_seconds=120, + write_scopes=["src/**"], + )["lease"] + return { + "todo_id": acquired["todo_id"], + "owner": acquired["owner"], + "idempotency_key": acquired["idempotency_key"], + "token": task_lease_fencing_token(acquired), + } + + +def append_fixture_event( + tmp_path: Path, + fence: dict[str, object], + *, + payload: dict[str, object] | None = None, +) -> dict[str, object]: + return append_turn_event( + runtime_root=tmp_path, + goal_id="fixture-goal", + turn_key=TURN_KEY, + event_type="phase_completed", + phase_key=f"{TURN_KEY}:validation:completed", + fencing=fence, + payload=payload or {"phase": "validation", "receipt_ref": "receipt:fixture"}, + ) + + +def test_append_is_idempotent_for_same_phase_key( + tmp_path: Path, + active_fence: dict[str, object], +) -> None: + first = append_fixture_event(tmp_path, active_fence) + replay = append_fixture_event(tmp_path, active_fence) + + assert replay == first + assert len(load_turn_events(tmp_path, "fixture-goal", TURN_KEY)) == 1 + + +def test_phase_key_conflict_fails_closed( + tmp_path: Path, + active_fence: dict[str, object], +) -> None: + append_fixture_event(tmp_path, active_fence) + + with pytest.raises(TurnJournalError, match="phase key conflict"): + append_fixture_event( + tmp_path, + active_fence, + payload={"phase": "quota_spend", "receipt_ref": "receipt:other"}, + ) + + +def test_stale_fence_cannot_append( + tmp_path: Path, + active_fence: dict[str, object], +) -> None: + stale = {**active_fence, "token": "fence:" + "0" * 64} + + with pytest.raises(TaskLeaseError) as error: + append_fixture_event(tmp_path, stale) + + assert error.value.code == "stale_fencing_token" + assert not turn_journal_path(tmp_path, "fixture-goal", TURN_KEY).exists() + + +def test_projection_rebuilds_from_jsonl_after_projection_loss( + tmp_path: Path, + active_fence: dict[str, object], +) -> None: + append_fixture_event(tmp_path, active_fence) + turn_projection_path(tmp_path, "fixture-goal", TURN_KEY).unlink() + + projection = rebuild_turn_projection(tmp_path, "fixture-goal", TURN_KEY) + + assert projection["last_phase"] == "validation" + assert projection["event_count"] == 1 + assert projection["last_event_hash"] + + +def test_idempotent_replay_restores_a_missing_projection( + tmp_path: Path, + active_fence: dict[str, object], +) -> None: + first = append_fixture_event(tmp_path, active_fence) + projection_path = turn_projection_path(tmp_path, "fixture-goal", TURN_KEY) + projection_path.unlink() + + assert append_fixture_event(tmp_path, active_fence) == first + assert projection_path.exists() + + +def test_append_obeys_lease_then_journal_lock_order( + tmp_path: Path, + active_fence: dict[str, object], + monkeypatch: pytest.MonkeyPatch, +) -> None: + entered: list[str] = [] + active: list[str] = [] + + @contextmanager + def record_lock(_path: Path, **kwargs: object): + operation = str(kwargs.get("operation") or "") + entered.append(operation) + active.append(operation) + if operation == "turn_journal_append": + assert active == ["turn_journal_lease_fence", "turn_journal_append"] + try: + yield _path + finally: + active.pop() + + monkeypatch.setattr(journal, "exclusive_file_lock", record_lock) + + append_fixture_event(tmp_path, active_fence) + + assert entered == ["turn_journal_lease_fence", "turn_journal_append"] + + +@pytest.mark.parametrize("corruption", ["truncated", "hash", "schema"]) +def test_corrupt_journal_fails_closed( + tmp_path: Path, + active_fence: dict[str, object], + corruption: str, +) -> None: + event = append_fixture_event(tmp_path, active_fence) + path = turn_journal_path(tmp_path, "fixture-goal", TURN_KEY) + if corruption == "truncated": + path.write_bytes(path.read_bytes().rstrip(b"\n")[:-1]) + else: + changed = dict(event) + if corruption == "hash": + changed["payload"] = {"phase": "tampered"} + else: + changed["schema_version"] = "unsupported" + path.write_text(json.dumps(changed) + "\n", encoding="utf-8") + + with pytest.raises(TurnJournalError): + load_turn_events(tmp_path, "fixture-goal", TURN_KEY) diff --git a/tests/test_loopx_turn_transaction.py b/tests/test_loopx_turn_transaction.py index 446e628ad..329931a68 100644 --- a/tests/test_loopx_turn_transaction.py +++ b/tests/test_loopx_turn_transaction.py @@ -5,6 +5,7 @@ from loopx.control_plane.turn_driver import ( LOOPX_TURN_RESULT_SCHEMA_VERSION, LoopXTurnResultKind, + TurnEffectEnvelope, build_loopx_turn_transaction_plan, loopx_turn_execution_committed, loopx_turn_execution_has_durable_effects, @@ -275,3 +276,34 @@ def test_public_execution_outcome_predicates_share_transaction_semantics() -> No repair["effects"] = {"state_written": True, "quota_spent": False} assert loopx_turn_execution_has_durable_effects(repair) is True + + +def test_failed_closed_receipt_stops_at_next_uncompleted_phase() -> None: + plan = _plan() + + receipt = validate_loopx_turn_receipt( + plan, + _result( + plan, + result_kind=LoopXTurnResultKind.FAILED_CLOSED, + completed_phases=["host_execute", "typed_result"], + failed_phase="validation", + ), + ) + + assert receipt["ok"] is True + assert receipt["status"] == "failed" + assert receipt["next_phase"] == "validation" + assert receipt["commit_eligibility"]["quota_spend"] is False + + +def test_turn_effect_envelope_rejects_noncanonical_phase_key() -> None: + plan = _plan() + + with pytest.raises(ValueError, match="phase_key"): + TurnEffectEnvelope( + turn_key=str(plan["turn_key"]), + phase="durable_writeback", + phase_key="sha256:wrong:durable_writeback", + fencing_token="fence:" + "a" * 64, + ) diff --git a/tests/test_skillsbench_turn_runtime.py b/tests/test_skillsbench_turn_runtime.py index e67c8ca21..151637c76 100644 --- a/tests/test_skillsbench_turn_runtime.py +++ b/tests/test_skillsbench_turn_runtime.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import shlex import subprocess import threading import time @@ -24,6 +25,9 @@ sync_skillsbench_loopx_turn_trace_into_compact, ) from loopx.control_plane.turn_driver import executor as turn_executor +from loopx.control_plane.turn_driver import build_loopx_turn_plan +from loopx.control_plane.turn_driver.journal import build_turn_event +from loopx.control_plane.work_items.task_lease import task_lease_fencing_token def test_agent_prompt_runner_alias_is_bootstrap_python_39_safe() -> None: @@ -42,6 +46,181 @@ def _config(tmp_path: Path) -> runtime.SkillsBenchTurnRuntimeConfig: ) +def _synthetic_turn_plan( + *, + turn_instance_id: str | None = None, +) -> dict[str, Any]: + return build_loopx_turn_plan( + { + "ok": True, + "schema_version": "loopx_turn_envelope_v0", + "goal_id": "synthetic-goal", + "agent_id": "synthetic-agent", + "should_run": True, + "effective_action": "normal_run", + "action": { + "must_attempt": True, + "delivery_allowed": True, + "quiet_noop_allowed": False, + "selected_todo": { + "todo_id": "todo_fixture0001", + "text": "Advance one public fixture", + "required_write_scopes": ["src/**"], + }, + }, + "user": { + "action_required": False, + "open_count": 0, + "notify": "DONT_NOTIFY", + }, + "writeback": {"spend_after_validation": True}, + "scheduler": {"action": "run_now"}, + "action_signature": { + "matches": True, + "source_hash": "sha256:fixture", + "envelope_hash": "sha256:fixture", + }, + "compaction": {"within_budget": True}, + }, + host="generic-cli", + execution_mode="isolated-headless", + turn_instance_id=turn_instance_id, + ) + + +def test_real_turn_path_uses_scored_workspace_lease_journal_and_effect_fence( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + plan = _synthetic_turn_plan() + transaction = plan["transaction"] + assert isinstance(transaction, dict) + turn_key = str(transaction["turn_key"]) + lease = { + "schema_version": "task_lease_v0", + "goal_id": "synthetic-goal", + "todo_id": "todo_fixture0001", + "owner": "synthetic-agent", + "idempotency_key": f"turn:{turn_key}", + "write_scopes": ["src/**"], + "status": "active", + "version": 1, + "fencing_generation": 1, + "acquired_at": "2026-08-11T00:00:00+00:00", + "renewed_at": "2026-08-11T00:00:00+00:00", + "expires_at": "2099-08-11T00:00:00+00:00", + } + remote_events: list[dict[str, Any]] = [] + bridge_commands: list[str] = [] + host_calls: list[str] = [] + + class ScoredWorkspaceBridge: + def __init__(self, _config: Any) -> None: + self.meaningful_operation_count = 0 + + def exec( + self, + command: str, + *, + meaningful: bool = False, + allow_nonzero: bool = False, + ) -> dict[str, Any]: + bridge_commands.append(command) + if allow_nonzero: + if meaningful: + self.meaningful_operation_count += 1 + return {"ok": True, "exit_code": 0, "stdout": "", "elapsed_ms": 1} + return {"ok": False, "exit_code": 3, "elapsed_ms": 1} + return {"ok": True, "exit_code": 0, "stdout": "", "elapsed_ms": 1} + + def loopx_json(self, command: str) -> dict[str, Any]: + bridge_commands.append(command) + argv = shlex.split(command) + if "turn" in argv and "plan" in argv: + return plan + if "task-lease" in argv: + action = argv[argv.index("task-lease") + 1] + if action == "renew": + lease["version"] = int(lease["version"]) + 1 + elif action == "release": + lease["status"] = "released" + return { + "ok": True, + "active": lease["status"] == "active", + "lease": dict(lease), + } + if "journal-read" in argv: + return {"ok": True, "events": list(remote_events)} + if "journal-append" in argv: + request = json.loads(argv[argv.index("--event-json") + 1]) + fencing = request["fencing"] + assert fencing["token"] == task_lease_fencing_token(lease) + event = build_turn_event( + turn_key=turn_key, + goal_id="synthetic-goal", + event_type=request["event_type"], + phase_key=request["phase_key"], + fencing_token=fencing["token"], + payload=request["payload"], + ) + prior = next( + ( + item + for item in remote_events + if item["phase_key"] == event["phase_key"] + ), + None, + ) + if prior is None: + remote_events.append(event) + else: + assert prior == event + return {"ok": True, "event": event} + if "refresh-state" in argv: + assert argv[argv.index("--turn-effect-key") + 1] == ( + f"{turn_key}:durable_writeback" + ) + assert argv[argv.index("--turn-fencing-token") + 1] == ( + task_lease_fencing_token(lease) + ) + return {"ok": True, "appended": True} + if "spend-slot" in argv: + assert argv[argv.index("--turn-effect-key") + 1] == ( + f"{turn_key}:quota_spend" + ) + assert argv[argv.index("--turn-fencing-token") + 1] == ( + task_lease_fencing_token(lease) + ) + return {"ok": True, "appended": True, "slots": 1} + if "should-run" in argv: + return { + "scheduler_hint": { + "execution_phase": { + "disposition": "outer_controller_owned", + "completed": True, + "acknowledged": False, + "apply_needed": False, + } + } + } + raise AssertionError(f"unexpected scored-workspace command: {command}") + + monkeypatch.setattr(runtime, "SkillsBenchTurnBridge", ScoredWorkspaceBridge) + + execution, validation = runtime.run_skillsbench_loopx_turn( + prompt="synthetic prompt", + agent_runner=lambda prompt: host_calls.append(prompt) or "done", + config=_config(tmp_path), + ) + + assert execution["status"] == "committed" + assert validation["status"] == "passed" + assert host_calls == ["synthetic prompt"] + assert lease["status"] == "released" + assert remote_events + assert not (tmp_path / "goals" / "synthetic-goal" / "turn-journals").exists() + + def _install_sequence_runtime( monkeypatch: pytest.MonkeyPatch, validation_runs: list[list[int] | tuple[int, ...]], @@ -107,7 +286,7 @@ def fake_plan( ) -> dict[str, Any]: plan_instance_ids.append(turn_instance_id) sequence_baseline_paths.append(_config.sequence_baseline_path) - return {"ok": True, "turn_key": turn_instance_id} + return _synthetic_turn_plan(turn_instance_id=turn_instance_id) def fake_turn_once( plan: dict[str, Any], @@ -119,16 +298,42 @@ def fake_turn_once( scheduler: Any, **_kwargs: Any, ) -> dict[str, Any]: - result = host_runner({"turn_key": plan["turn_key"]}) + transaction = plan["transaction"] + assert isinstance(transaction, dict) + turn_key = str(transaction["turn_key"]) + result = host_runner({"turn_key": turn_key}) validation = turn_executor._run_task_validator( plan, result, validator=task_validator, ) assert validation["status"] in {"progress", "passed"} - writeback_payload = writeback(result) - spend_payload = spend() - scheduler(spend_payload) + writeback_payload = writeback( + runtime.TurnEffectEnvelope( + turn_key=turn_key, + phase="durable_writeback", + phase_key=f"{turn_key}:durable_writeback", + fencing_token="fence:" + "0" * 64, + ), + result, + ) + spend_payload = spend( + runtime.TurnEffectEnvelope( + turn_key=turn_key, + phase="quota_spend", + phase_key=f"{turn_key}:quota_spend", + fencing_token="fence:" + "0" * 64, + ) + ) + scheduler( + runtime.TurnEffectEnvelope( + turn_key=turn_key, + phase="scheduler_apply", + phase_key=f"{turn_key}:scheduler_apply", + fencing_token="fence:" + "0" * 64, + ), + spend_payload, + ) return { "status": "committed", "validation": validation, @@ -185,6 +390,72 @@ def test_nonzero_validation_probe_does_not_return_private_output( assert "private" not in json.dumps(result) +@pytest.mark.parametrize( + ("typed_field", "typed_value", "expected_category"), + [ + ("reason", "stale_fencing_token", "stale_fencing_token"), + ("error_code", "lease_not_active", "lease_not_active"), + ], +) +def test_bridge_classifies_typed_loopx_cli_failures( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + typed_field: str, + typed_value: str, + expected_category: str, +) -> None: + payload = { + "schema_version": runtime.SKILLSBENCH_BRIDGE_OPERATION_SCHEMA_VERSION, + "ok": False, + "exit_code": 2, + "stdout": json.dumps( + { + "ok": False, + typed_field: typed_value, + } + ), + "stderr": "", + } + monkeypatch.setattr( + runtime.subprocess, + "run", + lambda *args, **kwargs: SimpleNamespace( + returncode=0, + stdout=json.dumps(payload), + stderr="", + ), + ) + + with pytest.raises(runtime.SkillsBenchTurnBridgeError) as exc_info: + runtime.SkillsBenchTurnBridge(_config(tmp_path)).exec("quota spend-slot") + + assert exc_info.value.category == expected_category + + +def test_remote_lease_preserves_non_authority_bridge_failure_category( + tmp_path: Path, +) -> None: + class FailingBridge: + def loopx_json(self, _command: str) -> dict[str, Any]: + raise runtime.SkillsBenchTurnBridgeError( + "LoopX CLI is unavailable", + category="case_loopx_cli_missing", + ) + + controller = runtime.SkillsBenchTurnLeaseController( + bridge=FailingBridge(), + config=_config(tmp_path), + turn_key="sha256:" + "a" * 64, + todo_id="todo_fixture0001", + write_scopes=["src/**"], + ) + + with pytest.raises(runtime.SkillsBenchTurnBridgeError) as exc_info: + controller.acquire() + + assert exc_info.value.category == "case_loopx_cli_missing" + + def test_bridge_progress_receipt_requires_successful_task_content_change( tmp_path: Path, ) -> None: @@ -395,7 +666,11 @@ def fake_turn_once( } monkeypatch.setattr(runtime, "SkillsBenchTurnBridge", NonFileProgressBridge) - monkeypatch.setattr(runtime, "_turn_plan", lambda *_args, **_kwargs: {"ok": True}) + monkeypatch.setattr( + runtime, + "_turn_plan", + lambda *_args, **_kwargs: _synthetic_turn_plan(), + ) monkeypatch.setattr(runtime, "run_loopx_turn_once", fake_turn_once) evidence = { "schema_version": "skillsbench_bridge_task_progress_receipt_v0", @@ -478,7 +753,11 @@ def fake_turn_once( } monkeypatch.setattr(runtime, "SkillsBenchTurnBridge", BaselineSatisfiedBridge) - monkeypatch.setattr(runtime, "_turn_plan", lambda *_args: {"ok": True}) + monkeypatch.setattr( + runtime, + "_turn_plan", + lambda *_args: _synthetic_turn_plan(), + ) monkeypatch.setattr(runtime, "run_loopx_turn_once", fake_turn_once) execution, validation = runtime.run_skillsbench_loopx_turn( @@ -534,7 +813,11 @@ def fake_turn_once( } monkeypatch.setattr(runtime, "SkillsBenchTurnBridge", ProgressBridge) - monkeypatch.setattr(runtime, "_turn_plan", lambda *_args: {"ok": True}) + monkeypatch.setattr( + runtime, + "_turn_plan", + lambda *_args: _synthetic_turn_plan(), + ) monkeypatch.setattr(runtime, "run_loopx_turn_once", fake_turn_once) execution, validation = runtime.run_skillsbench_loopx_turn( @@ -601,7 +884,7 @@ def fake_turn_once( monkeypatch.setattr( runtime, "_turn_plan", - lambda *_args: {"ok": True, "turn_key": "synthetic-turn"}, + lambda *_args: _synthetic_turn_plan(), ) monkeypatch.setattr(runtime, "run_loopx_turn_once", fake_turn_once) diff --git a/tests/test_state_refresh.py b/tests/test_state_refresh.py new file mode 100644 index 000000000..0ea10f8cd --- /dev/null +++ b/tests/test_state_refresh.py @@ -0,0 +1,164 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from loopx.state_refresh import refresh_state_run + + +def _fixture(tmp_path: Path) -> tuple[Path, Path, Path]: + registry = tmp_path / "registry.json" + runtime = tmp_path / "runtime" + state = tmp_path / "ACTIVE_GOAL_STATE.md" + state.write_text( + "\n".join( + [ + "---", + "status: active", + "updated_at: 2026-01-01T00:00:00+00:00", + "---", + "", + "# Fixture Goal", + "", + "## Next Action", + "", + "- Continue the fixture.", + "", + ] + ), + encoding="utf-8", + ) + registry.write_text( + json.dumps( + { + "schema_version": 1, + "common_runtime_root": str(runtime), + "goals": [ + { + "id": "fixture-goal", + "status": "active", + "repo": str(tmp_path), + "state_file": state.name, + } + ], + } + ) + + "\n", + encoding="utf-8", + ) + return registry, runtime, state + + +def _refresh( + tmp_path: Path, + *, + classification: str = "fixture_progress", + next_action: str | None = None, + sync_global: bool = False, + turn_effect_key: str | None = "sha256:" + "a" * 64 + ":durable_writeback", +) -> dict[str, object]: + if not (tmp_path / "registry.json").exists(): + registry, runtime, state = _fixture(tmp_path) + else: + registry = tmp_path / "registry.json" + runtime = tmp_path / "runtime" + state = tmp_path / "ACTIVE_GOAL_STATE.md" + return refresh_state_run( + registry_path=registry, + runtime_root_override=str(runtime), + goal_id="fixture-goal", + project=tmp_path, + state_file=state, + classification=classification, + recommended_action="Continue the public fixture.", + next_action=next_action, + progress_scope="goal", + dry_run=False, + sync_global=sync_global, + turn_effect_key=turn_effect_key, + ) + + +def test_refresh_state_deduplicates_same_turn_effect_key(tmp_path: Path) -> None: + first = _refresh(tmp_path) + replay = _refresh(tmp_path) + index_path = ( + tmp_path / "runtime" / "goals" / "fixture-goal" / "runs" / "index.jsonl" + ) + index_record = json.loads(index_path.read_text(encoding="utf-8")) + run_record = json.loads(Path(str(first["json_path"])).read_text(encoding="utf-8")) + + assert first["appended"] is True + assert replay["appended"] is False + assert replay["idempotent"] is True + assert replay["json_path"] == first["json_path"] + assert replay["markdown_path"] == first["markdown_path"] + assert run_record["turn_effect_key"] == first["turn_effect_key"] + assert index_record["turn_effect_key"] == first["turn_effect_key"] + assert run_record["effect_input_hash"] == first["effect_input_hash"] + assert index_record["effect_input_hash"] == first["effect_input_hash"] + assert len(index_path.read_text(encoding="utf-8").splitlines()) == 1 + + +def test_refresh_state_rejects_turn_effect_key_content_drift(tmp_path: Path) -> None: + _refresh(tmp_path) + + with pytest.raises(ValueError, match="turn effect key conflict"): + _refresh(tmp_path, classification="different_progress") + + index_path = ( + tmp_path / "runtime" / "goals" / "fixture-goal" / "runs" / "index.jsonl" + ) + records = [json.loads(line) for line in index_path.read_text().splitlines()] + assert len(records) == 1 + + +def test_refresh_state_deduplicates_same_normalized_turn_effect_content( + tmp_path: Path, +) -> None: + first = _refresh(tmp_path, next_action="Continue the fixture.") + replay = _refresh(tmp_path, next_action=" Continue the fixture. ") + + assert first["appended"] is True + assert replay["appended"] is False + assert replay["idempotent"] is True + assert replay["json_path"] == first["json_path"] + + +@pytest.mark.parametrize("effect_key", ["unsafe key", "x" * 161]) +def test_refresh_state_rejects_unsafe_turn_effect_key( + tmp_path: Path, + effect_key: str, +) -> None: + with pytest.raises(ValueError, match="turn_effect_key must be a public-safe token"): + _refresh(tmp_path, turn_effect_key=effect_key) + + +def test_unkeyed_refresh_keeps_local_receipt_when_global_sync_raises( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _registry, runtime, _state = _fixture(tmp_path) + monkeypatch.setattr( + "loopx.state_refresh.resolve_runtime_projection_route", + lambda **_kwargs: { + "status": "single_runtime", + "target_runtime_root": str(runtime), + }, + ) + + def raise_sync_error(**_kwargs: object) -> dict[str, object]: + raise RuntimeError("fixture global sync failure") + + monkeypatch.setattr( + "loopx.state_refresh.sync_project_registry_to_global", + raise_sync_error, + ) + + with pytest.raises(RuntimeError, match="fixture global sync failure"): + _refresh(tmp_path, sync_global=True, turn_effect_key=None) + + index_path = runtime / "goals" / "fixture-goal" / "runs" / "index.jsonl" + assert len(index_path.read_text(encoding="utf-8").splitlines()) == 1 diff --git a/tests/test_turn_envelope.py b/tests/test_turn_envelope.py index 482d8fc4f..21a448382 100644 --- a/tests/test_turn_envelope.py +++ b/tests/test_turn_envelope.py @@ -763,6 +763,41 @@ def test_action_signature_detects_semantic_drift() -> None: ) != turn_envelope_action_signature_document(envelope) +@pytest.mark.parametrize( + ("write_scopes", "include_response_plan", "expected_coverage"), + [ + ([], False, "turn_envelope_action_dimensions_v2"), + ([], True, "turn_envelope_action_dimensions_v3"), + (["src/**"], False, "turn_envelope_action_dimensions_v2"), + (["src/**"], True, "turn_envelope_action_dimensions_v3"), + ], +) +def test_action_signature_versions_required_write_scope_coverage( + write_scopes: list[str], + include_response_plan: bool, + expected_coverage: str, +) -> None: + source = _full_decision() + source["selected_todo"]["required_write_scopes"] = write_scopes + if include_response_plan: + source["interaction_contract"]["response_plan"] = { + "schema_version": "interaction_response_plan_v0", + "kind": "continue_agent_work", + "decision": "run", + "action_sequence": ["execute"], + "silent_wait_allowed": False, + } + + envelope = build_turn_envelope(source) + + assert envelope["action"]["selected_todo"]["required_write_scopes"] == write_scopes + assert envelope["action_signature"]["coverage"] == expected_coverage + envelope["action"]["selected_todo"]["required_write_scopes"] = ["docs/**"] + assert quota_action_signature_document( + source + ) != turn_envelope_action_signature_document(envelope) + + def test_protocol_packet_derivation_keeps_only_real_residue() -> None: source = _full_decision() source["interaction_contract"]["agent_channel"]["primary_action"] = (