From f6549a9e9f1f21cb1600de2765b461840c26a196 Mon Sep 17 00:00:00 2001 From: hufeide <704725096@qq.com> Date: Fri, 14 Aug 2026 21:26:10 +0800 Subject: [PATCH] refactor: migrate control plane to event-driven task scheduling --- loopx/bootstrap_command_pack.py | 82 ++ loopx/canary/module_metric_baseline.json | 17 +- loopx/chat_agent.py | 12 +- loopx/claude_goal_mode/hooks/goal_policy.py | 24 +- .../claude_goal_mode/scripts/goalmode_cmd.py | 15 +- .../statusline/goal_status.py | 21 +- loopx/cli.py | 96 +-- loopx/cli_commands/project_lifecycle.py | 276 ++++++ loopx/cli_commands/starter_scheduler.py | 513 +++++++++++ loopx/codex_cli_probe.py | 1 + loopx/codex_cli_probe_markdown.py | 89 ++ loopx/contract.py | 19 +- loopx/control_plane/capabilities_bridge.py | 504 +++++++++++ loopx/control_plane/goals/goal_acceptance.py | 390 +++++++++ .../goals/goal_channel_projection.py | 65 ++ loopx/control_plane/goals/goal_closure.py | 313 +++++++ loopx/control_plane/heartbeat/event_source.py | 243 ++++++ loopx/control_plane/heartbeat/rules.py | 14 + loopx/control_plane/heartbeat/task_body.py | 3 +- loopx/control_plane/new_architecture.py | 34 + .../capabilities-bridge-migration-notes.md | 57 ++ loopx/control_plane/policy/__init__.py | 48 ++ loopx/control_plane/policy/decision.py | 253 ++++++ loopx/control_plane/policy/decision_events.py | 258 ++++++ loopx/control_plane/policy/engine.py | 155 ++++ loopx/control_plane/quota/cost_projection.py | 282 ++++++ loopx/control_plane/quota/live_decision.py | 133 ++- loopx/control_plane/runtime/checkpoint.py | 214 +++++ loopx/control_plane/runtime/replay.py | 243 ++++++ .../scheduler/event_driven_dispatch.py | 768 +++++++++++++++++ loopx/control_plane/scheduler/merge.py | 327 +++++++ loopx/control_plane/scheduler/resident.py | 811 ++++++++++++++++++ .../control_plane/scheduler/task_lifecycle.py | 677 +++++++++++++++ .../status/control_plane_observability.py | 295 +++++++ .../testing/cli_output_budget.py | 8 +- loopx/heartbeat_prequota.py | 111 ++- loopx/heartbeat_prompt.py | 2 + .../goal-bridge-runtime.mjs | 144 ++++ loopx/pi_goal_mode/loopx-goal.ts | 19 + loopx/pi_goal_mode/pi-goal-loop-runtime.mjs | 125 ++- .../presentation/renderers/status_markdown.py | 120 +++ loopx/registry.py | 37 + loopx/rollout_event_log.py | 12 + loopx/slash_command_install.py | 4 +- loopx/todos.py | 82 +- .../control_plane/test_capabilities_bridge.py | 271 ++++++ tests/control_plane/test_checkpoint_replay.py | 349 ++++++++ .../test_control_plane_observability.py | 157 ++++ tests/control_plane/test_cost_projection.py | 195 +++++ .../test_event_driven_dispatch.py | 789 +++++++++++++++++ tests/control_plane/test_goal_acceptance.py | 240 ++++++ .../test_goal_channel_projection_quota.py | 121 +++ tests/control_plane/test_goal_closure.py | 125 +++ .../test_heartbeat_event_source.py | 164 ++++ .../control_plane/test_heartbeat_prequota.py | 107 +++ .../test_policy_decision_events.py | 177 ++++ tests/control_plane/test_policy_engine.py | 298 +++++++ .../control_plane/test_policy_integration.py | 122 +++ .../control_plane/test_policy_pilot_wiring.py | 259 ++++++ tests/control_plane/test_rich_decision.py | 91 ++ .../test_scheduler_resident_merge.py | 678 +++++++++++++++ .../test_start_goal_compact_projection.py | 53 ++ tests/control_plane/test_task_lifecycle.py | 362 ++++++++ .../test_todo_mutation_authority.py | 25 + tests/opencode_goal_bridge_runtime.test.mjs | 102 +++ tests/pi_goal_loop_runtime.test.mjs | 142 +++ tests/test_claude_goal_policy.py | 39 + 67 files changed, 12685 insertions(+), 97 deletions(-) create mode 100644 loopx/control_plane/capabilities_bridge.py create mode 100644 loopx/control_plane/goals/goal_acceptance.py create mode 100644 loopx/control_plane/goals/goal_closure.py create mode 100644 loopx/control_plane/heartbeat/event_source.py create mode 100644 loopx/control_plane/new_architecture.py create mode 100644 loopx/control_plane/plan/capabilities-bridge-migration-notes.md create mode 100644 loopx/control_plane/policy/__init__.py create mode 100644 loopx/control_plane/policy/decision.py create mode 100644 loopx/control_plane/policy/decision_events.py create mode 100644 loopx/control_plane/policy/engine.py create mode 100644 loopx/control_plane/quota/cost_projection.py create mode 100644 loopx/control_plane/runtime/checkpoint.py create mode 100644 loopx/control_plane/runtime/replay.py create mode 100644 loopx/control_plane/scheduler/event_driven_dispatch.py create mode 100644 loopx/control_plane/scheduler/merge.py create mode 100644 loopx/control_plane/scheduler/resident.py create mode 100644 loopx/control_plane/scheduler/task_lifecycle.py create mode 100644 loopx/control_plane/status/control_plane_observability.py create mode 100644 tests/control_plane/test_capabilities_bridge.py create mode 100644 tests/control_plane/test_checkpoint_replay.py create mode 100644 tests/control_plane/test_control_plane_observability.py create mode 100644 tests/control_plane/test_cost_projection.py create mode 100644 tests/control_plane/test_event_driven_dispatch.py create mode 100644 tests/control_plane/test_goal_acceptance.py create mode 100644 tests/control_plane/test_goal_channel_projection_quota.py create mode 100644 tests/control_plane/test_goal_closure.py create mode 100644 tests/control_plane/test_heartbeat_event_source.py create mode 100644 tests/control_plane/test_heartbeat_prequota.py create mode 100644 tests/control_plane/test_policy_decision_events.py create mode 100644 tests/control_plane/test_policy_engine.py create mode 100644 tests/control_plane/test_policy_integration.py create mode 100644 tests/control_plane/test_policy_pilot_wiring.py create mode 100644 tests/control_plane/test_rich_decision.py create mode 100644 tests/control_plane/test_scheduler_resident_merge.py create mode 100644 tests/control_plane/test_task_lifecycle.py create mode 100644 tests/test_claude_goal_policy.py diff --git a/loopx/bootstrap_command_pack.py b/loopx/bootstrap_command_pack.py index 3f7fee528..87a30f237 100644 --- a/loopx/bootstrap_command_pack.py +++ b/loopx/bootstrap_command_pack.py @@ -30,6 +30,7 @@ shell_arg, ) from .registry import registry_goals, resolve_state_file +from .rollout_event_log import load_rollout_events, rollout_event_log_path from .slash_commands import build_slash_command_catalog from .thread_agent_binding import normalize_thread_id, resolve_thread_agent_binding @@ -455,6 +456,49 @@ def _select_goal(goals: list[dict[str, Any]], goal_id: str | None) -> tuple[str, return "", None +def _goal_already_closed( + *, + registry: dict[str, Any], + goal_id: str, +) -> bool: + """Return True when the goal's rollout event log records a ``goal_closed`` + event. + + This is the reliable signal that a previous loop already derived closure. + Reusing a closed goal — instead of starting a fresh one — is what caused the + website1 "yellow -> purple" state pollution: the agent appended new todos to + an already-closed goal, then hand-edited the objective and Next Action. We + surface that here so the start-goal flow can require a fresh goal. + """ + runtime_root_raw = registry.get("common_runtime_root") + if not runtime_root_raw: + return False + try: + log_path = rollout_event_log_path(Path(str(runtime_root_raw)), goal_id) + events = load_rollout_events(log_path) + except Exception: + return False + return any(e.get("event_kind") == "goal_closed" for e in events) + + +def _next_goal_id_suggestion(closed_goal_id: str, known_goal_ids: list[str]) -> str: + """Suggest a fresh goal id derived from a closed one (``-2``, ``-3``...). + + Keeps the full closed id as the base and appends a ``-`` suffix, so any + trailing digits that are NOT a sequence number (a year ``2024``, a version + ``3``, ...) are preserved rather than stripped. Collisions with the closed + goal and any sibling ids are skipped. + """ + base = str(closed_goal_id) + known = {str(g) for g in known_goal_ids} + index = 2 + while True: + candidate = f"{base}-{index}" + if candidate not in known and candidate != closed_goal_id: + return candidate + index += 1 + + def inspect_bootstrap_connection( project: Path, *, @@ -568,6 +612,24 @@ def inspect_bootstrap_connection( "reason": "registry goal points at a state_file that is missing", } + if _goal_already_closed(registry=registry, goal_id=resolved_goal_id): + return { + **base_connection, + "registry_exists": True, + "goal_id": resolved_goal_id, + "goal_found": True, + "state_file": str(state_file), + "state_file_exists": True, + "connection_state": "goal_reuse_closed", + "should_start_new_goal": True, + "mutation_confirmation_required": True, + "reason": ( + "registry goal is present but its rollout log already records a " + "goal_closed event; start a fresh goal instead of appending to a " + "closed one" + ), + } + return { **base_connection, "registry_exists": True, @@ -834,6 +896,7 @@ def _goal_start_prompt(*, goal_text: str | None, goal_id: str, agent_id: str | N 5. Prefer executable Agent Todo items with `task_class=advancement_task`; use User Todo only for concrete owner decisions or private-material gates. 6. After writing todos, run `loopx refresh-state --goal-id {goal_id}`, activate the host loop if it is missing, unknown, or stale (Codex App automation, Codex CLI `/goal `, Claude Code `/loop`, OpenCode bridge, or a custom host-loop gate), then run its typed `quota_guard` and begin the first allowed bounded segment. 7. Enter issue-fix only when `selected_capability_route.capability_id=issue-fix`; never infer it from goal text or URLs. Run workflow-plan and feasibility before implementation, write only the admitted successor or no-follow-up, preserve private/external/destructive gates, verify reviewer requests, and reconcile PR lifecycle one PR per message. +8. Drive execution through the event-driven scheduler, not by hand-editing files and calling `todo complete` yourself. Use `loopx codex-cli-local-scheduler-dispatch --goal-id {goal_id} --project . --agent-id --event-driven [--completed-todo-id ] --acceptance-criteria = --evidence =grep==`. The dispatcher recomputes READY successors, enqueues and claims for a worker, and — when the queue empties and acceptance evidence satisfies — atomically emits goal_closure_ready + goal_closed in one tick. You declare the plan and the acceptance criteria/evidence; let the dispatcher drive execution and closure. """ @@ -992,6 +1055,25 @@ def build_loopx_bootstrap_command_pack( "summary": identity_selection_gate.get("reason"), "identity_selection_gate": identity_selection_gate, } + elif bool(inspection.get("should_start_new_goal")) and explicit_goal_start: + # The requested goal id is already closed (its rollout log records a + # goal_closed event). Reusing it appends new todos to a finished goal and + # forces the agent to hand-edit the objective/Next Action — the exact + # state pollution seen in the website1 yellow->purple session. Instead, + # require a fresh goal id and a fresh bootstrap before planning. + recommended_next_step = { + "kind": "goal_reuse_closed_require_new_goal", + "requires_user_confirmation": True, + "summary": ( + f"Goal `{resolved_goal_id}` is already closed. Start a fresh goal " + "instead of appending to a finished one." + ), + "closed_goal_id": resolved_goal_id, + "suggested_new_goal_id": _next_goal_id_suggestion( + resolved_goal_id, known_goal_ids=[g["id"] for g in registry_goals(registry_payload or {})] + ), + "connect_command_if_needed": goal_start_bootstrap_command, + } elif explicit_goal_start: recommended_next_step = { "kind": "goal_plan_write_and_activate", diff --git a/loopx/canary/module_metric_baseline.json b/loopx/canary/module_metric_baseline.json index 5bed5aa2d..ccb466031 100644 --- a/loopx/canary/module_metric_baseline.json +++ b/loopx/canary/module_metric_baseline.json @@ -6,9 +6,14 @@ }, "module_metric_ceilings": { "loopx/bootstrap_command_pack.py": { - "any_count": 32, + "any_count": 33, "dict_any_count": 0, - "lines": 2240 + "lines": 2322 + }, + "loopx/chat_actions.py": { + "any_count": 52, + "dict_any_count": 0, + "lines": 1590 }, "loopx/canary/planner.py": { "any_count": 22, @@ -38,7 +43,7 @@ "loopx/codex_cli_probe.py": { "any_count": 37, "dict_any_count": 0, - "lines": 1530 + "lines": 1531 }, "loopx/control_plane/agents/agent_scope.py": { "any_count": 107, @@ -76,9 +81,9 @@ "lines": 1508 }, "loopx/presentation/renderers/status_markdown.py": { - "any_count": 47, + "any_count": 49, "dict_any_count": 0, - "lines": 1570 + "lines": 1690 }, "loopx/quota.py": { "any_count": 219, @@ -93,7 +98,7 @@ "loopx/todos.py": { "any_count": 48, "dict_any_count": 0, - "lines": 2142 + "lines": 2205 }, "loopx/worker_bridge.py": { "any_count": 28, diff --git a/loopx/chat_agent.py b/loopx/chat_agent.py index 110862a53..1825ef210 100644 --- a/loopx/chat_agent.py +++ b/loopx/chat_agent.py @@ -124,8 +124,14 @@ def _turn_prompt(user_message: str, *, context_summary: str = "") -> str: "Do not use tools for ordinary conversation, exact-wording requests, or status questions " "that can be answered from the supplied LoopX context. " "Do not edit files, mutate LoopX state, create commits, send messages, or request elevated access. " - "When the operator requests a durable Goal, Todo, Agent binding, heartbeat, monitor, gate, or correction change, " - "describe the bounded proposal clearly so LoopX can route it through typed preview and explicit apply. " + "When the operator asks you to do something, add a todo, remember a task, or make a durable change " + "(Goal, Todo, Agent binding, heartbeat, monitor, gate, or correction), you MUST put at least one " + "matching `todo` item into the `proposals` array of the envelope below. Never describe a todo as " + "\"待确认\" or \"pending\" in prose without also emitting the corresponding structured proposal. " + "Each proposal must be a single bounded, reviewable item with a concrete `text`, a `priority` " + "(P0/P1/P2), and a short `rationale`. LoopX will route each proposal through typed preview and " + "explicit operator approval before any write happens; the operator's approval in the UI is what " + "actually writes the todo, so do not claim the todo has been written yet. " "Never claim the change has been written without a verified control-plane receipt. " "If you encounter an identity, approval, or host-tool gate, stop and describe it in gate. " "Reply in Chinese unless the operator asks for another language. Keep proposals bounded and reviewable. " @@ -133,6 +139,8 @@ def _turn_prompt(user_message: str, *, context_summary: str = "") -> str: "First write the complete operator-facing answer as ordinary text. Start with the conclusion, " "use short sentences or lines so the answer can stream, and include at most five actionable items. " "Then append exactly one machine-readable envelope whose message field repeats that complete answer. " + "The envelope's `proposals` array must carry every durable todo the operator requested in this turn; " + "if the operator asked for no durable change, use an empty `proposals` list. " "Do not write anything after the closing tag. Use these tags and shape:\n" f"{CHAT_REVIEW_OPEN_TAG}{json.dumps(envelope, ensure_ascii=False)}{CHAT_REVIEW_CLOSE_TAG}\n\n" + (f"LoopX context (supporting context only):\n{context_summary.strip()}\n\n" if context_summary.strip() else "") diff --git a/loopx/claude_goal_mode/hooks/goal_policy.py b/loopx/claude_goal_mode/hooks/goal_policy.py index 3f38f9e90..7f461f9c9 100644 --- a/loopx/claude_goal_mode/hooks/goal_policy.py +++ b/loopx/claude_goal_mode/hooks/goal_policy.py @@ -112,6 +112,27 @@ def _runtime_profile_flag_is_unsupported(out: subprocess.CompletedProcess) -> bo ) +def resolve_should_run(data: dict) -> bool | None: + """Resolve the run gate from a ``quota should-run`` payload. + + New-architecture aware: when the payload carries the unified + ``policy_decision`` (on by default), its ``outcome`` is the authoritative + control-plane decision. ``run`` -> True; ``deny``/``wait`` -> False (the + PolicyEngine may compose a stricter capability/scheduler layer that the + legacy quota ``should_run`` does not reflect). Falls back to the legacy + ``should_run`` bool when no unified decision is present. + """ + pd = data.get("policy_decision") + if isinstance(pd, dict): + outcome = pd.get("outcome") + if outcome == "run": + return True + if outcome in ("deny", "wait"): + return False + should_run_value = data.get("should_run") + return should_run_value if isinstance(should_run_value, bool) else None + + def should_run(registry, goal_id, agent_id=None) -> bool | None: """Return True/False from loopx quota should-run, or None if unknown.""" if not goal_id: @@ -142,8 +163,7 @@ def should_run(registry, goal_id, agent_id=None) -> bool | None: data = json.loads(out.stdout or "{}") except (json.JSONDecodeError, TypeError, ValueError): return None - should_run_value = data.get("should_run") - return should_run_value if isinstance(should_run_value, bool) else None + return resolve_should_run(data) def decide(ev: dict) -> dict: diff --git a/loopx/claude_goal_mode/scripts/goalmode_cmd.py b/loopx/claude_goal_mode/scripts/goalmode_cmd.py index 2009089a6..f9ddc8377 100644 --- a/loopx/claude_goal_mode/scripts/goalmode_cmd.py +++ b/loopx/claude_goal_mode/scripts/goalmode_cmd.py @@ -116,6 +116,19 @@ def goal_detail(ctx): return objective, payload +def _should_run(data: dict) -> bool | None: + """New-architecture aware run gate (see goal_policy.resolve_should_run).""" + pd = data.get("policy_decision") + if isinstance(pd, dict): + outcome = pd.get("outcome") + if outcome == "run": + return True + if outcome in ("deny", "wait"): + return False + sr = data.get("should_run") + return sr if isinstance(sr, bool) else None + + def print_status(ctx): gid = ctx.get("goal_id") objective, d = goal_detail(ctx) @@ -124,7 +137,7 @@ def print_status(ctx): gate = d.get("gate_prompt") if gate: state = f"⚠ needs you: {gate}" - elif d.get("should_run") is True: + elif _should_run(d) is True: state = "▶ running" else: state = f"⏸ {d.get('state') or 'paused'}" + (f" — {d['reason']}" if d.get("reason") else "") diff --git a/loopx/claude_goal_mode/statusline/goal_status.py b/loopx/claude_goal_mode/statusline/goal_status.py index a51e3f548..aa65e8fc7 100644 --- a/loopx/claude_goal_mode/statusline/goal_status.py +++ b/loopx/claude_goal_mode/statusline/goal_status.py @@ -37,6 +37,25 @@ def _clip(s, n: int) -> str: return s if len(s) <= n else s[: n - 1] + "…" +def _resolve_run(data: dict) -> bool | None: + """New-architecture aware run gate. + + Prefer the unified ``policy_decision.outcome`` (run -> True, deny/wait -> + False) so the statusline reflects the composed control-plane decision, not + just the quota-level ``should_run`` which may remain permissive under a + stricter capability/scheduler layer. + """ + pd = data.get("policy_decision") + if isinstance(pd, dict): + outcome = pd.get("outcome") + if outcome == "run": + return True + if outcome in ("deny", "wait"): + return False + sr = data.get("should_run") + return sr if isinstance(sr, bool) else None + + def _render(gid: str, d: dict) -> str: """Turn a `quota should-run` payload into one compact statusline string.""" agent = d.get("agent_todo_summary") or {} @@ -53,7 +72,7 @@ def _render(gid: str, d: dict) -> str: msg = gate or f"{users.get('open_count') or 'a'} user todo(s) to answer" return f"[loopx {gid} · ⚠ needs you: {_clip(msg, 46)}]" - if d.get("should_run") is True: + if _resolve_run(d) is True: nxt = d.get("recommended_action") head = f"▶ {prog}".rstrip() if prog else "▶ working" return f"[loopx {gid} · {head}" + (f" · next: {_clip(nxt, 40)}]" if nxt else "]") diff --git a/loopx/cli.py b/loopx/cli.py index 530126a9b..36883ae74 100644 --- a/loopx/cli.py +++ b/loopx/cli.py @@ -6,55 +6,26 @@ from pathlib import Path from . import __version__ -from .capabilities.content_ops.cli import ( - handle_content_ops_command, - register_content_ops_commands, -) -from .capabilities.agent_turn_recall.cli import ( - handle_agent_turn_recall_command, - register_agent_turn_recall_commands, -) -from .capabilities.change_quality.cli import ( - handle_change_quality_command, - register_change_quality_commands, -) -from .capabilities.integration_branch.cli import ( - handle_integration_branch_command, - register_integration_branch_commands, -) -from .capabilities.decision_context.cli import ( - handle_decision_context_command, - register_decision_context_commands, -) -from .capabilities.material_lifecycle.cli import ( - handle_material_lifecycle_command, - register_material_lifecycle_commands, -) -from .capabilities.issue_fix.cli import ( - handle_issue_fix_command, - register_issue_fix_commands, -) -from .capabilities.reward_memory.cli import ( - handle_reward_memory_command, - register_reward_memory_commands, -) -from .capabilities.periodic_report.cli import ( - handle_periodic_report_command, - register_periodic_report_commands, -) -from .capabilities.semantic_preference.cli import ( - handle_semantic_preference_command, - register_semantic_preference_commands, -) +from .control_plane.capabilities_bridge import register_all_capability_commands +from .capabilities.catalog import build_capability_registry +# Capability-pack command *registration* is registry-driven (see +# ``register_all_capability_commands`` below), so only the pack-specific command +# *dispatchers* are imported statically (their signatures cannot be generalized). +from .capabilities.content_ops.cli import handle_content_ops_command +from .capabilities.agent_turn_recall.cli import handle_agent_turn_recall_command +from .capabilities.change_quality.cli import handle_change_quality_command +from .capabilities.integration_branch.cli import handle_integration_branch_command +from .capabilities.decision_context.cli import handle_decision_context_command +from .capabilities.material_lifecycle.cli import handle_material_lifecycle_command +from .capabilities.issue_fix.cli import handle_issue_fix_command +from .capabilities.reward_memory.cli import handle_reward_memory_command +from .capabilities.periodic_report.cli import handle_periodic_report_command +from .capabilities.semantic_preference.cli import handle_semantic_preference_command from .capabilities.auto_research.cli import ( handle_auto_research_command, - register_auto_research_commands, rewrite_auto_research_question_argv, ) -from .capabilities.value_connectors.cli import ( - handle_value_connector_command, - register_value_connector_commands, -) +from .capabilities.value_connectors.cli import handle_value_connector_command from .cli_commands import ( handle_turn_command, handle_benchmark_command, @@ -230,36 +201,22 @@ def build_parser() -> LoopXArgumentParser: register_extension_commands(sub, add_subcommand_format) - register_change_quality_commands(sub, add_subcommand_format) - - register_integration_branch_commands(sub, add_subcommand_format) - - register_content_ops_commands(sub, add_subcommand_format) - - register_decision_context_commands(sub, add_subcommand_format) - - register_material_lifecycle_commands(sub, add_subcommand_format) + # Capability packs self-register their CLI from the capability catalog, so + # adding a pack no longer requires touching this wiring. The legacy static + # imports above are retained for their ``handle_*`` dispatchers (which carry + # pack-specific signatures and cannot be generalized). + register_all_capability_commands( + sub, + add_subcommand_format, + registry=build_capability_registry(), + ) register_project_skill_commands(sub, add_subcommand_format) - register_issue_fix_commands(sub, add_subcommand_format) - - register_reward_memory_commands(sub, add_subcommand_format) - - register_agent_turn_recall_commands(sub, add_subcommand_format) - register_review_batch_commands(sub, add_subcommand_format) - register_periodic_report_commands(sub, add_subcommand_format) - - register_semantic_preference_commands(sub, add_subcommand_format) - - register_value_connector_commands(sub, add_subcommand_format) - register_ml_experiment_commands(sub, add_subcommand_format) - register_auto_research_commands(sub, add_subcommand_format) - register_multi_agent_commands(sub, add_subcommand_format) register_turn_commands(sub, add_subcommand_format) register_host_mode_plan_command(sub, add_subcommand_format) @@ -316,7 +273,10 @@ def main(argv: list[str] | None = None) -> int: "codex-cli-exec-handoff", "codex-cli-visible-first-response-capture-plan", "codex-cli-local-driver-plan", + "codex-cli-local-scheduler-dispatch", "codex-cli-local-scheduler-exec", + "codex-cli-local-scheduler-merge", + "codex-cli-local-scheduler-resident", "codex-cli-local-scheduler-tick", "codex-cli-one-message-loop-pilot", "codex-cli-runtime-idle-detector", diff --git a/loopx/cli_commands/project_lifecycle.py b/loopx/cli_commands/project_lifecycle.py index db45a5bb5..859b8e5c7 100644 --- a/loopx/cli_commands/project_lifecycle.py +++ b/loopx/cli_commands/project_lifecycle.py @@ -5,6 +5,7 @@ import json from collections.abc import Callable, Mapping from pathlib import Path +from typing import Any from ..capabilities.explore.activation import ( sync_explore_graph_after_material_refresh, @@ -68,8 +69,42 @@ "read-only-map", "reward", "operator-gate", + "goal-closure", } + +def render_goal_closure_markdown(payload: dict[str, Any]) -> str: + """Render a goal-closure evaluation as compact Markdown.""" + evaluation = payload.get("evaluation") or {} + lines = [ + "# Goal Closure\n", + f"- goal_id: `{payload.get('goal_id')}`", + f"- ready: `{evaluation.get('ready')}`", + f"- tri_state: `{evaluation.get('tri_state')}`", + f"- reason: `{evaluation.get('reason')}`", + ] + evidence = evaluation.get("evidence") or {} + lines.append( + "- evidence: ready=" + f"`{len(evidence.get('ready_todo_ids') or [])}`, " + f"blocked=`{len(evidence.get('blocked_todo_ids') or [])}`, " + f"deferred=`{len(evidence.get('deferred_todo_ids') or [])}`, " + f"replan=`{evidence.get('replan_required')}`, " + f"acceptance_satisfied=`{evidence.get('acceptance_satisfied')}`, " + f"acceptance_gap_count=`{evidence.get('acceptance_gap_count')}`" + ) + acceptance = payload.get("acceptance") or {} + gaps = acceptance.get("acceptance_gaps") or [] + if gaps: + lines.append( + "- acceptance gaps: `" + + "`, `".join(str(g.get("criterion_id")) for g in gaps) + + "`" + ) + if payload.get("applied"): + lines.append("- applied: `true` (goal_closure_ready + goal_closed emitted)") + return "\n".join(lines) + "\n" + INLINE_VISION_FIELDS = { "vision_summary": "vision_summary", "vision_role_scope": "role_scope", @@ -497,6 +532,104 @@ def register_project_lifecycle_commands( help="Do not refresh the shared global registry after writing the gate decision.", ) + closure_parser = subparsers.add_parser( + "goal-closure", + help=( + "Evaluate whether a goal is closable (no ready work, no pending " + "dependencies, no replan, no external follow-up) and emit " + "goal_closure_ready + goal_closed events when it is." + ), + ) + add_subcommand_format(closure_parser) + closure_parser.add_argument( + "--goal-id", + required=True, + help="Goal id whose closure is being evaluated.", + ) + closure_parser.add_argument( + "--runtime-root", + default=None, + help="Runtime root where the task queue / rollout event log live. Defaults to the registry goal's runtime root.", + ) + closure_parser.add_argument( + "--ready-todo-id", + action="append", + default=[], + help="Ready todo id (repeatable). Any present value keeps the goal RUN/WAIT.", + ) + closure_parser.add_argument( + "--blocked-todo-id", + action="append", + default=[], + help="Blocked todo id (repeatable). Any present value keeps the goal WAIT.", + ) + closure_parser.add_argument( + "--deferred-todo-id", + action="append", + default=[], + help="Deferred todo id (repeatable). Any present value keeps the goal WAIT.", + ) + closure_parser.add_argument( + "--replan-required", + action="store_true", + help="Treat the goal as requiring replan (keeps it WAIT, not closable).", + ) + closure_parser.add_argument( + "--external-followup-required", + action="store_true", + help="Treat the goal as requiring external follow-up (keeps it WAIT).", + ) + closure_parser.add_argument( + "--apply", + action="store_true", + help="When closable, actually emit goal_closure_ready + goal_closed events.", + ) + closure_parser.add_argument( + "--dry-run", + action="store_true", + help="Print the evaluation without writing any events.", + ) + closure_parser.add_argument( + "--acceptance-criteria", + action="append", + default=[], + help=( + "Declare an acceptance criterion as criterion_id=description " + "(repeatable). A goal is NOT closable until every criterion has " + "satisfying evidence. e.g. color_green=theme color is #22c55e" + ), + ) + closure_parser.add_argument( + "--evidence", + action="append", + default=[], + help=( + "Declare evidence as criterion_id=kind=ref[=regex] (repeatable). " + "kind in {grep,snapshot,test,file,manual}. For kind=grep, an optional " + "4th segment is the regex pattern the framework independently matches " + "against ref (relative to --project), overriding any self-reported ok. " + "e.g. color_green=grep=index.html=#22c55e" + ), + ) + closure_parser.add_argument( + "--verify", + action="store_true", + help=( + "Run the Goal Acceptance evaluator: verify every acceptance " + "criterion against evidence. Emits goal_acceptance_pending when gaps " + "remain, blocking closure." + ), + ) + closure_parser.add_argument( + "--project", + default=None, + help=( + "Project root for independently verifying grep evidence. Defaults to " + "the registry goal repo. Without it, grep evidence degrades to " + "self-reported ok." + ), + ) + def handle_project_lifecycle_command( args: argparse.Namespace, @@ -802,6 +935,149 @@ def handle_project_lifecycle_command( print_payload(payload, fmt, render_reward_markdown) return 0 if payload.get("ok") else 1 + if args.command == "goal-closure": + try: + from ..control_plane.goals.goal_closure import ( + build_goal_closure_state, + emit_goal_closed, + emit_goal_closure_ready, + evaluate_goal_closure, + ) + from ..control_plane.scheduler.event_driven_dispatch import ( + load_task_queue, + task_queue_path, + ) + from ..rollout_event_log import rollout_event_log_path + + runtime_root = Path( + resolve_runtime_root(load_registry(registry_path), args.runtime_root) + ) + queue_view = load_task_queue( + task_queue_path(runtime_root, goal_id=args.goal_id) + ) + # Goal Acceptance evaluation (when criteria/evidence supplied). + acceptance = None + if ( + bool(getattr(args, "verify", False)) + or getattr(args, "acceptance_criteria", None) + or getattr(args, "evidence", None) + ): + from ..control_plane.goals.goal_acceptance import ( + evaluate_goal_acceptance, + ) + + criteria = [] + for spec in getattr(args, "acceptance_criteria", None) or []: + if "=" in spec: + cid, _, desc = spec.partition("=") + criteria.append( + {"criterion_id": cid.strip(), "description": desc.strip()} + ) + evidence = [] + for spec in getattr(args, "evidence", None) or []: + # format: criterion_id=kind=ref[=regex] + parts = spec.split("=") + if len(parts) < 3: + continue + kind = parts[1].strip() + item: dict[str, Any] = { + "criterion_ids": [parts[0].strip()], + "kind": kind, + "ref": parts[2].strip(), + "ok": True, + } + if len(parts) >= 4: + item["pattern"] = parts[3].strip() + if kind == "grep": + # Independent verification when a regex is provided; the + # self-reported ok is only a fallback without one. + item["ok"] = bool(item.get("pattern")) + evidence.append(item) + # Resolve the project root for independent grep verification. + base_dir = None + if getattr(args, "project", None): + base_dir = Path(args.project).expanduser().resolve() + acceptance = evaluate_goal_acceptance( + acceptance_criteria=criteria or None, + evidence=evidence or None, + base_dir=base_dir, + ) + state = build_goal_closure_state( + ready_todo_ids=list(getattr(args, "ready_todo_id", None) or []) + or queue_view.get("pending_todo_ids", []), + blocked_todo_ids=list(getattr(args, "blocked_todo_id", None) or []), + deferred_todo_ids=list(getattr(args, "deferred_todo_id", None) or []), + pending_dependency_ids=list(getattr(args, "blocked_todo_id", None) or []), + replan_required=bool(getattr(args, "replan_required", False)), + external_followup_required=bool( + getattr(args, "external_followup_required", False) + ), + open_todo_count=queue_view.get("pending_count", 0), + claimed_advancement_count=queue_view.get("claimed_count", 0), + acceptance=acceptance, + ) + evaluation = evaluate_goal_closure(state) + applied = False + log_path = rollout_event_log_path(runtime_root, goal_id=args.goal_id) + if evaluation["ready"] and bool(getattr(args, "apply", False)): + emit_goal_closure_ready( + log_path=log_path, + goal_id=args.goal_id, + reason=evaluation["reason"], + evidence=evaluation["evidence"], + ) + emit_goal_closed( + log_path=log_path, + goal_id=args.goal_id, + kind="derived", + reason=evaluation["reason"], + ) + applied = True + # Keep the registry goal entry's status in lockstep with the + # rollout log so `status`/registry and start-goal's guided packet + # (which reads the rollout log) agree that the goal is closed. + from ..registry import sync_registry_goal_closed + + sync_registry_goal_closed(registry_path, args.goal_id) + # When acceptance has gaps and --apply (or --verify), record pending. + if ( + acceptance is not None + and acceptance.get("satisfied") is not True + and ( + bool(getattr(args, "apply", False)) + or bool(getattr(args, "verify", False)) + ) + ): + from ..control_plane.goals.goal_acceptance import ( + emit_goal_acceptance_pending, + ) + + gaps = acceptance.get("acceptance_gaps") or [] + if gaps: + emit_goal_acceptance_pending( + log_path=log_path, + goal_id=args.goal_id, + acceptance_gaps=gaps, + ) + payload = { + "ok": True, + "goal_id": args.goal_id, + "evaluation": evaluation, + "acceptance": acceptance, + "applied": applied, + "dry_run": bool(getattr(args, "dry_run", False)), + } + except Exception as exc: # noqa: BLE001 + payload = { + "ok": False, + "goal_id": args.goal_id, + "evaluation": None, + "applied": False, + "error": str(exc), + } + print_payload(payload, fmt, render_goal_closure_markdown) + return 0 if payload.get("ok") else 1 + try: payload = record_operator_gate( registry_path=registry_path, diff --git a/loopx/cli_commands/starter_scheduler.py b/loopx/cli_commands/starter_scheduler.py index d1396b229..fb51e5795 100644 --- a/loopx/cli_commands/starter_scheduler.py +++ b/loopx/cli_commands/starter_scheduler.py @@ -4,20 +4,35 @@ import json from collections.abc import Callable from pathlib import Path +from typing import Any from ..codex_cli_probe import ( DEFAULT_CODEX_BIN, DEFAULT_EXECUTOR_TIMEOUT_SECONDS, DEFAULT_TIMEOUT_SECONDS, load_codex_cli_visible_session_proof_fixture, + render_codex_cli_local_scheduler_dispatch_markdown, render_codex_cli_local_scheduler_executor_markdown, render_codex_cli_local_scheduler_tick_markdown, run_codex_cli_session_probe, ) +from ..bootstrap import default_goal_id from ..codex_cli_scheduler import ( build_codex_cli_local_scheduler_executor, build_codex_cli_local_scheduler_tick, ) +from ..control_plane.scheduler.event_driven_dispatch import ( + EVENT_DRIVEN_DISPATCH_ENV, + build_event_driven_dispatch, + event_driven_dispatch_enabled, +) +from ..control_plane.scheduler.merge import merge_event_driven_and_heartbeat +from ..control_plane.scheduler.resident import ( + _loaded_items as _resident_loaded_items, + run_resident_scheduler_bounded, +) +from ..paths import DEFAULT_RUNTIME_ROOT +from ..rollout_event_log import rollout_event_log_path from .starter_runtime_idle import ( _add_runtime_idle_observation_arguments, _load_codex_cli_runtime_idle_payload, @@ -113,6 +128,250 @@ def register_starter_scheduler_commands(subparsers: argparse._SubParsersAction) help="Allowed command prefix for --execute-candidate. Repeatable; required before candidate execution.", ) + codex_cli_local_scheduler_dispatch_parser = subparsers.add_parser( + "codex-cli-local-scheduler-dispatch", + help=( + "RFC Phase 6 event-driven scheduling pilot: recompute READY successors " + "from handoff gates, enqueue them, and optionally claim for a worker. " + "Opt-in via --event-driven or LOOPX_EVENT_DRIVEN_DISPATCH=1; disabled by default." + ), + ) + codex_cli_local_scheduler_dispatch_parser.add_argument( + "--project", + default=".", + help="Project directory to start from; used for the default goal id.", + ) + codex_cli_local_scheduler_dispatch_parser.add_argument( + "--goal-id", + help="Goal id. Defaults to -goal.", + ) + codex_cli_local_scheduler_dispatch_parser.add_argument( + "--runtime-root", + default=None, + help="Runtime root that contains goals//events.jsonl. Defaults to the global runtime root.", + ) + codex_cli_local_scheduler_dispatch_parser.add_argument( + "--completed-todo-id", + help="Optional completed todo id that triggered this dispatch tick.", + ) + codex_cli_local_scheduler_dispatch_parser.add_argument( + "--worker-id", + help="Optional worker id to claim the next queued task (Worker Pool acquire).", + ) + codex_cli_local_scheduler_dispatch_parser.add_argument( + "--agent-id", + help=( + "Optional registered LoopX agent id to record on the task_dispatched " + "audit event (falls back to --worker-id when omitted)." + ), + ) + codex_cli_local_scheduler_dispatch_parser.add_argument( + "--event-log-path", + help="Override rollout event log path (default: goals//rollout-event-log.jsonl).", + ) + codex_cli_local_scheduler_dispatch_parser.add_argument( + "--event-driven", + action="store_true", + default=None, + help="Enable event-driven dispatch for this tick (overrides env).", + ) + codex_cli_local_scheduler_dispatch_parser.add_argument( + "--no-reconcile", + action="store_true", + default=False, + help="Disable lease-expiry (zombie recovery) + retry promotion before claiming.", + ) + codex_cli_local_scheduler_dispatch_parser.add_argument( + "--lease-seconds", + type=float, + default=None, + help="Claim lease TTL in seconds; expired/unleased claims are reclaimed by reconcile.", + ) + codex_cli_local_scheduler_dispatch_parser.add_argument( + "--acceptance-criteria", + action="append", + default=[], + metavar="ID=DESCRIPTION", + help=( + "Goal Acceptance criterion (repeatable). When supplied, dispatch runs the " + "Goal Acceptance / Evidence Verification layer and closes the loop (emits " + "goal_closure_ready + goal_closed) in a single tick instead of requiring a " + "separate manual goal-closure --verify --apply." + ), + ) + codex_cli_local_scheduler_dispatch_parser.add_argument( + "--evidence", + action="append", + default=[], + metavar="CRITERION_ID=KIND=REF", + help=( + "Evidence for an acceptance criterion (repeatable), e.g. " + "yellow_bg=grep=index.html. KIND is one of grep|manual|file|command." + ), + ) + + codex_cli_local_scheduler_resident_parser = subparsers.add_parser( + "codex-cli-local-scheduler-resident", + help=( + "RFC Phase 5 resident scheduler: run the Task Queue + Worker Pool " + "for a bounded number of ticks. Opt-in via --event-driven or " + "LOOPX_EVENT_DRIVEN_DISPATCH=1; disabled by default." + ), + ) + codex_cli_local_scheduler_resident_parser.add_argument( + "--project", + default=".", + help="Project directory to start from; used for the default goal id.", + ) + codex_cli_local_scheduler_resident_parser.add_argument( + "--goal-id", + help="Goal id. Defaults to -goal.", + ) + codex_cli_local_scheduler_resident_parser.add_argument( + "--runtime-root", + default=None, + help="Runtime root that contains goals//events.jsonl.", + ) + codex_cli_local_scheduler_resident_parser.add_argument( + "--worker-id", + action="append", + default=[], + help="Worker id for the pool. Repeatable; each becomes a claimer.", + ) + codex_cli_local_scheduler_resident_parser.add_argument( + "--agent-id", + help=( + "Optional registered LoopX agent id to record on task_dispatched " + "audit events (falls back to the claimer worker id when omitted)." + ), + ) + codex_cli_local_scheduler_resident_parser.add_argument( + "--completed-todo-id", + help="Optional completed todo id that triggered this dispatch tick.", + ) + codex_cli_local_scheduler_resident_parser.add_argument( + "--iterations", + type=int, + default=1, + help="Number of resident ticks to run (bounded; 0 runs zero).", + ) + codex_cli_local_scheduler_resident_parser.add_argument( + "--interval-seconds", + type=float, + default=0.0, + help="Sleep between ticks (bounded runs only).", + ) + codex_cli_local_scheduler_resident_parser.add_argument( + "--event-driven", + action="store_true", + default=None, + help="Enable event-driven dispatch for this resident run (overrides env).", + ) + codex_cli_local_scheduler_resident_parser.add_argument( + "--execute-worker-command", + default=None, + help=( + "Optional shell command the worker runs after claiming a task " + "(opt-in; only runs when --guard-checked and an allowed prefix match)." + ), + ) + codex_cli_local_scheduler_resident_parser.add_argument( + "--worker-command-prefix", + action="append", + default=[], + help="Allow-list prefix for the worker exec command. Repeatable.", + ) + codex_cli_local_scheduler_resident_parser.add_argument( + "--guard-checked", + action="store_true", + default=False, + help="Confirm a fresh quota guard before worker exec (mirrors the original scheduler).", + ) + codex_cli_local_scheduler_resident_parser.add_argument( + "--no-reconcile", + action="store_true", + default=False, + help="Disable lease-expiry (zombie recovery) + retry promotion each tick.", + ) + codex_cli_local_scheduler_resident_parser.add_argument( + "--lease-seconds", + type=float, + default=None, + help="Claim lease TTL in seconds; expired claims are re-enqueued (zombie recovery).", + ) + codex_cli_local_scheduler_resident_parser.add_argument( + "--worker-capabilities", + action="append", + default=[], + help="worker_id=cap1,cap2 capability declarations for capability-matched claiming. Repeatable.", + ) + codex_cli_local_scheduler_resident_parser.add_argument( + "--control-plane-status", + action="store_true", + default=False, + help="Also emit the P2 control-plane observability snapshot (scheduler/worker/queue/task/decision/event history).", + ) + codex_cli_local_scheduler_resident_parser.add_argument( + "--acceptance-criteria", + action="append", + default=[], + help=( + "Declare an acceptance criterion as criterion_id=description " + "(repeatable). Closes the full loop: after worker execution, the goal " + "is closed only if every criterion has satisfying evidence." + ), + ) + codex_cli_local_scheduler_resident_parser.add_argument( + "--evidence", + action="append", + default=[], + help=( + "Declare evidence as criterion_id=kind=ref (repeatable). " + "kind in {grep,snapshot,test,file,manual}. Closes the full loop." + ), + ) + + codex_cli_local_scheduler_merge_parser = subparsers.add_parser( + "codex-cli-local-scheduler-merge", + help=( + "RFC Phase 5 merged path: heartbeat event-source fact + PolicyEngine " + "decision + event-driven dispatch in one tick. Requires " + "LOOPX_MERGE_EVENT_DRIVEN_AND_HEARTBEAT=1; disabled by default." + ), + ) + codex_cli_local_scheduler_merge_parser.add_argument( + "--project", + default=".", + help="Project directory to start from; used for the default goal id.", + ) + codex_cli_local_scheduler_merge_parser.add_argument( + "--goal-id", + help="Goal id. Defaults to -goal.", + ) + codex_cli_local_scheduler_merge_parser.add_argument( + "--agent-id", + help="Optional registered LoopX agent id for the heartbeat observation fact.", + ) + codex_cli_local_scheduler_merge_parser.add_argument( + "--runtime-root", + default=None, + help="Runtime root that contains goals//events.jsonl.", + ) + codex_cli_local_scheduler_merge_parser.add_argument( + "--completed-todo-id", + help="Optional completed todo id that triggered this dispatch tick.", + ) + codex_cli_local_scheduler_merge_parser.add_argument( + "--worker-id", + help="Optional worker/agent id to claim the next queued task.", + ) + codex_cli_local_scheduler_merge_parser.add_argument( + "--merge", + action="store_true", + default=None, + help="Enable the merged path for this invocation (overrides env).", + ) + def handle_codex_cli_local_scheduler_tick_command( args: argparse.Namespace, @@ -191,9 +450,263 @@ def handle_codex_cli_local_scheduler_exec_command( return 0 if payload.get("ok") else 1 +def handle_codex_cli_local_scheduler_dispatch_command( + args: argparse.Namespace, + print_payload: PrintPayload, +) -> int: + runtime_root = Path(args.runtime_root).expanduser() if args.runtime_root else Path( + DEFAULT_RUNTIME_ROOT + ) + goal_id = args.goal_id or default_goal_id(Path(args.project).expanduser().resolve()) + event_log_path = ( + Path(args.event_log_path).expanduser() + if args.event_log_path + else rollout_event_log_path(runtime_root, goal_id) + ) + # Load projected todo items from the dedicated events.jsonl state store when + # present; otherwise fall back to reconstructing them from the rollout event + # log (todo_add / todo_complete), which is the common shape for real goals. + items, _ = _resident_loaded_items(runtime_root, goal_id) + # Optional acceptance criteria + evidence close the full loop + # (task_completed -> acceptance -> goal_closed) in one dispatch tick. + acceptance_criteria = None + for spec in getattr(args, "acceptance_criteria", None) or []: + if "=" in spec: + cid, _, desc = spec.partition("=") + if acceptance_criteria is None: + acceptance_criteria = [] + acceptance_criteria.append({"criterion_id": cid.strip(), "description": desc.strip()}) + evidence = None + for spec in getattr(args, "evidence", None) or []: + parts = spec.split("=") + if len(parts) < 3: + continue + if evidence is None: + evidence = [] + # ID=KIND=REF[=REGEX]. When REGEX is supplied the framework independently + # greps REF for REGEX (relative to --project) instead of trusting `ok`. + # A trailing ``!`` on KIND (e.g. ``grep!``) marks an *absence* check: the + # criterion passes when REGEX is NOT found in REF. + kind = parts[1].strip() + expect = "present" + if kind.endswith("!"): + expect = "absent" + kind = kind[:-1].strip() + item: dict[str, Any] = { + "criterion_ids": [parts[0].strip()], + "kind": kind, + "ref": parts[2].strip(), + "ok": True, + "expect": expect, + } + if len(parts) >= 4: + item["pattern"] = parts[3].strip() + if kind == "grep": + # A grep evidence with a real regex is verified by the framework; the + # self-reported `ok` is only a fallback when no pattern is given. + item["ok"] = bool(item.get("pattern")) + evidence.append(item) + payload = build_event_driven_dispatch( + runtime_root=runtime_root, + goal_id=goal_id, + items=items, + completed_todo_id=args.completed_todo_id, + event_log_path=event_log_path, + worker_id=args.worker_id, + agent_id=args.agent_id, + use_event_driven=args.event_driven, + reconcile=not bool(getattr(args, "no_reconcile", False)), + worker_capabilities=None, + lease_seconds=getattr(args, "lease_seconds", None), + acceptance_criteria=acceptance_criteria, + evidence=evidence, + acceptance_base_dir=Path(args.project).expanduser().resolve() + if args.project + else None, + ) + # When this dispatch tick derived goal closure (acceptance-satisfied), keep + # the registry goal entry's status in lockstep with the rollout log so + # `status`/registry and start-goal's guided packet agree the goal is closed. + closure = (payload.get("event_driven_dispatch") or {}).get("closure") or {} + if payload.get("ok") and closure.get("ready") and getattr(args, "project", None): + try: + from ..registry import sync_registry_goal_closed + + sync_registry_goal_closed( + Path(args.project).expanduser().resolve() / ".loopx" / "registry.json", + goal_id, + ) + except Exception: + pass + print_payload(payload, args.format, render_codex_cli_local_scheduler_dispatch_markdown) + return 0 if payload.get("ok") else 1 + + +def handle_codex_cli_local_scheduler_resident_command( + args: argparse.Namespace, + print_payload: PrintPayload, +) -> int: + runtime_root = Path(args.runtime_root).expanduser() if args.runtime_root else Path( + DEFAULT_RUNTIME_ROOT + ) + goal_id = args.goal_id or default_goal_id(Path(args.project).expanduser().resolve()) + worker_capabilities: dict[str, list[str]] = {} + for spec in getattr(args, "worker_capabilities", None) or []: + if "=" in spec: + worker, _, caps = spec.partition("=") + worker_capabilities[worker.strip()] = [ + c.strip() for c in caps.split(",") if c.strip() + ] + # Optional acceptance criteria + evidence close the full loop + # (task_completed -> acceptance -> goal_closed). + acceptance_criteria = None + for spec in getattr(args, "acceptance_criteria", None) or []: + if "=" in spec: + cid, _, desc = spec.partition("=") + if acceptance_criteria is None: + acceptance_criteria = [] + acceptance_criteria.append({"criterion_id": cid.strip(), "description": desc.strip()}) + evidence = None + for spec in getattr(args, "evidence", None) or []: + parts = spec.split("=", 2) + if len(parts) >= 3: + if evidence is None: + evidence = [] + evidence.append( + { + "criterion_ids": [parts[0].strip()], + "kind": parts[1].strip(), + "ref": parts[2].strip(), + "ok": True, + } + ) + payload = run_resident_scheduler_bounded( + runtime_root=runtime_root, + goal_id=goal_id, + worker_ids=list(args.worker_id or []), + agent_id=args.agent_id, + max_iterations=int(getattr(args, "iterations", 1)), + interval_seconds=float(getattr(args, "interval_seconds", 0.0)), + completed_todo_id=args.completed_todo_id, + use_event_driven=args.event_driven, + worker_exec_command=getattr(args, "execute_worker_command", None), + worker_exec_command_prefixes=list(getattr(args, "worker_command_prefix", None) or []), + guard_checked=bool(getattr(args, "guard_checked", False)), + reconcile=not bool(getattr(args, "no_reconcile", False)), + worker_capabilities=worker_capabilities or None, + lease_seconds=getattr(args, "lease_seconds", None), + acceptance_criteria=acceptance_criteria, + evidence=evidence, + ) + # P2 observability snapshot (read-only control-plane status). + if getattr(args, "control_plane_status", False): + from ..control_plane.status.control_plane_observability import ( + build_control_plane_status, + ) + + payload["control_plane_status"] = build_control_plane_status( + runtime_root=runtime_root, + goal_id=goal_id, + worker_ids=list(args.worker_id or []), + scheduler_tick_count=payload.get("tick_count"), + ) + if args.format == "json": + print_payload(payload, args.format, None) + return 0 if payload.get("ok") else 1 + # Markdown: render the per-tick dispatch payload (the shared renderer reads a + # flat ``event_driven_dispatch`` key), then append the resident summary. + ticks = payload.get("ticks") or [] + if ticks: + last = ticks[-1] + if isinstance(last, dict) and isinstance(last.get("event_driven_dispatch"), dict): + render_payload = { + "ok": payload.get("ok"), + "goal_id": goal_id, + "event_driven_dispatch": last["event_driven_dispatch"], + } + print(render_codex_cli_local_scheduler_dispatch_markdown(render_payload)) + worker_executions = [] + if ticks and isinstance(ticks[-1].get("resident_scheduler"), dict): + worker_executions = ticks[-1]["resident_scheduler"].get("worker_executions") or [] + print( + "# Resident Scheduler\n\n" + f"- ok: `{payload.get('ok')}`\n" + f"- enabled: `{payload.get('enabled')}`\n" + f"- goal_id: `{goal_id}`\n" + f"- tick_count: `{payload.get('tick_count')}`" + ) + if worker_executions: + print("\n## Worker Executions") + for entry in worker_executions: + print( + f"- `{entry.get('claimed_by') or '?'}` / {entry.get('todo_id') or '?'} " + f"/ executed: `{entry.get('executed')}` / reason: `{entry.get('reason')}`" + ) + if payload.get("control_plane_status"): + status = payload["control_plane_status"] + q = status.get("queue") or {} + ex = q.get("extended") or {} + print( + "\n## Control-Plane Status (P2 observability)\n" + f"- queue: pending `{q.get('pending_count', 0)}`, claimed " + f"`{q.get('claimed_count', 0)}`, done `{q.get('done_count', 0)}`, " + f"in-flight `{q.get('in_flight_count', 0)}`, exceptions " + f"`{q.get('exception_count', 0)}`\n" + f"- lifecycle: retry_wait `{ex.get('retry_wait_count', 0)}`, failed " + f"`{ex.get('failed_count', 0)}`, dead_letter `{ex.get('dead_letter_count', 0)}`, " + f"cancelled `{ex.get('cancelled_count', 0)}`\n" + f"- workers: `{status.get('workers', {}).get('worker_count', 0)}` active\n" + f"- events: `{status.get('event_history', {}).get('event_count', 0)}` recorded\n" + f"- decisions: `{status.get('decision_history', {}).get('decision_count', 0)}` recorded" + ) + if payload.get("finalize"): + fin = payload["finalize"] + acc = fin.get("acceptance") or {} + gaps = acc.get("acceptance_gaps") or [] + print( + "\n## Closed Loop (task_completed -> acceptance -> closure)\n" + f"- tasks completed: `{len(fin.get('task_results') or [])}`\n" + f"- acceptance satisfied: `{acc.get('satisfied')}`" + ) + if gaps: + print( + "- acceptance gaps: `" + + "`, `".join(str(g.get("criterion_id")) for g in gaps) + + "`" + ) + print(f"- goal closed: `{fin.get('closed')}`") + return 0 if payload.get("ok") else 1 + + +def handle_codex_cli_local_scheduler_merge_command( + args: argparse.Namespace, + print_payload: PrintPayload, +) -> int: + runtime_root = Path(args.runtime_root).expanduser() if args.runtime_root else Path( + DEFAULT_RUNTIME_ROOT + ) + goal_id = args.goal_id or default_goal_id(Path(args.project).expanduser().resolve()) + payload = merge_event_driven_and_heartbeat( + runtime_root=runtime_root, + goal_id=goal_id, + agent_id=args.agent_id, + completed_todo_id=args.completed_todo_id, + worker_id=args.worker_id, + items=None, + use_event_driven=None, + use_event_source=None, + use_merge=args.merge, + ) + print_payload(payload, args.format, render_codex_cli_local_scheduler_dispatch_markdown) + return 0 if payload.get("ok") else 1 + + _SCHEDULER_HANDLERS: dict[str, Callable[[argparse.Namespace, PrintPayload], int]] = { "codex-cli-local-scheduler-tick": handle_codex_cli_local_scheduler_tick_command, "codex-cli-local-scheduler-exec": handle_codex_cli_local_scheduler_exec_command, + "codex-cli-local-scheduler-dispatch": handle_codex_cli_local_scheduler_dispatch_command, + "codex-cli-local-scheduler-resident": handle_codex_cli_local_scheduler_resident_command, + "codex-cli-local-scheduler-merge": handle_codex_cli_local_scheduler_merge_command, } diff --git a/loopx/codex_cli_probe.py b/loopx/codex_cli_probe.py index bba5c6f34..cad29135f 100644 --- a/loopx/codex_cli_probe.py +++ b/loopx/codex_cli_probe.py @@ -7,6 +7,7 @@ from .codex_cli_probe_markdown import ( render_codex_cli_bounded_visible_pilot_adapter_markdown as render_codex_cli_bounded_visible_pilot_adapter_markdown, render_codex_cli_local_driver_plan_markdown as render_codex_cli_local_driver_plan_markdown, + render_codex_cli_local_scheduler_dispatch_markdown as render_codex_cli_local_scheduler_dispatch_markdown, render_codex_cli_local_scheduler_executor_markdown as render_codex_cli_local_scheduler_executor_markdown, render_codex_cli_local_scheduler_tick_markdown as render_codex_cli_local_scheduler_tick_markdown, render_codex_cli_one_message_loop_pilot_markdown as render_codex_cli_one_message_loop_pilot_markdown, diff --git a/loopx/codex_cli_probe_markdown.py b/loopx/codex_cli_probe_markdown.py index 38cec7b3f..e3779740b 100644 --- a/loopx/codex_cli_probe_markdown.py +++ b/loopx/codex_cli_probe_markdown.py @@ -933,3 +933,92 @@ def render_codex_cli_visible_attach_acceptance_markdown(payload: dict[str, Any]) {warning_lines} """ + + +def render_codex_cli_local_scheduler_dispatch_markdown(payload: dict[str, Any]) -> str: + dispatch = payload.get("event_driven_dispatch") + if not isinstance(dispatch, dict): + dispatch = {} + queue = dispatch.get("queue") + if not isinstance(queue, dict): + queue = {} + ready_successors = ( + dispatch.get("ready_successors") if isinstance(dispatch.get("ready_successors"), list) else [] + ) + newly_enqueued = ( + dispatch.get("newly_enqueued") if isinstance(dispatch.get("newly_enqueued"), list) else [] + ) + skipped_duplicates = ( + dispatch.get("skipped_duplicates") + if isinstance(dispatch.get("skipped_duplicates"), list) + else [] + ) + dispatched = dispatch.get("dispatched") + if not isinstance(dispatched, dict): + dispatched = {} + pending = queue.get("pending_todo_ids") if isinstance(queue.get("pending_todo_ids"), list) else [] + claimed = queue.get("claimed_todo_ids") if isinstance(queue.get("claimed_todo_ids"), list) else [] + done = queue.get("done_todo_ids") if isinstance(queue.get("done_todo_ids"), list) else [] + ready_lines = "\n".join(f"- {todo_id}" for todo_id in ready_successors) if ready_successors else "- none" + enqueued_lines = "\n".join(f"- {todo_id}" for todo_id in newly_enqueued) if newly_enqueued else "- none" + skipped_lines = "\n".join(f"- {todo_id}" for todo_id in skipped_duplicates) if skipped_duplicates else "- none" + pending_lines = "\n".join(f"- {todo_id}" for todo_id in pending) if pending else "- none" + claimed_lines = "\n".join(f"- {todo_id}" for todo_id in claimed) if claimed else "- none" + done_lines = "\n".join(f"- {todo_id}" for todo_id in done) if done else "- none" + closure = dispatch.get("closure") + if not isinstance(closure, dict): + closure = None + closure_section = "" + if closure is not None: + closure_evidence = ( + closure.get("evidence") if isinstance(closure.get("evidence"), dict) else {} + ) + closure_section = f""" + +## Closure + +- ready: `{closure.get("ready")}` +- tri_state: `{closure.get("tri_state")}` +- reason: `{closure.get("reason")}` +- acceptance_satisfied: `{closure_evidence.get("acceptance_satisfied")}` +- acceptance_gap_count: `{closure_evidence.get("acceptance_gap_count")}` +- ready_todo_ids: `{len(closure_evidence.get("ready_todo_ids") or [])}` +- blocked_todo_ids: `{len(closure_evidence.get("blocked_todo_ids") or [])}` +- deferred_todo_ids: `{len(closure_evidence.get("deferred_todo_ids") or [])}` +""" + return f"""# Codex CLI Local Scheduler Dispatch + +- ok: `{payload.get("ok")}` +- disabled: `{payload.get("disabled")}` +- goal_id: `{payload.get("goal_id")}` +- event_driven_dispatch_enabled: `{dispatch.get("enabled")}` + +## Ready Successors (handoff gates recomputed) + +{ready_lines} + +## Newly Enqueued + +{enqueued_lines} + +## Skipped Duplicates + +{skipped_lines} + +## Dispatched + +- todo_id: `{dispatched.get("todo_id")}` +- claimed_by: `{dispatched.get("claimed_by")}` +- status: `{dispatched.get("status")}` +{closure_section} +## Queue + +- pending: `{queue.get("pending_count")}` / {pending_lines} +- claimed: `{queue.get("claimed_count")}` / {claimed_lines} +- done: `{queue.get("done_count")}` / {done_lines} + +## Recorded Events + +- task_ready: `{len((payload.get("recorded_events") or {}).get("task_ready") or [])}` +- task_enqueued: `{len((payload.get("recorded_events") or {}).get("task_enqueued") or [])}` +""" diff --git a/loopx/contract.py b/loopx/contract.py index b398810d1..82dfa6bc1 100644 --- a/loopx/contract.py +++ b/loopx/contract.py @@ -82,6 +82,10 @@ "dist", "node_modules", "runtime", + # loopx's own smoke/example fixtures are test stubs (incl. negative + # credential tests) and must not be treated as real project secrets. + "examples", + "docs", } LOCAL_PRIVATE_STATE_PARTS = { ".codex", @@ -92,7 +96,15 @@ "logs", "runtime", } -LOCAL_PRIVATE_STATE_FILE_NAMES = {"ACTIVE_GOAL_STATE.md", "ACTIVE_GOAL_STATE.md.lock"} +LOCAL_PRIVATE_STATE_FILE_NAMES = { + "ACTIVE_GOAL_STATE.md", + "ACTIVE_GOAL_STATE.md.lock", + # Session exports capture prior conversation text (including discussion of + # test-stub credentials / false-positive findings); scanning them as real + # secrets produces false-positive health blocks. + "session-export.json", + "session-export-.json", +} TERMINAL_TODO_STATUSES = {TODO_STATUS_DONE, TODO_STATUS_DEFERRED, "completed", "closed", "archived"} @@ -646,6 +658,11 @@ def _tracked_scan_files(scan_root: Path) -> list[Path]: path = (repo_root / rel_path).resolve() if path.name.endswith(".local.json"): continue + # Keep git-tracked files consistent with the os.walk skip rules so that + # skip-dir members (e.g. loopx's own "examples" fixtures) are not + # re-introduced into the scan via the tracked file list. + if any(part in DEFAULT_SKIP_DIRS for part in path.parts): + continue if path.is_file() and path.suffix in DEFAULT_SCAN_SUFFIXES: files.append(path) return files diff --git a/loopx/control_plane/capabilities_bridge.py b/loopx/control_plane/capabilities_bridge.py new file mode 100644 index 000000000..ff1f08877 --- /dev/null +++ b/loopx/control_plane/capabilities_bridge.py @@ -0,0 +1,504 @@ +"""Bridge the legacy capability-pack system into the new control plane. + +The legacy ``loopx/capabilities`` system exposes self-contained *capability +packs* (``issue-fix``, ``change-quality-qualification``, ``pull-request-review``, +...). Each pack declares, via ``catalog.py``, an ``entry_command`` and an ordered +``commands[]`` pipeline where every command carries a ``purpose`` and a +``write_boundary``. The new control plane models capability as *tokens* +(``required_capabilities`` / ``target_capabilities`` on todos) matched against a +worker's declared tokens in ``eligible(worker, task)``. + +This module closes the gap in three stages (see ``plan/`` for the design): + +P1 — capability packs enter ``eligible``: unify capability-token normalization + and let a task's ``capability_binding_ref`` resolve through the + ``CapabilityRegistry`` provider lifecycle (declared/installed/enabled/ready). + +P2 — capability packs self-register their CLI: a registry-driven command + registry so capability packs no longer need static ``import`` wiring in + ``cli.py``. + +P3 — capability-pack hooks become event subscriptions: a minimal event + subscription hub over the existing ``rollout_event_log`` event kinds so + legacy hooks (projection / decision-input) can be re-attached as + subscribers instead of being hard-coded into quota/configure_goal/etc. + +All three stages are written defensively against the *variety* of capability +packs in ``catalog.BUILTIN_CAPABILITIES``: missing ``commands``, missing +``workflow_skill``, missing ``default_enabled``, and internal-only packs are all +tolerated. +""" + +from __future__ import annotations + +import inspect +from collections.abc import Callable, Iterable, Mapping, Sequence +from typing import Any + +from .todos.contract import ( + TODO_CAPABILITY_BINDING_REF_PATTERN, + TODO_CAPABILITY_PATTERN, +) + + +# --------------------------------------------------------------------------- +# Shared token normalization (P1) +# --------------------------------------------------------------------------- + + +def capability_token(value: Any) -> str | None: + """Normalize a capability-pack id or raw token to a public-safe token. + + Returns ``None`` when the value cannot form a valid capability token, which + lets callers safely skip unknown/malformed inputs instead of crashing on a + capability pack they do not understand. + """ + if value is None: + return None + text = str(value).strip().lower() + if not text: + return None + # Collapse hyphen/space separators to underscores, same as the todo contract. + text = text.replace("-", "_").replace(" ", "_") + if not TODO_CAPABILITY_PATTERN.fullmatch(text): + return None + return text + + +def capability_token_set(values: Any) -> set[str]: + """Normalize a list/CSV/string of capability tokens to a token set.""" + result: set[str] = set() + if isinstance(values, str): + raw = values.replace(",", " ").split() + elif isinstance(values, (list, tuple, set)): + raw = values # type: ignore[assignment] + else: + return result + for item in raw: + token = capability_token(item) + if token: + result.add(token) + return result + + +def split_binding_ref(binding_ref: str | None) -> tuple[str, str] | None: + """Split ``namespace:value`` into ``(namespace, value)``, or None. + + The namespace is expected to be a capability-pack id (``issue-fix``); the + value is a pack-local key (``feasibility_v0``). + """ + if not binding_ref: + return None + text = str(binding_ref).strip().lower() + if not TODO_CAPABILITY_BINDING_REF_PATTERN.fullmatch(text): + return None + namespace, _, value = text.partition(":") + return namespace, value + + +# --------------------------------------------------------------------------- +# P1: registry-driven eligibility +# --------------------------------------------------------------------------- + + +def _capability_pack_states(registry: Any, include_internal: bool) -> dict[str, dict[str, bool]]: + """Map capability-pack id -> provider lifecycle state from a registry. + + Keys are the raw pack ids (e.g. ``issue-fix``). Works against any object + exposing ``records(include_internal=...)`` returning records that carry + ``provider_state`` (the ``CapabilityRegistry`` contract). Returns an empty + dict if the registry is unavailable, so eligibility degrades gracefully to + token-only matching. + """ + states: dict[str, dict[str, bool]] = {} + if registry is None: + return states + try: + records = registry.records(include_internal=include_internal) + except (AttributeError, TypeError, ValueError): + return states + for record in records: + if not isinstance(record, Mapping): + continue + rid = record.get("id") + provider_state = record.get("provider_state") or {} + states[str(rid)] = { + key: bool(provider_state.get(key)) for key in ("declared", "installed", "enabled", "ready") + } + return states + + +def capability_pack_ready(registry: Any, capability_id: str, *, include_internal: bool = False) -> bool: + """Whether a capability pack is ``ready`` in its provider lifecycle. + + Accepts either the raw id (``issue-fix``) or its normalized token + (``issue_fix``). A pack that is not registered at all is considered *not + ready* (fail closed), so a task bound to an unknown/disabled pack is not + claimable. + """ + states = _capability_pack_states(registry, include_internal) + # Build a token->id index so both spellings resolve to the same record. + by_token: dict[str, str] = {} + for rid in states: + token = capability_token(rid) + if token: + by_token.setdefault(token, rid) + token = capability_token(capability_id) + if token is None: + return False + raw_id = by_token.get(token) + if raw_id is None: + return False + state = states.get(raw_id) + return bool(state and state.get("ready", False)) + + +def resolve_required_tokens(task: Mapping[str, Any], *, registry: Any = None) -> list[str]: + """Resolve a task's effective required capability tokens. + + Merges the task's explicit ``required_capabilities`` with the capability + pack referenced by its ``capability_binding_ref`` (when present). The pack id + is added as an additional required token so workers must declare it to claim + the task. + """ + tokens: list[str] = [] + seen: set[str] = set() + + for token in capability_token_set(task.get("required_capabilities")): + if token not in seen: + seen.add(token) + tokens.append(token) + + binding = split_binding_ref(task.get("capability_binding_ref")) + if binding is not None: + pack_token = capability_token(binding[0]) + if pack_token and pack_token not in seen: + seen.add(pack_token) + tokens.append(pack_token) + + return tokens + + +def eligible_bridged( + worker: Mapping[str, Any], + task: Mapping[str, Any], + *, + registry: Any = None, +) -> bool: + """Capability-pack-aware eligibility. + + Extends plain token matching with two rules: + + * a ``capability_binding_ref`` contributes the pack id as a required token; + * when a registry is supplied, a bound pack must be ``ready``, otherwise the + task is not claimable (fail closed). + + With no binding and no registry this reduces to the original token matching. + """ + required = resolve_required_tokens(task, registry=registry) + worker_tokens = capability_token_set(worker.get("capabilities")) + + binding = split_binding_ref(task.get("capability_binding_ref")) + if binding is not None and registry is not None: + if not capability_pack_ready(registry, binding[0]): + return False + + if not required: + return True + return all(token in worker_tokens for token in required) + + +# --------------------------------------------------------------------------- +# P2: registry-driven command registration +# --------------------------------------------------------------------------- + +# A capability pack's CLI self-registration hook. Legacy packs expose +# ``register__commands(subparsers, add_subcommand_format)`` and +# ``handle__command(args, ...)``. To avoid static imports, the bridge +# calls ``register_commands`` (a single canonical entrypoint) when present, and +# falls back to ``register__commands``. Both are invoked with +# ``(subparsers, add_subcommand_format)`` because the legacy CLI injects a +# shared ``--format`` helper into every subcommand parser. +CommandRegistrar = Callable[[Any, Any], None] +CommandHandler = Callable[[Any], Any] + + +def _candidate_cli_modules(record: Mapping[str, Any], capability_id: str) -> list[tuple[str, str]]: + """Candidate ``(cli_module_path, pkg)`` pairs for a capability-pack record. + + Capability-pack ids are kebab-case but do **not** map 1:1 to CLI module + names (``pull-request-review`` lives in ``pr_review_queue.cli``; + ``integration-branch-reconcile`` in ``integration_branch.cli``). The reliable + source is the record's ``implemented_protocols[].module``, which names a + real module inside the pack (``loopx.capabilities..``). We derive + the pack package from the first such module and try ``.cli`` first, then + fall back to an id-derived snake-case path for packs without protocols. The + package name is returned alongside the path so the fallback registrar name + can be derived from the package (not the id). + """ + candidates: list[tuple[str, str]] = [] + seen: set[str] = set() + for protocol in record.get("implemented_protocols") or []: + if not isinstance(protocol, Mapping): + continue + module = str(protocol.get("module") or "").strip() + if not module.startswith("loopx.capabilities."): + continue + remainder = module[len("loopx.capabilities."):] + parts = [p for p in remainder.split(".") if p] + if not parts: + continue + pkg = parts[0] + path = f"loopx.capabilities.{pkg}.cli" + if path not in seen: + seen.add(path) + candidates.append((path, pkg)) + token = capability_token(capability_id) + if token and ":" not in token: + path = f"loopx.capabilities.{token}.cli" + if path not in seen: + candidates.append((path, token)) + return candidates + + +def _find_registrar(module: Any) -> CommandRegistrar | None: + """Find a ``register_commands`` / ``register_*_commands`` callable. + + The canonical entrypoint is ``register_commands``. Legacy packs instead use a + per-pack ``register__commands`` whose ```` does **not** follow a + single rule (``change_quality`` -> ``register_change_quality_commands`` but + ``value_connectors`` -> ``register_value_connector_commands``). We therefore + reflect over the module for any ``register_*_commands`` callable instead of + guessing the name, which is what keeps the bridge compatible with the full + capability variety. + """ + registrar = getattr(module, "register_commands", None) + if callable(registrar): + return registrar + for name in dir(module): + if not name.startswith("register_") or not name.endswith("_commands"): + continue + candidate = getattr(module, name, None) + if callable(candidate): + return candidate + return None + + +def discover_cli_registrars( + capability_records: Iterable[Mapping[str, Any]], +) -> dict[str, CommandRegistrar]: + """Discover ``register_commands`` callables from capability-pack records. + + ``capability_records`` are registry records (each carrying ``id`` and, when + present, ``implemented_protocols``). Returns a ``{capability_id: registrar}`` + map. Missing modules, missing entrypoints, import errors, and packs without + a CLI are all skipped — this tolerance is what keeps the bridge compatible + with the full capability variety (some packs have no CLI at all). + """ + import importlib + + registrars: dict[str, CommandRegistrar] = {} + for record in capability_records: + if not isinstance(record, Mapping): + continue + capability_id = str(record.get("id") or "").strip() + if not capability_id: + continue + for module_path, _pkg in _candidate_cli_modules(record, capability_id): + try: + module = importlib.import_module(module_path) + except (ImportError, AttributeError): + continue + registrar = _find_registrar(module) + if registrar is not None: + registrars[capability_id] = registrar + break + return registrars + + +def _registrar_accepts_second_arg(registrar: CommandRegistrar) -> bool: + """Whether ``registrar`` accepts a second positional argument. + + Decides the arity up front with ``inspect.signature`` instead of probing + with a trial call, which would re-invoke a two-arg registrar that raises + ``TypeError`` *inside* its body. When the signature cannot be introspected + (builtins, opaque callable objects) it falls back to the legacy two-arg + shape. + """ + try: + signature = inspect.signature(registrar) + except (TypeError, ValueError): + return True + try: + signature.bind(None, None) + except TypeError: + return False + return True + + +def register_all_capability_commands( + subparsers: Any, + add_subcommand_format: Any = None, + *, + registry: Any = None, + capability_records: Iterable[Mapping[str, Any]] | None = None, +) -> dict[str, CommandRegistrar]: + """Register every capability pack's CLI commands onto ``subparsers``. + + Capability packs are discovered from ``registry.records()`` (or the explicit + ``capability_records``), so they live in one place and no longer need static + ``import`` wiring. Each registrar is invoked with + ``(subparsers, add_subcommand_format)`` so it can attach the shared + ``--format`` helper exactly like the legacy wiring did, except registrars + whose signature only accepts ``subparsers`` are called with a single + argument. Returns the ``{capability_id: registrar}`` map that was wired up. + """ + if capability_records is None: + capability_records = [] + if registry is not None: + try: + capability_records = registry.records(include_internal=False) + except (AttributeError, TypeError, ValueError): + capability_records = [] + registrars = discover_cli_registrars(capability_records) + for registrar in registrars.values(): + if _registrar_accepts_second_arg(registrar): + registrar(subparsers, add_subcommand_format) + else: + registrar(subparsers) + return registrars + + +# --------------------------------------------------------------------------- +# P3: event subscription hub +# --------------------------------------------------------------------------- + + +class CapabilityEventHub: + """A minimal event-subscription hub for capability-pack hooks. + + Legacy capability-pack hooks were hard-coded into ``quota.py``, + ``configure_goal.py``, ``pr_review.py``, etc. This hub lets them re-attach as + subscribers keyed by the existing ``rollout_event_log`` event kinds + (``pr_merge``, ``pr_review_ack``, ``task_completed``, ...), so they no longer + need to be imported at the call site. + + Subscribers are ``callable(event)`` and may return a projection dict (for + projection hooks) or a decision input dict (for decision hooks). Unknown + event kinds are ignored rather than raising, keeping the hub tolerant of + packs that subscribe to events that never fire. + """ + + def __init__(self) -> None: + self._subscribers: dict[str, list[tuple[str, Callable[[Mapping[str, Any]], Any]]]] = {} + + def subscribe(self, event_kind: str, subscriber: Callable[[Mapping[str, Any]], Any], *, source: str = "") -> None: + """Attach ``subscriber`` to ``event_kind`` (idempotent per source).""" + kind = str(event_kind or "").strip() + if not kind: + return + bucket = self._subscribers.setdefault(kind, []) + existing = any(s == source and fn is subscriber for s, fn in bucket) + if not existing: + bucket.append((source, subscriber)) + + def publish(self, event_kind: str, event: Mapping[str, Any]) -> tuple[list[Any], list[dict[str, Any]]]: + """Deliver ``event`` to all subscribers of ``event_kind``. + + Returns ``(results, errors)``: ``results`` holds the non-None subscriber + return values; ``errors`` lists ``{"source", "error", "event_kind"}`` + dicts for subscribers that raised. Errors are reported separately so a + caller can distinguish a broken subscriber from a legitimate result + without sniffing for an ``"error"`` key. A raising subscriber is isolated + so one broken hook does not break delivery to the rest. + """ + kind = str(event_kind or "").strip() + results: list[Any] = [] + errors: list[dict[str, Any]] = [] + for source, subscriber in self._subscribers.get(kind, []): + try: + result = subscriber(event) + except Exception as exc: # noqa: BLE001 - isolate subscriber failures + errors.append({"source": source, "error": str(exc), "event_kind": kind}) + continue + if result is not None: + results.append(result) + return results, errors + + def subscribers_for(self, event_kind: str) -> list[Callable[[Mapping[str, Any]], Any]]: + return [fn for _, fn in self._subscribers.get(str(event_kind or "").strip(), [])] + + def kinds(self) -> list[str]: + return sorted(self._subscribers) + + +# A capability-pack lifecycle hook: ``callable(**kwargs) -> dict``. The hook +# returns a result dict that the host flow merges into its own payload, so a +# pack can contribute a projection or a decision input without the host needing +# to know its concrete type. +CapabilityHook = Callable[..., Mapping[str, Any]] + + +class CapabilityHookRegistry: + """A registry of capability-pack lifecycle hooks. + + Legacy packs were imported and called directly inside ``quota.py``, + ``configure_goal.py``, ``heartbeat_prequota.py``, etc. This registry lets the + same hooks register under a named *hook point* (``pre_quota``, ``goal_policy``, + ...) so host flows can collect and run them without a static import — the + capability pack self-registers instead of being wired in by hand. + + Registration is per (hook_point, source) and idempotent. Each hook is invoked + with keyword arguments and may raise; the runner isolates exceptions so one + broken pack does not break the rest, mirroring the tolerance of the event + hub. + """ + + def __init__(self) -> None: + self._hooks: dict[str, list[tuple[str, CapabilityHook]]] = {} + + def register(self, hook_point: str, hook: CapabilityHook, *, source: str = "") -> None: + """Register ``hook`` under ``hook_point`` (idempotent per source).""" + point = str(hook_point or "").strip() + if not point or not callable(hook): + return + bucket = self._hooks.setdefault(point, []) + if not any(s == source and fn is hook for s, fn in bucket): + bucket.append((source, hook)) + + def hooks_for(self, hook_point: str) -> list[CapabilityHook]: + return [fn for _, fn in self._hooks.get(str(hook_point or "").strip(), [])] + + def run(self, hook_point: str, **kwargs: Any) -> list[Mapping[str, Any]]: + """Run every hook under ``hook_point`` with ``kwargs``. + + Returns the list of hook result dicts. A raising hook contributes an + ``{"ok": False, "error": ...}`` result instead of aborting the run. + """ + results: list[Mapping[str, Any]] = [] + for hook in self.hooks_for(hook_point): + try: + result = hook(**kwargs) + except Exception as exc: # noqa: BLE001 - isolate hook failures + results.append({"ok": False, "error": str(exc), "hook_point": hook_point}) + continue + if isinstance(result, Mapping): + results.append(dict(result)) + return results + + def hook_points(self) -> list[str]: + return sorted(self._hooks) + + +__all__ = [ + "capability_token", + "capability_token_set", + "split_binding_ref", + "capability_pack_ready", + "resolve_required_tokens", + "eligible_bridged", + "discover_cli_registrars", + "register_all_capability_commands", + "CapabilityEventHub", + "CapabilityHookRegistry", +] diff --git a/loopx/control_plane/goals/goal_acceptance.py b/loopx/control_plane/goals/goal_acceptance.py new file mode 100644 index 000000000..586f10273 --- /dev/null +++ b/loopx/control_plane/goals/goal_acceptance.py @@ -0,0 +1,390 @@ +"""Goal Acceptance / Evidence Verification (plan §5.10). + +Closure is NOT just "no work left" — a goal must also be *actually realized* with +sufficient evidence. This module answers: "was the goal truly achieved, and is +the evidence adequate to prove it?". + +The evaluation runs BEFORE the Closure Evaluator's `is_goal_closable`: + + last todo done -> Scheduler (ready = []) + -> Goal Acceptance Evaluator (evidence sufficient?) + -> satisfied -> Closure Evaluator -> goal_closed + -> insufficient -> emit goal_acceptance_pending (not close) + +An acceptance criterion is a declarative check, e.g. "theme color is green +(#22c55e)". Evidence is a collection of artifacts (grep hit, test pass, file +snapshot, ...). A criterion is *satisfied* when there is at least one +``satisfying`` evidence ref for it; otherwise it becomes an ``acceptance_gap``. +""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import Any, Mapping, Sequence + +from ...rollout_event_log import append_rollout_event_once, build_rollout_event + +GOAL_ACCEPTANCE_EVALUATION_SCHEMA_VERSION = "goal_acceptance_evaluation_v0" +GOAL_ACCEPTANCE_CRITERIA_SCHEMA_VERSION = "goal_acceptance_criteria_v0" + +# Evidence kinds understood by the evaluator. +EVIDENCE_KIND_GREP = "grep" +EVIDENCE_KIND_SNAPSHOT = "snapshot" +EVIDENCE_KIND_TEST = "test" +EVIDENCE_KIND_FILE = "file" +EVIDENCE_KIND_MANUAL = "manual" +EVIDENCE_KINDS = { + EVIDENCE_KIND_GREP, + EVIDENCE_KIND_SNAPSHOT, + EVIDENCE_KIND_TEST, + EVIDENCE_KIND_FILE, + EVIDENCE_KIND_MANUAL, +} + + +def normalize_acceptance_criteria( + criteria: Sequence[Mapping[str, Any]] | None, +) -> list[dict[str, Any]]: + """Normalize acceptance criteria into ``{criterion_id, description, kind}``.""" + result: list[dict[str, Any]] = [] + for item in criteria or (): + if not isinstance(item, dict): + continue + criterion_id = str(item.get("criterion_id") or item.get("id") or "").strip() + description = str(item.get("description") or "").strip() + if not criterion_id: + # Derive an id from the description hash when absent. + criterion_id = _slug(description) + kind = str(item.get("kind") or "assert").strip() or "assert" + result.append( + { + "criterion_id": criterion_id, + "description": description, + "kind": kind, + } + ) + return result + + +def _slug(text: str) -> str: + return "".join(ch if ch.isalnum() or ch in "-_" else "_" for ch in text)[:64] or "criterion" + + +def normalize_evidence( + evidence: Sequence[Mapping[str, Any]] | None, +) -> list[dict[str, Any]]: + """Normalize evidence into ``{evidence_id, kind, ref, content, ok}``.""" + result: list[dict[str, Any]] = [] + for item in evidence or (): + if not isinstance(item, dict): + continue + evidence_id = str(item.get("evidence_id") or item.get("id") or "").strip() + if not evidence_id: + evidence_id = f"ev:{len(result)}" + kind = str(item.get("kind") or EVIDENCE_KIND_MANUAL).strip() + if kind not in EVIDENCE_KINDS: + kind = EVIDENCE_KIND_MANUAL + result.append( + { + "evidence_id": evidence_id, + "kind": kind, + "ref": str(item.get("ref") or ""), + "pattern": str(item.get("pattern") or ""), + "content": str(item.get("content") or ""), + "ok": bool(item.get("ok", True)), + "expect": str(item.get("expect") or "present").strip() or "present", + "criterion_ids": [ + str(c) for c in (item.get("criterion_ids") or []) if str(c) + ], + } + ) + return result + + +def verify_criterion( + criterion: Mapping[str, Any], + evidence: Sequence[Mapping[str, Any]], +) -> dict[str, Any]: + """Verify a single criterion against the evidence list. + + A criterion is *satisfied* when at least one piece of ``ok`` evidence + references it (by ``criterion_id``) or, if the criterion declares no specific + evidence requirement, when there is at least one ``ok`` manual/snapshot + evidence. Returns ``{criterion_id, satisfied, evidence_refs}``. + """ + criterion_id = str(criterion.get("criterion_id") or "").strip() + description = str(criterion.get("description") or "").strip() + matching: list[str] = [] + for ev in evidence: + if not ev.get("ok"): + continue + ref_ids = ev.get("criterion_ids") or [] + if criterion_id in ref_ids: + matching.append(str(ev.get("evidence_id") or "")) + satisfied = bool(matching) + return { + "criterion_id": criterion_id, + "description": description, + "satisfied": satisfied, + "evidence_refs": matching, + } + + +def evaluate_goal_acceptance( + *, + acceptance_criteria: Sequence[Mapping[str, Any]] | None, + evidence: Sequence[Mapping[str, Any]] | None, + base_dir: Path | str | None = None, +) -> dict[str, Any]: + """Run the Acceptance Evaluator. + + Args: + base_dir : when provided, ``grep``-kind evidence is *independently verified* + against the real file (framework reads the file + regex-matches), + instead of trusting the caller-supplied ``ok`` flag. This is the + "self-reported -> verified" step. Relative ``ref`` paths resolve + under ``base_dir``. + + Returns: + satisfied : every criterion has satisfying evidence + acceptance_gaps : list of unsatisfied criteria + criteria_results : per-criterion verification + evidence_count : number of ok evidence items + verified_count : number of evidence items independently re-verified + """ + criteria = normalize_acceptance_criteria(acceptance_criteria) + evidence_list = normalize_evidence(evidence) + # Independent verification first: recompute ``ok`` for grep evidence from the + # actual file when a base_dir is available. The caller's self-reported flag is + # only a fallback when the target/pattern is absent or kind is not grep. + if base_dir is not None: + evidence_list = [ + verify_grep_evidence(e, base_dir=Path(base_dir)) for e in evidence_list + ] + ok_evidence = [e for e in evidence_list if e.get("ok")] + verified_count = sum(1 for e in evidence_list if e.get("verified")) + + # When no criteria are declared, treat acceptance as satisfied (nothing to + # prove) unless the caller explicitly marks acceptance required. + if not criteria: + return { + "schema_version": GOAL_ACCEPTANCE_EVALUATION_SCHEMA_VERSION, + "satisfied": True, + "acceptance_gaps": [], + "criteria_results": [], + "evidence_count": len(ok_evidence), + "verified_count": verified_count, + "criteria_count": 0, + } + + results = [verify_criterion(c, evidence_list) for c in criteria] + gaps = [r for r in results if not r["satisfied"]] + return { + "schema_version": GOAL_ACCEPTANCE_EVALUATION_SCHEMA_VERSION, + "satisfied": len(gaps) == 0, + "acceptance_gaps": gaps, + "criteria_results": results, + "evidence_count": len(ok_evidence), + "verified_count": verified_count, + "criteria_count": len(criteria), + } + + +def build_grep_evidence( + *, + ref: str, + pattern: str, + match: bool, + content: str = "", + criterion_ids: Sequence[str] = (), +) -> dict[str, Any]: + """Convenience builder for a grep-based evidence item.""" + return { + "evidence_id": f"grep:{pattern}", + "kind": EVIDENCE_KIND_GREP, + "ref": ref, + "pattern": pattern, + "content": content or pattern, + "ok": match, + "criterion_ids": list(criterion_ids), + } + + +def build_manual_evidence( + *, + ref: str, + content: str, + ok: bool = True, + criterion_ids: Sequence[str] = (), +) -> dict[str, Any]: + """Convenience builder for a manual/operator-confirmed evidence item.""" + return { + "evidence_id": f"manual:{ref}", + "kind": EVIDENCE_KIND_MANUAL, + "ref": ref, + "content": content, + "ok": ok, + "criterion_ids": list(criterion_ids), + } + + +def verify_grep_evidence( + evidence: dict[str, Any], + base_dir: Path | None = None, +) -> dict[str, Any]: + """Independently verify a ``grep`` evidence item against the actual file. + + This is the step that moves acceptance from *self-reported* to *verified*: + instead of trusting a caller-supplied ``ok`` flag, the framework reads the + target file and checks whether the regex pattern actually matches. The + evidence's ``ok`` is recomputed from the real result. + + Resolution rules for the target file: + * if ``ref`` is an absolute path, it is used directly; + * otherwise ``base_dir / ref`` (when ``base_dir`` is provided) or ``cwd / ref``. + + Returns a copy of ``evidence`` with ``ok``/``content``/``verified`` updated. + ``verified=True`` means the framework actually performed the check; when the + file is unreadable the evidence is treated as *not ok* but still ``verified`` + (the mismatch is the finding). Evidence lacking a regex ``pattern`` is left + untouched (``verified=False``) so it degrades to the prior manual path. + """ + if evidence.get("kind") != EVIDENCE_KIND_GREP: + return dict(evidence) + ref = str(evidence.get("ref") or "").strip() + pattern = str(evidence.get("pattern") or evidence.get("content") or "").strip() + if not ref or not pattern: + return dict(evidence) # not enough info to verify — keep caller's ok + + candidate: Path + p = Path(ref) + if p.is_absolute(): + candidate = p + elif base_dir is not None: + candidate = Path(base_dir) / p + else: + candidate = p + candidate = candidate.resolve() + + result = dict(evidence) + try: + text = candidate.read_text(encoding="utf-8", errors="replace") + except Exception: + # Unreadable target is a genuine negative finding. + result["ok"] = False + result["verified"] = True + result["verification_error"] = f"unreadable target: {candidate}" + return result + + try: + hit = re.search(pattern, text) is not None + except re.error as exc: + result["ok"] = False + result["verified"] = True + result["verification_error"] = f"invalid regex {pattern!r}: {exc}" + return result + + # Expectation semantics: ``expect="absent"`` means the criterion is an + # absence check ("X must NOT be present"), so a *miss* is the passing + # outcome. ``present`` (default) keeps the positive match semantics. + expect = str(result.get("expect") or "present").strip().lower() + result["ok"] = (not hit) if expect == "absent" else hit + result["verified"] = True + result["matched_lines"] = _count_matches(text, pattern) if hit else 0 + return result + + +def _count_matches(text: str, pattern: str) -> int: + try: + return len(re.findall(pattern, text)) + except re.error: + return 0 + + +def emit_goal_acceptance_pending( + *, + log_path: Path, + goal_id: str, + acceptance_gaps: Sequence[Mapping[str, Any]], + agent_id: str | None = None, + recorded_at: str | None = None, +) -> dict[str, Any]: + """Emit a ``goal_acceptance_pending`` audit event (idempotent by goal+kind). + + The goal is NOT closable until the acceptance gaps are resolved. + """ + event = build_rollout_event( + goal_id=goal_id, + event_kind="goal_acceptance_pending", + agent_id=agent_id, + recorded_at=recorded_at, + ) + event["acceptance_gaps"] = list(acceptance_gaps) + appended, _is_new = append_rollout_event_once( + Path(log_path), + event, + identity_fields=("goal_id", "event_kind"), + ) + return appended + + +def emit_goal_acceptance_satisfied( + *, + log_path: Path, + goal_id: str, + criteria_results: Sequence[Mapping[str, Any]], + agent_id: str | None = None, + recorded_at: str | None = None, +) -> dict[str, Any]: + """Emit a ``goal_acceptance_satisfied`` audit event (idempotent by goal+kind).""" + event = build_rollout_event( + goal_id=goal_id, + event_kind="goal_acceptance_satisfied", + agent_id=agent_id, + recorded_at=recorded_at, + ) + event["criteria_results"] = list(criteria_results) + appended, _is_new = append_rollout_event_once( + Path(log_path), + event, + identity_fields=("goal_id", "event_kind"), + ) + return appended + + +def acceptance_blocker( + acceptance: Mapping[str, Any] | None, +) -> str | None: + """Return a closure-blocker reason when acceptance is not satisfied, else None. + + Designed to be folded into ``is_goal_closable`` / ``goal_closure_reason`` so + a goal with unsatisfied acceptance criteria is treated as NOT closable. + """ + if not isinstance(acceptance, dict): + return None + if acceptance.get("satisfied") is True: + return None + gaps = acceptance.get("acceptance_gaps") or [] + return "acceptance_gaps_remaining" if gaps else "acceptance_not_verified" + + +__all__ = [ + "GOAL_ACCEPTANCE_EVALUATION_SCHEMA_VERSION", + "GOAL_ACCEPTANCE_CRITERIA_SCHEMA_VERSION", + "EVIDENCE_KIND_GREP", + "EVIDENCE_KIND_SNAPSHOT", + "EVIDENCE_KIND_TEST", + "EVIDENCE_KIND_FILE", + "EVIDENCE_KIND_MANUAL", + "EVIDENCE_KINDS", + "normalize_acceptance_criteria", + "normalize_evidence", + "verify_criterion", + "evaluate_goal_acceptance", + "build_grep_evidence", + "build_manual_evidence", + "emit_goal_acceptance_pending", + "emit_goal_acceptance_satisfied", + "acceptance_blocker", +] diff --git a/loopx/control_plane/goals/goal_channel_projection.py b/loopx/control_plane/goals/goal_channel_projection.py index 3dde301ed..4cbcb3086 100644 --- a/loopx/control_plane/goals/goal_channel_projection.py +++ b/loopx/control_plane/goals/goal_channel_projection.py @@ -127,6 +127,56 @@ def _source_refs( } +def _compact_scheduler_hint(value: Any) -> dict[str, Any] | None: + """Compact a Phase 5 ``scheduler_hint`` for the frontstage projection. + + Whitelists only public-safe, scalar/lightweight scheduler contract fields so + the frontstage Budget & Governance block can show real cadence decisions + instead of demo placeholders. Raw/private material is never copied. + """ + source = _as_mapping(value) + if not source: + return None + compact: dict[str, Any] = {} + for key in ("action", "cadence_class", "reason_code", "spend_policy"): + text = _text(source.get(key), limit=180) + if text is not None: + compact[key] = text + reason = _text(source.get("reason"), limit=300) + if reason is not None: + compact["reason"] = reason + heartbeat = _as_mapping(source.get("heartbeat_recommendation")) + if heartbeat: + heart_compact: dict[str, Any] = {} + for key in ("recommended_mode", "cadence_class", "recommended_interval_seconds"): + text = _text(heartbeat.get(key), limit=120) + if text is not None: + heart_compact[key] = text + if heart_compact: + compact["heartbeat_recommendation"] = heart_compact + return compact + + +def _compact_policy_decision(value: Any) -> dict[str, Any] | None: + """Compact a Phase 5 ``policy_decision`` for the frontstage projection. + + Only the unified decision outcome/source/reason/retry surface is exposed; + never the raw underlying payloads. + """ + source = _as_mapping(value) + if not source: + return None + compact: dict[str, Any] = {} + for key in ("outcome", "source", "reason", "retry_after_seconds", "manual_approval_required"): + if key not in source: + continue + child = source[key] + text = _text(child, limit=220) if not isinstance(child, Mapping) else None + if text is not None: + compact[key] = text + return compact + + def _compact_quota(quota_payload: Mapping[str, Any], project_asset: Mapping[str, Any]) -> dict[str, Any]: source = quota_payload.get("quota") if isinstance(quota_payload.get("quota"), Mapping) else {} if not source and isinstance(project_asset.get("quota"), Mapping): @@ -136,6 +186,21 @@ def _compact_quota(quota_payload: Mapping[str, Any], project_asset: Mapping[str, value = _text(source.get(key), limit=220) if value is not None: compact[key] = value + # Phase 5 new-architecture pass-through: when the quota source carries the + # unified scheduler hint / policy decision, surface their public-safe fields + # so the frontstage Budget & Governance block reads real cadence decisions. + # When absent (legacy path) the output is byte-identical to before (opt-in). + scheduler_hint = _compact_scheduler_hint(source.get("scheduler_hint")) + if scheduler_hint: + compact["scheduler_hint"] = scheduler_hint + policy_decision = _compact_policy_decision(source.get("policy_decision")) + if policy_decision: + compact["policy_decision"] = policy_decision + # Frontstage also reads top-level cadence scalar keys directly. + for key in ("scheduler_rrule", "scheduler_reset_token", "cadence_class"): + text = _text(source.get(key), limit=160) + if text is not None: + compact[key] = text return compact diff --git a/loopx/control_plane/goals/goal_closure.py b/loopx/control_plane/goals/goal_closure.py new file mode 100644 index 000000000..ea6ec6bab --- /dev/null +++ b/loopx/control_plane/goals/goal_closure.py @@ -0,0 +1,313 @@ +"""Goal Closure Evaluator + Controller (event-driven, decoupled from Todo). + +Implements the elegant goal-closure design: **Todo lifecycle and Goal lifecycle +are separate**. A Todo only answers "did this piece of work get done?"; a Goal +answers "should this goal keep running?". The Closure Evaluator derives whether +a goal is closable purely from current state — it does NOT require every todo to +carry an explicit ``no_followup`` intent. + +Closure is a *derived, deterministic event*, not something the agent has to +"figure out". The flow: + + Todo/Event state change + -> State Reducer + -> Scheduler (advance_ready_todo_ids) + -> Closure Evaluator: is_goal_closable(state)? + - ready_todo_ids empty? + - pending_dependencies empty? + - replan_required False? + - external_followup_required False? + -> YES -> emit goal_closure_ready(reason, evidence) + -> Goal Controller -> goal_closed + (kind=derived | explicit) +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Mapping, Sequence + +from ...rollout_event_log import append_rollout_event_once, build_rollout_event + +GOAL_CLOSURE_EVALUATION_SCHEMA_VERSION = "goal_closure_evaluation_v0" +GOAL_CLOSURE_STATE_SCHEMA_VERSION = "goal_closure_state_v0" + +# Wait/close classification (the RUN/WAIT/CLOSE tri-state for a goal). +GOAL_RUN = "RUN" +GOAL_WAIT = "WAIT" +GOAL_CLOSE = "CLOSE" + + +def _empty(value: Any) -> bool: + return value is None or value == [] or value == {} or value == set() + + +def _zero(value: Any) -> bool: + try: + return int(value or 0) <= 0 + except (TypeError, ValueError): + return True + + +def goal_closure_reason(state: Mapping[str, Any]) -> str | None: + """Return the reason a goal is not closable, or None when it is closable. + + ``state`` is a compact goal-state read model with the fields: + ready_todo_ids (sequence) + pending_dependency_ids (sequence) - blocked / waiting-for-user deps + replan_required (bool) + external_followup_required (bool) + open_todo_count (int) - optional fallback for open work + claimed_advancement_count (int) - optional fallback + """ + if not isinstance(state, dict): + return "state_missing" + if state.get("ready_todo_ids"): + return "ready_work_remaining" + if state.get("pending_dependency_ids"): + return "pending_dependencies" + if state.get("blocked_todo_ids"): + return "blocked_work_pending" + if state.get("deferred_todo_ids"): + return "deferred_work_pending" + if state.get("replan_required") is True: + return "replan_required" + if state.get("external_followup_required") is True: + return "external_followup_required" + # Goal Acceptance: the goal must be *actually realized* with sufficient + # evidence before it can close. Unsatisfied acceptance criteria block close. + acceptance = state.get("acceptance") + if isinstance(acceptance, dict): + from .goal_acceptance import acceptance_blocker + + blocker = acceptance_blocker(acceptance) + if blocker is not None: + return blocker + # Fallbacks when only counts are supplied. (ready_todo_ids is already known + # to be empty here — it is checked first above — so only the count fallbacks + # remain meaningful.) + if not _zero(state.get("open_todo_count")): + return "open_work_remaining" + if not _zero(state.get("claimed_advancement_count")): + return "claimed_advancement_in_flight" + return None + + +def is_goal_closable(state: Mapping[str, Any]) -> bool: + """The elegant single rule: a goal is closable iff there is no work left. + + ``not ready_todos and not pending_dependencies and not replan_required + and not external_followup_required`` — with no per-todo ``no_followup`` + requirement. + """ + return goal_closure_reason(state) is None + + +def classify_goal_continuation(state: Mapping[str, Any]) -> str: + """Return the RUN/WAIT/CLOSE tri-state for a goal. + + * RUN — there is ready, executable work. + * WAIT — there is future (blocked / deferred / waiting) work, not closable. + * CLOSE — no executable work, no pending deps, no replan, no follow-up. + """ + if not isinstance(state, dict): + return GOAL_WAIT + if state.get("ready_todo_ids"): + return GOAL_RUN + reason = goal_closure_reason(state) + if reason is None: + return GOAL_CLOSE + return GOAL_WAIT + + +def evaluate_goal_closure(state: Mapping[str, Any]) -> dict[str, Any]: + """Run the Closure Evaluator and return a structured evaluation. + + Returns ``{ready: bool, tri_state, reason, evidence}``. ``ready=True`` means + the goal should be closed now (no further agent action required to close it). + """ + reason = goal_closure_reason(state) + ready = reason is None + return { + "schema_version": GOAL_CLOSURE_EVALUATION_SCHEMA_VERSION, + "ready": ready, + "tri_state": GOAL_CLOSE if ready else classify_goal_continuation(state), + "reason": reason or "no_followup_work", + "evidence": { + "ready_todo_ids": list(state.get("ready_todo_ids") or []), + "blocked_todo_ids": list(state.get("blocked_todo_ids") or []), + "deferred_todo_ids": list(state.get("deferred_todo_ids") or []), + "pending_dependency_ids": list(state.get("pending_dependency_ids") or []), + "replan_required": state.get("replan_required") is True, + "external_followup_required": state.get("external_followup_required") is True, + "acceptance_satisfied": bool( + (state.get("acceptance") or {}).get("satisfied") + ), + "acceptance_gap_count": len( + (state.get("acceptance") or {}).get("acceptance_gaps") or [] + ), + }, + } + + +def build_goal_closure_state( + *, + ready_todo_ids: Sequence[str] = (), + pending_dependency_ids: Sequence[str] = (), + blocked_todo_ids: Sequence[str] = (), + deferred_todo_ids: Sequence[str] = (), + replan_required: bool = False, + external_followup_required: bool = False, + open_todo_count: int | None = None, + claimed_advancement_count: int | None = None, + acceptance: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + """Build a compact goal-state read model for the Closure Evaluator. + + ``acceptance`` is the output of :func:`evaluate_goal_acceptance`; when its + criteria are unsatisfied, the goal is blocked from closing (WAIT / pending). + """ + return { + "schema_version": GOAL_CLOSURE_STATE_SCHEMA_VERSION, + "ready_todo_ids": list(ready_todo_ids or []), + "blocked_todo_ids": list(blocked_todo_ids or []), + "deferred_todo_ids": list(deferred_todo_ids or []), + "pending_dependency_ids": list(pending_dependency_ids or []), + "replan_required": bool(replan_required), + "external_followup_required": bool(external_followup_required), + "open_todo_count": open_todo_count, + "claimed_advancement_count": claimed_advancement_count, + "acceptance": dict(acceptance) if acceptance is not None else None, + } + + +def emit_goal_closure_ready( + *, + log_path: Path, + goal_id: str, + reason: str, + evidence: Mapping[str, Any] | None = None, + agent_id: str | None = None, + recorded_at: str | None = None, +) -> dict[str, Any]: + """Emit a ``goal_closure_ready`` audit event (idempotent by goal+kind). + + This is the single point where the system declares "there is no next step"; + it is derived from state, not from an agent calling a tool. + """ + event = build_rollout_event( + goal_id=goal_id, + event_kind="goal_closure_ready", + agent_id=agent_id, + recorded_at=recorded_at, + ) + event["reason"] = reason + if evidence: + event["evidence"] = dict(evidence) + appended, _is_new = append_rollout_event_once( + Path(log_path), + event, + identity_fields=("goal_id", "event_kind"), + ) + return appended + + +def emit_goal_closed( + *, + log_path: Path, + goal_id: str, + kind: str = "derived", + reason: str = "no_followup_work", + agent_id: str | None = None, + recorded_at: str | None = None, +) -> dict[str, Any]: + """Emit a ``goal_closed`` audit event and return it (Goal Controller action). + + ``kind`` distinguishes a system-derived close (``derived``) from an explicit + user/agent requested close (``explicit``). + """ + event = build_rollout_event( + goal_id=goal_id, + event_kind="goal_closed", + agent_id=agent_id, + recorded_at=recorded_at, + ) + event["kind"] = "derived" if kind == "derived" else "explicit" + event["reason"] = reason + appended, _is_new = append_rollout_event_once( + Path(log_path), + event, + identity_fields=("goal_id", "event_kind"), + ) + return appended + + +def maybe_close_goal( + *, + log_path: Path, + goal_id: str, + state: Mapping[str, Any], + agent_id: str | None = None, +) -> dict[str, Any]: + """Run the Closure Evaluator and, when ready, emit closure_ready + goal_closed. + + This is the complete, atomic "Goal Controller" step for the event-driven + path: it evaluates closure and, if closable, records both the readiness event + and the closed event. Returns the evaluation (``ready`` tells whether it + closed). + + When the goal-state carries an **unsatisfied acceptance evaluation**, the + goal is NOT closed; instead a ``goal_acceptance_pending`` event is emitted so + the evidence gaps must be resolved before the goal can finish. + """ + evaluation = evaluate_goal_closure(state) + if not evaluation["ready"]: + # Surface a goal_acceptance_pending event when evidence is insufficient, + # so the operator/agent knows the goal is held for verification. + acceptance = state.get("acceptance") if isinstance(state, dict) else None + if isinstance(acceptance, dict) and acceptance.get("satisfied") is not True: + gaps = acceptance.get("acceptance_gaps") or [] + if gaps: + from .goal_acceptance import emit_goal_acceptance_pending + + emit_goal_acceptance_pending( + log_path=log_path, + goal_id=goal_id, + acceptance_gaps=gaps, + agent_id=agent_id, + ) + return evaluation + emit_goal_closure_ready( + log_path=log_path, + goal_id=goal_id, + reason=evaluation["reason"], + evidence=evaluation["evidence"], + agent_id=agent_id, + ) + emit_goal_closed( + log_path=log_path, + goal_id=goal_id, + kind="derived", + reason=evaluation["reason"], + agent_id=agent_id, + ) + evaluation["closed"] = True + evaluation["closed_at"] = None # timestamp set by the caller via recorded_at + return evaluation + + +__all__ = [ + "GOAL_CLOSURE_EVALUATION_SCHEMA_VERSION", + "GOAL_CLOSURE_STATE_SCHEMA_VERSION", + "GOAL_RUN", + "GOAL_WAIT", + "GOAL_CLOSE", + "goal_closure_reason", + "is_goal_closable", + "classify_goal_continuation", + "evaluate_goal_closure", + "build_goal_closure_state", + "emit_goal_closure_ready", + "emit_goal_closed", + "maybe_close_goal", +] diff --git a/loopx/control_plane/heartbeat/event_source.py b/loopx/control_plane/heartbeat/event_source.py new file mode 100644 index 000000000..6a374e751 --- /dev/null +++ b/loopx/control_plane/heartbeat/event_source.py @@ -0,0 +1,243 @@ +"""Heartbeat as an event source (RFC Phase 5 comprehensive eventing). + +Heartbeat is demoted from a control-plane decision owner to a *trigger*: it +only produces observable event facts (``heartbeat_observed``), while the +"should this task run?" decision is owned by :class:`PolicyEngine`. + +Design rules (RFC §11.2, §5.3): + +* The heartbeat bounded context still *renders* prompts (``builder.py``), but + the decision to run is delegated to the unified :class:`PolicyEngine`. +* ``record_heartbeat_observation`` writes a public-safe ``heartbeat_observed`` + audit fact only. It never mutates Task / Goal state and never carries raw + task text, transcripts, or credentials (the rollout event boundary already + strips those). +* Recording is idempotent per (goal, agent, source, heartbeat tick) via a + deterministic observation fingerprint, so repeated polling does not grow the + event log without bound. +* Everything here is opt-in behind ``LOOPX_HEARTBEAT_EVENT_SOURCE`` (or an + explicit ``use_event_source`` flag); the default path is unchanged. +""" + +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +from typing import Any, Mapping + +from ...rollout_event_log import ( + append_rollout_event_once, + build_rollout_event, + rollout_event_log_path, +) +from ..new_architecture import master_switch_enabled +from ..runtime.time import now_utc_iso + +HEARTBEAT_EVENT_SOURCE_ENV = "LOOPX_HEARTBEAT_EVENT_SOURCE" + +HEARTBEAT_OBSERVED_EVENT_KIND = "heartbeat_observed" +HEARTBEAT_EVENT_SCHEMA_VERSION = "loopx_heartbeat_observation_v0" + +DEFAULT_HEARTBEAT_SOURCE = "heartbeat_poll" + + +def heartbeat_event_source_enabled(use_event_source: bool | None = None) -> bool: + """Enable the heartbeat event source. + + An explicit ``use_event_source`` wins; otherwise the dedicated env var wins; + otherwise the new-architecture master switch decides (on by default). + """ + if use_event_source is not None: + return bool(use_event_source) + value = os.environ.get(HEARTBEAT_EVENT_SOURCE_ENV, "").strip().lower() + if value: + return value in {"1", "true", "yes", "on"} + return master_switch_enabled() + + +def compute_observation_fingerprint( + *, + goal_id: str, + agent_id: str | None, + source: str, + tick_id: str | None = None, +) -> str: + """Deterministic identity for one heartbeat observation occurrence. + + The fingerprint covers the stable observation identity, deliberately + excluding timestamps and free-form detail so an unchanged poll collapses + to a single ``heartbeat_observed`` event. + """ + stable: dict[str, Any] = { + "goal_id": str(goal_id or "").strip(), + "agent_id": str(agent_id or "").strip(), + "source": str(source or "").strip(), + "tick_id": str(tick_id or "").strip(), + } + encoded = json.dumps(stable, sort_keys=True, ensure_ascii=True).encode("utf-8") + return hashlib.sha256(encoded).hexdigest()[:16] + + +def build_heartbeat_observation_event( + *, + goal_id: str, + agent_id: str | None = None, + source: str = DEFAULT_HEARTBEAT_SOURCE, + tick_id: str | None = None, + status: str | None = None, + details: Mapping[str, Any] | None = None, + recorded_at: str | None = None, +) -> dict[str, Any]: + """Build one public-safe ``heartbeat_observed`` rollout event fact. + + This is a fact-only payload: it records *that* a heartbeat observation + occurred and which trigger produced it. It carries no business decision + and no sensitive content. The decision belongs to ``PolicyEngine``. + """ + event = build_rollout_event( + goal_id=goal_id, + event_kind=HEARTBEAT_OBSERVED_EVENT_KIND, + agent_id=agent_id, + classification=str(source or DEFAULT_HEARTBEAT_SOURCE), + status=status, + summary="heartbeat observation fact", + details={ + "schema_version": HEARTBEAT_EVENT_SCHEMA_VERSION, + "source": str(source or DEFAULT_HEARTBEAT_SOURCE), + "tick_id": str(tick_id or "").strip() if tick_id else None, + **(details or {}), + }, + recorded_at=recorded_at or now_utc_iso(), + ) + event["heartbeat_fingerprint"] = compute_observation_fingerprint( + goal_id=goal_id, + agent_id=agent_id, + source=source, + tick_id=tick_id, + ) + return event + + +def record_heartbeat_observation( + *, + runtime_root: Path, + goal_id: str, + agent_id: str | None = None, + event_log_path: Path | None = None, + source: str = DEFAULT_HEARTBEAT_SOURCE, + tick_id: str | None = None, + status: str | None = None, + details: Mapping[str, Any] | None = None, + recorded_at: str | None = None, + use_event_source: bool | None = None, +) -> dict[str, Any]: + """Record one ``heartbeat_observed`` event fact (idempotent per observation). + + Returns a summary with the appended event and whether it was new. When the + opt-in flag is off, returns a ``disabled`` marker and writes nothing. + """ + if not heartbeat_event_source_enabled(use_event_source): + return { + "ok": True, + "disabled": True, + "reason": f"{HEARTBEAT_EVENT_SOURCE_ENV} not enabled", + "goal_id": str(goal_id or "").strip(), + } + log_path = ( + Path(event_log_path) + if event_log_path is not None + else rollout_event_log_path(runtime_root, goal_id) + ) + event = build_heartbeat_observation_event( + goal_id=goal_id, + agent_id=agent_id, + source=source, + tick_id=tick_id, + status=status, + details=details, + recorded_at=recorded_at, + ) + appended, is_new = append_rollout_event_once( + log_path, + event, + identity_fields=( + "goal_id", + "heartbeat_fingerprint", + ), + ) + return { + "ok": True, + "goal_id": str(goal_id or "").strip(), + "event": appended, + "new": is_new, + } + + +class HeartbeatEventSource: + """Degrade heartbeat into a fact-only event source. + + ``observe()`` writes the ``heartbeat_observed`` fact. Decision evaluation is + *not* performed here; callers pass the fact to :class:`PolicyEngine` + (or use ``merge_event_driven_control_plane``) when eventing is enabled. + """ + + def __init__( + self, + *, + runtime_root: Path, + goal_id: str, + agent_id: str | None = None, + event_log_path: Path | None = None, + source: str = DEFAULT_HEARTBEAT_SOURCE, + ) -> None: + self._runtime_root = Path(runtime_root) + self._goal_id = str(goal_id or "").strip() + self._agent_id = agent_id + self._event_log_path = ( + Path(event_log_path) + if event_log_path is not None + else rollout_event_log_path(self._runtime_root, self._goal_id) + ) + self._source = source + + @property + def goal_id(self) -> str: + return self._goal_id + + def observe( + self, + *, + tick_id: str | None = None, + status: str | None = None, + details: Mapping[str, Any] | None = None, + recorded_at: str | None = None, + use_event_source: bool | None = None, + ) -> dict[str, Any]: + """Record one heartbeat observation event fact.""" + return record_heartbeat_observation( + runtime_root=self._runtime_root, + goal_id=self._goal_id, + agent_id=self._agent_id, + event_log_path=self._event_log_path, + source=self._source, + tick_id=tick_id, + status=status, + details=details, + recorded_at=recorded_at, + use_event_source=use_event_source, + ) + + +__all__ = [ + "HEARTBEAT_EVENT_SOURCE_ENV", + "HEARTBEAT_OBSERVED_EVENT_KIND", + "HEARTBEAT_EVENT_SCHEMA_VERSION", + "DEFAULT_HEARTBEAT_SOURCE", + "heartbeat_event_source_enabled", + "compute_observation_fingerprint", + "build_heartbeat_observation_event", + "record_heartbeat_observation", + "HeartbeatEventSource", +] diff --git a/loopx/control_plane/heartbeat/rules.py b/loopx/control_plane/heartbeat/rules.py index a3f92e679..f5ff3fd9a 100644 --- a/loopx/control_plane/heartbeat/rules.py +++ b/loopx/control_plane/heartbeat/rules.py @@ -41,6 +41,20 @@ "Normal turns use CLI `interaction_contract`; use `loopx-project` for " "lifecycle/registry and `loopx-self-repair` for runtime/projection drift." ) +EVENT_DRIVEN_EXECUTION_RULE = ( + "Prefer the event-driven scheduler for advancing agent todos: run " + "`codex-cli-local-scheduler-dispatch --goal-id --project . " + "--agent-id --event-driven [--completed-todo-id ] " + "--acceptance-criteria = --evidence =grep==` " + "instead of hand-editing files and calling `todo complete` yourself. The " + "dispatcher recomputes READY successors from todo events, enqueues them, " + "claims for a worker, and — when the queue is empty and acceptance evidence " + "satisfies — atomically emits goal_closure_ready + goal_closed in one tick. " + "You only declare the plan (todos), provide acceptance criteria + evidence; " + "let the dispatcher drive execution and closure. Fall back to manual " + "`todo complete --no-follow-up` only when no advancement todo remains and " + "you must close a goal without new work." +) CODEX_NATIVE_GOAL_UNCHANGED_WAIT_RULE = """ Native Codex `/goal` owns its blocked state. At the matching diff --git a/loopx/control_plane/heartbeat/task_body.py b/loopx/control_plane/heartbeat/task_body.py index b280219cd..6d1f434b1 100644 --- a/loopx/control_plane/heartbeat/task_body.py +++ b/loopx/control_plane/heartbeat/task_body.py @@ -8,6 +8,7 @@ CODEX_NATIVE_GOAL_UNCHANGED_WAIT_RULE, DEFAULT_MATERIAL_QUEUE_RULE, DEFAULT_PERMISSION_RULE, + EVENT_DRIVEN_EXECUTION_RULE, HEARTBEAT_NOTIFICATION_RULE_SHORT, HEARTBEAT_VISION_WRITEBACK_RULE_SHORT, RUNTIME_CAPABILITY_PROJECTION_THIN_RULE, @@ -171,7 +172,7 @@ def render_heartbeat_task_body( 4. Choose one bounded, verifiable progress segment from that audit. It may be a coherent batch across related implementation, test, doc, and state-writeback files when the write scope is clear and validation is explicit; it should not - be forced into a tiny single-file step. + be forced into a tiny single-file step. {EVENT_DRIVEN_EXECUTION_RULE} 5. Do that segment only. Stay inside `goal_boundary` when present and keep public/private boundaries intact. Public-safe repo publication is not an operator gate by itself: for routine public project work, commit, push, and diff --git a/loopx/control_plane/new_architecture.py b/loopx/control_plane/new_architecture.py new file mode 100644 index 000000000..8e66cdbdb --- /dev/null +++ b/loopx/control_plane/new_architecture.py @@ -0,0 +1,34 @@ +"""Phase 5 new-architecture master switch. + +The Phase 5 control-plane features (unified ``policy_decision``, event-driven +dispatch, heartbeat-as-event-source, and the merged tick) are enabled by default +under a single master switch. Each individual feature can still be forced on or +off via its own environment variable; the priority is: + + explicit feature flag env value > master switch > off + +Concretely, an unset feature flag inherits the master switch value, so setting +``LOOPX_NEW_ARCHITECTURE=1`` turns everything on while ``=0`` turns it all off +(unless a specific feature flag is set explicitly). +""" + +from __future__ import annotations + +import os + +MASTER_ENV = "LOOPX_NEW_ARCHITECTURE" + +_TRUTHY = {"1", "true", "yes", "on"} + + +def master_switch_enabled() -> bool: + """Whether the new architecture is enabled globally. + + The new architecture is ON by default: an unset ``LOOPX_NEW_ARCHITECTURE`` + enables it. Set ``LOOPX_NEW_ARCHITECTURE=0`` (or ``false``/``no``/``off``) + to disable it globally; individual feature flags can still override. + """ + value = os.environ.get(MASTER_ENV, "").strip().lower() + if not value: + return True + return value in _TRUTHY diff --git a/loopx/control_plane/plan/capabilities-bridge-migration-notes.md b/loopx/control_plane/plan/capabilities-bridge-migration-notes.md new file mode 100644 index 000000000..4623588ff --- /dev/null +++ b/loopx/control_plane/plan/capabilities-bridge-migration-notes.md @@ -0,0 +1,57 @@ +# Capability-Pack Bridge — Migration Notes + +> 状态：P1/P2/P3 已实现并测试通过（`tests/control_plane/test_capabilities_bridge.py`）。 +> 本文件记录桥接引入的**行为变更**与**边界**，供迁移与运维参考。 + +## 背景 + +老框架（`loopx/capabilities`）通过三条独立路径挂载能力包： + +1. `cli.py` 静态 import 各包的 `register_*_commands`（能力包 = CLI 子命令）； +2. `catalog.py` 的 `BUILTIN_CAPABILITIES` 元数据 + `CapabilityRegistry`（只读发现）； +3. 能力包函数被硬编码 import 进 `quota.py` / `configure_goal.py` / `heartbeat_prequota.py` / `lark_inbox.py` 等 90+ 处钩子。 + +新框架的 capability 标签（`required_capabilities` / `target_capabilities`）与老框架能力包目录原本完全解耦。 +`capabilities_bridge.py` 将两者桥接起来，分三阶段： + +- **P1**：token 归一化（`capability_token`），任务 `capability_binding_ref` 参与 `eligible` 判定； +- **P2**：`discover_cli_registrars` 反射式注册替代 12 处静态 import； +- **P3**：`CapabilityEventHub` 事件订阅底座 + `CapabilityHookRegistry` 钩子注册表（heartbeat pre-quota 已接入）。 + +## 行为变更（重要） + +1. **带 binding 的任务要求 worker 声明 pack token**。 + 旧行为：任务只要带 `capability_binding_ref` 且无显式 `required_capabilities`，任何 worker 都可 claim。 + 新行为：`eligible()` 在任务带 binding 时走 `eligible_bridged`，把绑定包 token 并入 required 集合， + worker 必须声明该 pack token 才能 claim。 + +2. **pack 未 ready 时 fail-closed**。 + `event_driven_dispatch.claim_next_task` 现已向 `claim_next_eligible_task` 补传 + `build_capability_registry()`，因此「任务绑定一个 registry 中未知/未 ready 的 pack」时不可 claim。 + 内置包（`loopx-core` provider）默认全部 `ready: True`，不受影响； + 只有未安装/未启用的扩展能力包会触发 fail-closed——这正是期望行为。 + +3. **CLI 注册顺序由 registry 决定**。 + `build_parser()` 的命令注册改为 `register_all_capability_commands` 驱动， + 顺序来自 `CapabilityRegistry.records()`，而非源文件 import 顺序。命令名与 handler 分发未变。 + +## P3 边界 + +- 已完成：`CapabilityEventHub` 事件订阅/发布底座（错误与正常结果分离返回）；`CapabilityHookRegistry` + 进程级钩子注册；`heartbeat_prequota` 已改为通过 hook registry 收集钩子（保留 + `acknowledged_pr_reviews` 兼容键 + 新增 `hooks` 键）。 +- 未完成：`quota.py` / `configure_goal.py` / `lark_inbox.py` 等其余 90+ 处硬编码钩子 + **仍是静态 import**，尚未迁移到事件订阅。这是刻意的阶段边界，不是已完成状态。 + +## Hook 契约 + +`register_pre_quota_hook(hook, *, source="")` 的 hook 签名： +`hook(*, registry_path, runtime_root_arg, goal_id, agent_id, fetch_timeout_seconds=10) -> dict`。 +签名不符 / 抛异常 / 返回非 dict 均视为失败（`degraded`），不影响其他 hook。 +结果以 hook `__name__` 为 key 并入 `checks.hooks`。 + +## 迁移注意事项 + +- 存量队列中带 `capability_binding_ref` 的 pending 任务，在 worker 未声明对应 pack token 前不可 claim； + 上线前需核对在途 binding 的 pack token 是否已被 worker 声明。 +- 扩展能力包若未在本机 `ready`，其绑定任务会被 fail-closed；确认扩展包的安装/启用状态。 diff --git a/loopx/control_plane/policy/__init__.py b/loopx/control_plane/policy/__init__.py new file mode 100644 index 000000000..075c23b47 --- /dev/null +++ b/loopx/control_plane/policy/__init__.py @@ -0,0 +1,48 @@ +"""Unified policy decision layer for the LoopX control plane. + +RFC: Event-Driven Control Plane and Unified Policy Decision Architecture. + +This package is a facade over the existing decision modules +(``quota/should_run``, ``agents/capability_gate``, ``agents/agent_scope``, +``scheduler/execution_context``). It composes and normalizes their results +into a single stable ``Decision`` contract without reimplementing any domain +rules and without touching existing execution paths. +""" + +from __future__ import annotations + +from .decision import ( + CAPABILITY_ACTION_MAP, + DECISION_MAP, + Decision, + DecisionOutcome, + combine_decisions, + normalize_capability_action, + normalize_quota_decision, + normalize_scheduler_resolution, +) +from .decision_events import ( + POLICY_DECISION_EVENT_KIND, + PolicyDecisionRecorder, + compute_decision_fingerprint, + policy_decision_events, + record_policy_decision, +) +from .engine import PolicyEngine, decide, policy_engine + +__all__ = [ + "CAPABILITY_ACTION_MAP", + "DECISION_MAP", + "POLICY_DECISION_EVENT_KIND", + "PolicyDecisionRecorder", + "PolicyEngine", + "combine_decisions", + "compute_decision_fingerprint", + "decide", + "normalize_capability_action", + "normalize_quota_decision", + "normalize_scheduler_resolution", + "policy_decision_events", + "policy_engine", + "record_policy_decision", +] diff --git a/loopx/control_plane/policy/decision.py b/loopx/control_plane/policy/decision.py new file mode 100644 index 000000000..16ebb8e2b --- /dev/null +++ b/loopx/control_plane/policy/decision.py @@ -0,0 +1,253 @@ +"""Normalized policy decision contract. + +RFC C1 (Unified Policy Decision Contract): the three outcomes are normalized +control semantics; the reason/source fields preserve domain-specific +information. This avoids collapsing ``backoff``, ``recovery``, +``repair_bridge``, and ``ask_owner`` into indistinguishable states. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Literal + +from ..scheduler.execution_context import SchedulerExecutionContextResolution + +DecisionOutcome = Literal["run", "wait", "deny"] + +# Rich action vocabulary (``plan/new_plan.md`` §5, P1): PolicyEngine returns an +# actionable verb rather than a bare should-run boolean. Each action is backed +# by a normalized outcome so existing consumers keep working unchanged. +DecisionAction = Literal[ + "ALLOW", "DENY", "DEFER", "RETRY", "BLOCK", "CANCEL", "ESCALATE", +] + +# action -> (normalized outcome, default reason suffix) +DECISION_ACTION_MAP: dict[str, tuple[DecisionOutcome, str]] = { + "ALLOW": ("run", "allow"), + "DENY": ("deny", "deny"), + "DEFER": ("wait", "defer"), + "RETRY": ("wait", "retry"), + "BLOCK": ("wait", "block"), + "CANCEL": ("deny", "cancel"), + "ESCALATE": ("wait", "escalate"), +} + + +def default_action_for_outcome(outcome: DecisionOutcome) -> DecisionAction: + """Map a normalized outcome to a canonical rich action (backward default).""" + if outcome == "run": + return "ALLOW" + if outcome == "deny": + return "DENY" + return "DEFER" + + +@dataclass(frozen=True) +class Decision: + """Normalized control decision. + + ``outcome`` is one of three normalized control semantics: + + * ``run`` — the Task may execute now; + * ``wait`` — the Task may execute later (backoff, recovery, repair, gate); + * ``deny`` — the Task must not execute. + + ``reason`` preserves the domain-specific explanation, while ``source`` + records which policy layer produced the decision (``quota``, ``capability``, + ``scope``, or ``scheduler``). ``detail`` may carry additional structured + context. ``retry_*`` fields carry optional explicit retry metadata. + + ``action`` is the richer actionable verb from the + ``ALLOW / DENY / DEFER / RETRY / BLOCK / CANCEL / ESCALATE`` vocabulary + (``plan/new_plan.md`` §5). When omitted it is derived from ``outcome`` so + the rich vocabulary is fully backward compatible. Optional scheduler-facing + hints (``max_attempts``, ``priority``, ``required_capability``, + ``resource_class``) let the policy layer steer the Task Queue. + """ + + outcome: DecisionOutcome + reason: str + source: str + detail: dict[str, Any] = field(default_factory=dict) + retry_at: str | None = None + retry_after_seconds: int | None = None + manual_approval_required: bool = False + action: str | None = None + max_attempts: int | None = None + priority: int | None = None + required_capability: str | None = None + resource_class: str | None = None + + @property + def rich_action(self) -> str: + """Return the rich action (derived from outcome when not set).""" + if self.action: + return self.action + return default_action_for_outcome(self.outcome) + + def to_dict(self) -> dict[str, Any]: + payload: dict[str, Any] = { + "outcome": self.outcome, + "reason": self.reason, + "source": self.source, + } + # Only serialize an explicit action so round-tripping a legacy Decision + # (action=None) stays lossless; derived actions are available via + # ``rich_action`` for new consumers. + if self.action: + payload["action"] = self.action + if self.detail: + payload["detail"] = dict(self.detail) + if self.max_attempts is not None: + payload["max_attempts"] = self.max_attempts + if self.priority is not None: + payload["priority"] = self.priority + if self.required_capability: + payload["required_capability"] = self.required_capability + if self.resource_class: + payload["resource_class"] = self.resource_class + if self.retry_at: + payload["retry_at"] = self.retry_at + if self.retry_after_seconds is not None: + payload["retry_after_seconds"] = self.retry_after_seconds + if self.manual_approval_required: + payload["manual_approval_required"] = True + return payload + + @classmethod + def from_dict(cls, value: Any) -> "Decision": + if isinstance(value, cls): + return value + payload = value if isinstance(value, dict) else {} + raw_outcome = str(payload.get("outcome") or "deny") + outcome: DecisionOutcome + if raw_outcome in {"run", "wait", "deny"}: + outcome = raw_outcome # type: ignore[assignment] + else: + outcome = "deny" + return cls( + outcome=outcome, + reason=str(payload.get("reason") or "unknown"), + source=str(payload.get("source") or "unknown"), + detail=dict(payload.get("detail") or {}), + retry_at=payload.get("retry_at"), + retry_after_seconds=payload.get("retry_after_seconds"), + manual_approval_required=payload.get("manual_approval_required") is True, + action=str(payload.get("action") or "") or None, + max_attempts=payload.get("max_attempts"), + priority=payload.get("priority"), + required_capability=str(payload.get("required_capability") or "") or None, + resource_class=str(payload.get("resource_class") or "") or None, + ) + + def __bool__(self) -> bool: + return self.outcome == "run" + + +# --------------------------------------------------------------------------- +# Quota decision normalization +# +# ``quota/should_run.py`` exposes a ``decision`` field with values such as: +# run | observe | safe_bypass_recovery | self_repair | repair_bridge +# | workspace_guard | automation_prompt_upgrade | skip +# | agent_scope_exhausted | agent_scope_wait | reassignment_required +# | successor_replan_required | autonomous_replan_required +# +# RFC requires an exhaustive, testable mapping from every existing decision +# value to the normalized (outcome, reason) contract. +# --------------------------------------------------------------------------- + +DECISION_MAP: dict[str, tuple[DecisionOutcome, str]] = { + "run": ("run", "normal_delivery"), + "observe": ("run", "external_evidence_observe"), + "safe_bypass_recovery": ("run", "safe_bypass_recovery"), + "recovery": ("run", "recovery"), + "self_repair": ("run", "self_repair"), + "repair_bridge": ("wait", "capability_repair_bridge"), + "workspace_guard": ("wait", "workspace_guard"), + "automation_prompt_upgrade": ("wait", "automation_prompt_upgrade"), + "autonomous_replan_required": ("run", "autonomous_replan_required"), + "agent_scope_exhausted": ("wait", "agent_scope_exhausted"), + "agent_scope_wait": ("wait", "agent_scope_wait"), + "reassignment_required": ("wait", "reassignment_required"), + "successor_replan_required": ("wait", "successor_replan_required"), + "skip": ("deny", "skip"), +} + +# --------------------------------------------------------------------------- +# Capability gate normalization +# +# ``agents/capability_gate.py`` exposes an ``action`` field with values: +# run | repair_bridge | ask_owner | skip | denied +# --------------------------------------------------------------------------- + +CAPABILITY_ACTION_MAP: dict[str, tuple[DecisionOutcome, str]] = { + "run": ("run", "capability_ok"), + "repair_bridge": ("wait", "capability_repair_bridge"), + "ask_owner": ("wait", "ask_owner"), + "deny": ("deny", "capability_denied"), + "denied": ("deny", "capability_denied"), + "skip": ("deny", "capability_skip"), +} + +# Strictness ordering for combining decisions: deny > wait > run. +_OUTCOME_RANK: dict[DecisionOutcome, int] = {"run": 0, "wait": 1, "deny": 2} + + +def _rank(outcome: DecisionOutcome) -> int: + return _OUTCOME_RANK.get(outcome, 1) + + +def combine_decisions(primary: Decision, secondary: Decision) -> Decision: + """Combine two decisions, keeping the most restrictive outcome. + + When outcomes tie, the primary decision wins so callers can control + precedence (for example, quota before capability). + """ + if _rank(secondary.outcome) > _rank(primary.outcome): + return secondary + return primary + + +def normalize_quota_decision(decision_value: Any, *, extra_detail: dict[str, Any] | None = None) -> Decision: + """Normalize a ``quota/should_run`` decision value into a ``Decision``.""" + value = str(decision_value or "").strip() + if not value: + value = "skip" + outcome, reason = DECISION_MAP.get(value, ("deny", f"unknown_quota_decision:{value}")) + detail: dict[str, Any] = {"quota_decision": value} + if extra_detail: + detail.update(extra_detail) + return Decision(outcome=outcome, reason=reason, source="quota", detail=detail) + + +def normalize_capability_action(action_value: Any, *, extra_detail: dict[str, Any] | None = None) -> Decision: + """Normalize a capability gate ``action`` value into a ``Decision``.""" + value = str(action_value or "").strip() + if not value: + value = "skip" + outcome, reason = CAPABILITY_ACTION_MAP.get(value, ("deny", f"unknown_capability_action:{value}")) + detail: dict[str, Any] = {"capability_action": value} + if extra_detail: + detail.update(extra_detail) + return Decision(outcome=outcome, reason=reason, source="capability", detail=detail) + + +def normalize_scheduler_resolution( + resolution: SchedulerExecutionContextResolution, + *, + extra_detail: dict[str, Any] | None = None, +) -> Decision: + """Normalize a scheduler execution-context resolution into a ``Decision``.""" + if resolution.ok: + return Decision(outcome="run", reason="scheduler_context_ok", source="scheduler") + detail: dict[str, Any] = {"errors": list(resolution.errors)} + if extra_detail: + detail.update(extra_detail) + return Decision( + outcome="deny", + reason="invalid_scheduler_execution_context", + source="scheduler", + detail=detail, + ) diff --git a/loopx/control_plane/policy/decision_events.py b/loopx/control_plane/policy/decision_events.py new file mode 100644 index 000000000..a7ec8220a --- /dev/null +++ b/loopx/control_plane/policy/decision_events.py @@ -0,0 +1,258 @@ +"""Opt-in policy decision recording. + +RFC C2 (Policy Decision Events): decisions must be auditable through persisted +events without turning the Policy Engine into a persistence layer. + +Design rules: + +* ``PolicyEngine`` itself stays free of persistence side effects. Callers that + want auditability wrap ``decide()`` with ``record_policy_decision()``. +* Recording is opt-in (default off). +* Events are written through the existing public-safe ``rollout_event_log`` + append-only idempotent writer (``event_kind="policy_decision"``), so no new + storage system is introduced. +* Deduplication (RFC §8.4): repeated identical decisions must not create + unbounded event growth. Two mechanisms are supported: + ``transition_only=True`` records only outcome/source transitions, while + ``transition_only=False`` relies on deterministic decision fingerprints + written into the event payload. +* Sensitive task contents and credentials are never persisted (the rollout + event boundary already strips them). +""" + +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +from typing import Any, Mapping, Sequence + +from ...rollout_event_log import ( + DEFAULT_ROLLOUT_EVENT_LOG_NAME, + append_rollout_event_once, + build_rollout_event, +) +from ..runtime.time import now_utc_iso +from .decision import Decision + +POLICY_DECISION_EVENT_KIND = "policy_decision" + +#: Identity fields used for idempotent appends. ``decision_fingerprint`` is +#: deterministic for the same (goal, todo, agent, outcome, reason, source). +_DECISION_IDENTITY_FIELDS: tuple[str, ...] = ( + "goal_id", + "todo_id", + "agent_id", + "decision_fingerprint", +) + +_DEFAULT_STATE_DIR_NAME = "policy-decision-state" + + +def compute_decision_fingerprint( + decision: Decision, + *, + goal_id: str, + todo_id: str | None = None, + agent_id: str | None = None, + run_id: str | None = None, +) -> str: + """Deterministic fingerprint over the stable decision identity. + + The fingerprint intentionally excludes timestamps and free-form detail so + that repeated polling of an unchanged state collapses to a single event. + """ + stable: dict[str, Any] = { + "goal_id": str(goal_id or "").strip(), + "todo_id": str(todo_id or "").strip(), + "agent_id": str(agent_id or "").strip(), + "run_id": str(run_id or "").strip(), + "outcome": decision.outcome, + "reason": decision.reason, + "source": decision.source, + } + encoded = json.dumps(stable, sort_keys=True, ensure_ascii=True).encode("utf-8") + return hashlib.sha256(encoded).hexdigest()[:16] + + +class PolicyDecisionRecorder: + """Wraps a rollout event log with transition-only deduplication state. + + ``transition_only=True`` (default) persists a compact state file recording + the last seen fingerprint per (goal, todo, agent). A repeated identical + decision is suppressed; a new decision (outcome/reason/source change) is + recorded. This implements RFC §8.4 transition-only recording. + """ + + def __init__( + self, + *, + log_path: Path | None = None, + state_dir: Path | None = None, + transition_only: bool = True, + ) -> None: + if log_path is None: + log_path = Path(".") / DEFAULT_ROLLOUT_EVENT_LOG_NAME + if state_dir is None: + state_dir = Path(".") / _DEFAULT_STATE_DIR_NAME + self._log_path = Path(log_path) + self._state_dir = Path(state_dir) + self._transition_only = bool(transition_only) + self._state_path = self._state_dir / "transition-state.json" + self._cache: dict[str, str] | None = None + + # -- public API --------------------------------------------------------- + + @property + def log_path(self) -> Path: + return self._log_path + + @property + def transition_only(self) -> bool: + return self._transition_only + + def record( + self, + decision: Decision, + *, + goal_id: str, + todo_id: str | None = None, + agent_id: str | None = None, + run_id: str | None = None, + extra_details: Mapping[str, Any] | None = None, + recorded_at: str | None = None, + ) -> tuple[dict[str, Any], bool]: + """Record one decision; returns (event, was_new). + + When ``transition_only`` is enabled, a decision identical to the last + recorded one for the same (goal, todo, agent) is suppressed. + """ + fingerprint = compute_decision_fingerprint( + decision, + goal_id=goal_id, + todo_id=todo_id, + agent_id=agent_id, + run_id=run_id, + ) + if self._transition_only and self._is_repeat(goal_id, todo_id, agent_id, fingerprint): + return {}, False + + event = build_rollout_event( + goal_id=goal_id, + event_kind=POLICY_DECISION_EVENT_KIND, + agent_id=agent_id, + todo_id=todo_id, + run_id=run_id, + status=decision.outcome, + classification=decision.reason, + summary=f"policy decision: {decision.source}:{decision.reason}", + details={ + "decision_outcome": decision.outcome, + "decision_reason": decision.reason, + "decision_source": decision.source, + "decision_fingerprint_detail": fingerprint, + **(extra_details or {}), + }, + recorded_at=recorded_at or now_utc_iso(), + ) + # The fingerprint is not part of build_rollout_event's schema; attach it + # as a stable top-level field for idempotency identity matching. + event["decision_fingerprint"] = fingerprint + # Identity fields must be non-empty in the payload; include only the + # identifiers actually supplied so optional goal/todo/agent all work. + identity_fields = [ + field + for field in _DECISION_IDENTITY_FIELDS + if field in ("goal_id", "decision_fingerprint") or event.get(field) + ] + appended, was_new = append_rollout_event_once( + self._log_path, + event, + identity_fields=identity_fields, + ) + if was_new: + self._remember(goal_id, todo_id, agent_id, fingerprint) + return appended, was_new + + def replay_identity(self, event: Mapping[str, Any]) -> tuple[str, str, str, str]: + """Return the stable identity of a recorded decision event.""" + return ( + str(event.get("goal_id") or ""), + str(event.get("todo_id") or ""), + str(event.get("agent_id") or ""), + str(event.get("decision_fingerprint") or ""), + ) + + # -- transition state helpers ------------------------------------------ + + def _state(self) -> dict[str, str]: + if self._cache is None: + try: + self._cache = json.loads(self._state_path.read_text(encoding="utf-8")) + except (OSError, ValueError): + self._cache = {} + return self._cache + + def _state_key(self, goal_id: str, todo_id: str | None, agent_id: str | None) -> str: + return "|".join( + [ + str(goal_id or "").strip(), + str(todo_id or "").strip(), + str(agent_id or "").strip(), + ] + ) + + def _is_repeat(self, goal_id: str, todo_id: str | None, agent_id: str | None, fingerprint: str) -> bool: + if not self._transition_only: + return False + return self._state().get(self._state_key(goal_id, todo_id, agent_id)) == fingerprint + + def _remember(self, goal_id: str, todo_id: str | None, agent_id: str | None, fingerprint: str) -> None: + if not self._transition_only: + return + state = self._state() + state[self._state_key(goal_id, todo_id, agent_id)] = fingerprint + self._state_dir.mkdir(parents=True, exist_ok=True) + tmp_path = self._state_path.with_suffix(".json.tmp") + tmp_path.write_text(json.dumps(state, sort_keys=True), encoding="utf-8") + os.replace(tmp_path, self._state_path) + + +def record_policy_decision( + decision: Decision, + *, + goal_id: str, + todo_id: str | None = None, + agent_id: str | None = None, + run_id: str | None = None, + log_path: Path | None = None, + state_dir: Path | None = None, + transition_only: bool = True, + extra_details: Mapping[str, Any] | None = None, + recorded_at: str | None = None, +) -> tuple[dict[str, Any], bool]: + """One-shot convenience wrapper around :class:`PolicyDecisionRecorder`.""" + recorder = PolicyDecisionRecorder( + log_path=log_path, + state_dir=state_dir, + transition_only=transition_only, + ) + return recorder.record( + decision, + goal_id=goal_id, + todo_id=todo_id, + agent_id=agent_id, + run_id=run_id, + extra_details=extra_details, + recorded_at=recorded_at, + ) + + +def policy_decision_events(events: Sequence[Mapping[str, Any]]) -> list[dict[str, Any]]: + """Read projection: filter a rollout event list down to policy decisions.""" + return [ + dict(event) + for event in events + if str(event.get("event_kind") or "") == POLICY_DECISION_EVENT_KIND + ] diff --git a/loopx/control_plane/policy/engine.py b/loopx/control_plane/policy/engine.py new file mode 100644 index 000000000..3e6a1f529 --- /dev/null +++ b/loopx/control_plane/policy/engine.py @@ -0,0 +1,155 @@ +"""Policy Engine: composes existing decision modules into a unified Decision. + +RFC C1 (Unified Policy Decision Contract) + §7.3 Implementation Rule: + +* ``PolicyEngine`` must not duplicate existing domain rules; +* it composes existing pure functions (quota, capability, scope, + execution-context validation); +* it owns composition and normalization, not the underlying business rules; +* it remains free of persistence side effects (decision recording is an + opt-in concern handled by the caller / ``policy/decision_events.py``). +""" + +from __future__ import annotations + +from typing import Any, Callable, Mapping + +from ..agents.capability_gate import build_capability_gate +from ..quota.should_run import build_quota_should_run +from ..scheduler.execution_context import ( + SchedulerExecutionContextResolution, + resolve_scheduler_execution_context, +) +from .decision import ( + Decision, + combine_decisions, + normalize_capability_action, + normalize_quota_decision, + normalize_scheduler_resolution, +) + + +class PolicyEngine: + """Unified decision facade over quota / capability / scope / scheduler.""" + + def __init__(self) -> None: + self._supports_capability_gate = True + + def decide_live( + self, + *, + status_payload: dict[str, Any], + goal_id: str, + agent_id: str | None, + available_capabilities: list[str] | None, + include_scheduler_detail: bool, + codex_app_current_rrule: str | None, + registry_path: Any = None, + runtime_root: Any = None, + host_observation_resolver: Callable[..., Mapping[str, Any]] | None = None, + scheduler_execution_context: ( + Mapping[str, Any] | SchedulerExecutionContextResolution | None + ) = None, + operator_inbox_urgency_projector: Callable[..., dict[str, Any]] | None = None, + capability_agent_todo_summary: dict[str, Any] | None = None, + capability_agent_identity: dict[str, Any] | None = None, + ) -> Decision: + """Phase 5 integration adapter (opt-in; does not change existing paths). + + Mirrors ``quota/live_decision.build_live_quota_should_run_decision`` + signature so a live caller can switch to the unified contract without + restructuring its inputs. Returns the normalized ``Decision``. + """ + return self.decide( + status_payload=status_payload, + goal_id=goal_id, + agent_id=agent_id, + available_capabilities=available_capabilities, + include_scheduler_detail=include_scheduler_detail, + codex_app_current_rrule=codex_app_current_rrule, + scheduler_execution_context=scheduler_execution_context, + operator_inbox_urgency_projector=operator_inbox_urgency_projector, + capability_agent_todo_summary=capability_agent_todo_summary, + capability_agent_identity=capability_agent_identity, + ) + + def decide( + self, + *, + status_payload: dict[str, Any], + goal_id: str, + agent_id: str | None = None, + available_capabilities: Any = None, + scheduler_execution_context: ( + Mapping[str, Any] | SchedulerExecutionContextResolution | None + ) = None, + codex_app_current_rrule: Any = None, + operator_inbox_urgency_projector: Callable[..., dict[str, Any]] | None = None, + include_scheduler_detail: bool = False, + capability_agent_todo_summary: dict[str, Any] | None = None, + capability_agent_identity: dict[str, Any] | None = None, + ) -> Decision: + """Return the unified decision for a single Goal / Task evaluation. + + Composition order (most restrictive wins on tie): + + 1. scheduler execution-context validation; + 2. quota should_run (already embeds agent-scope frontier semantics); + 3. optional capability gate when ``capability_agent_todo_summary`` + is supplied. + """ + scheduler_resolution = resolve_scheduler_execution_context( + scheduler_execution_context + ) + scheduler_decision = normalize_scheduler_resolution(scheduler_resolution) + if not scheduler_resolution.ok: + return scheduler_decision + + quota_payload = build_quota_should_run( + status_payload, + goal_id=goal_id, + agent_id=agent_id, + available_capabilities=available_capabilities, + include_scheduler_detail=include_scheduler_detail, + codex_app_current_rrule=codex_app_current_rrule, + scheduler_execution_context=scheduler_resolution, + operator_inbox_urgency_projector=operator_inbox_urgency_projector, + ) + quota_decision = normalize_quota_decision( + quota_payload.get("decision"), + extra_detail={ + "effective_action": quota_payload.get("effective_action"), + "state": quota_payload.get("state"), + "should_run": quota_payload.get("should_run"), + }, + ) + decision = quota_decision + + if capability_agent_todo_summary is not None: + capability_gate = build_capability_gate( + capability_agent_todo_summary, + available_capabilities=list(available_capabilities or []), + agent_identity=capability_agent_identity, + ) + if capability_gate is not None: + capability_decision = normalize_capability_action( + capability_gate.get("action"), + extra_detail={"gate": capability_gate.get("gate")}, + ) + decision = combine_decisions(decision, capability_decision) + + return decision + + +# Shared engine instance for stateless callers. +policy_engine = PolicyEngine() + + +def decide( + *, + status_payload: dict[str, Any], + goal_id: str, + **kwargs: Any, +) -> Decision: + """Module-level convenience for ``PolicyEngine.decide``.""" + return policy_engine.decide(status_payload=status_payload, goal_id=goal_id, **kwargs) diff --git a/loopx/control_plane/quota/cost_projection.py b/loopx/control_plane/quota/cost_projection.py new file mode 100644 index 000000000..05297a29f --- /dev/null +++ b/loopx/control_plane/quota/cost_projection.py @@ -0,0 +1,282 @@ +"""Read-only usage / cost projection over existing spend events. + +RFC C3 (Usage and Cost Projection): + +* Raw spend facts already exist (``quota/slot_accounting``); callers must not + aggregate them manually. +* This module is a pure read projection: it never mutates accounting state. +* ``usage_units`` is deliberately distinguished from ``monetary_cost`` + (RFC §9.3): ``slots`` are usage units, not a currency. Monetary cost can + later be derived if provider pricing is introduced. +* First version aggregates directly from existing events (RFC §9.4); the same + contract may later be materialized without changing callers. +""" + +from __future__ import annotations + +import json +from collections import defaultdict +from pathlib import Path +from typing import Any, Iterable, Mapping, Sequence + +from ..runtime.time import now_utc_iso + +#: Classification marker produced by ``quota/slot_accounting``. +QUOTA_SLOT_SPENT_CLASSIFICATION = "quota_slot_spent" + +#: Rollout event kind that carries a quota spend receipt. +QUOTA_SPEND_EVENT_KIND = "quota_spend" + +#: Optional monetary conversion hook. When set, ``cost_projection`` exposes +#: ``monetary_cost`` in addition to ``usage_units``. +PRICING_PER_USAGE_UNIT: float | None = None + + +# --------------------------------------------------------------------------- +# Spend fact extraction (normalization over heterogeneous event shapes) +# --------------------------------------------------------------------------- + + +def _as_mapping(value: Any) -> Mapping[str, Any]: + return value if isinstance(value, Mapping) else {} + + +def _usage_units_of(event: Mapping[str, Any]) -> int: + """Extract the usage-unit count from any supported spend shape.""" + quota_event = _as_mapping(event.get("quota_event")) + if quota_event: + slots = quota_event.get("slots") + if slots is not None: + return max(0, _int(slots)) + # Rollout event shape: details carry flat scalar fields. + details = _as_mapping(event.get("details")) + if details: + slots = details.get("slots") + if slots is not None: + return max(0, _int(slots)) + slots = details.get("usage_units") + if slots is not None: + return max(0, _int(slots)) + return 0 + + +def _int(value: Any) -> int: + try: + return int(value) + except (TypeError, ValueError): + return 0 + + +def _text(value: Any) -> str: + return str(value or "").strip() + + +def _is_spend_event(event: Mapping[str, Any]) -> bool: + if _as_mapping(event.get("quota_event")): + return True + if _text(event.get("classification")) == QUOTA_SLOT_SPENT_CLASSIFICATION: + return True + if _text(event.get("event_kind")) == QUOTA_SPEND_EVENT_KIND: + return True + details = _as_mapping(event.get("details")) + return _text(details.get("event_type")) == QUOTA_SLOT_SPENT_CLASSIFICATION + + +def spend_fact(event: Mapping[str, Any]) -> dict[str, Any] | None: + """Normalize one event into a minimal spend fact, or ``None`` if it is not + a spend event.""" + if not _is_spend_event(event): + return None + quota_event = _as_mapping(event.get("quota_event")) + details = _as_mapping(event.get("details")) + agent_id = _text(event.get("agent_id")) or _text(quota_event.get("agent_id")) + todo_id = _text(event.get("todo_id")) or _text(quota_event.get("todo_id")) + source = _text(quota_event.get("source")) or _text(details.get("source")) + generated_at = _text(event.get("generated_at")) or _text(event.get("recorded_at")) + return { + "goal_id": _text(event.get("goal_id")), + "todo_id": todo_id, + "agent_id": agent_id, + "usage_units": _usage_units_of(event), + "source": source, + "generated_at": generated_at, + "day": generated_at[:10] if len(generated_at) >= 10 else "", + } + + +def spend_facts(events: Iterable[Mapping[str, Any]]) -> list[dict[str, Any]]: + """Project a heterogeneous event stream down to normalized spend facts.""" + facts: list[dict[str, Any]] = [] + for event in events: + fact = spend_fact(event) + if fact is not None: + facts.append(fact) + return facts + + +# --------------------------------------------------------------------------- +# Aggregation +# --------------------------------------------------------------------------- + + +def _cost(usage_units: int) -> float | None: + if PRICING_PER_USAGE_UNIT is None: + return None + return usage_units * PRICING_PER_USAGE_UNIT + + +def goal_cost_summary( + goal_id: str, + events: Sequence[Mapping[str, Any]] | None = None, + *, + facts: Sequence[Mapping[str, Any]] | None = None, +) -> dict[str, Any]: + """Aggregate usage for one Goal: total + by agent / task / day. + + Accepts either raw ``events`` or already-normalized ``facts``. + """ + source_facts = facts if facts is not None else spend_facts(events or []) + by_agent: dict[str, int] = defaultdict(int) + by_task: dict[str, int] = defaultdict(int) + by_day: dict[str, int] = defaultdict(int) + by_source: dict[str, int] = defaultdict(int) + total = 0 + for fact in source_facts: + if _text(fact.get("goal_id")) != _text(goal_id): + continue + usage = max(0, _int(fact.get("usage_units"))) + total += usage + if _text(fact.get("agent_id")): + by_agent[_text(fact.get("agent_id"))] += usage + if _text(fact.get("todo_id")): + by_task[_text(fact.get("todo_id"))] += usage + if _text(fact.get("day")): + by_day[_text(fact.get("day"))] += usage + if _text(fact.get("source")): + by_source[_text(fact.get("source"))] += usage + monetary = _cost(total) + summary: dict[str, Any] = { + "goal_id": _text(goal_id), + "total_usage": total, + "by_agent": dict(sorted(by_agent.items(), key=lambda kv: (-kv[1], kv[0]))), + "by_task": dict(sorted(by_task.items(), key=lambda kv: (-kv[1], kv[0]))), + "by_day": dict(sorted(by_day.items())), + "by_source": dict(sorted(by_source.items(), key=lambda kv: (-kv[1], kv[0]))), + "usage_units": total, + } + if monetary is not None: + summary["monetary_cost"] = round(monetary, 6) + return summary + + +def task_cost( + goal_id: str, + todo_id: str, + events: Sequence[Mapping[str, Any]] | None = None, + *, + facts: Sequence[Mapping[str, Any]] | None = None, +) -> dict[str, Any]: + """Aggregate usage for one Task inside a Goal.""" + source_facts = facts if facts is not None else spend_facts(events or []) + by_agent: dict[str, int] = defaultdict(int) + total = 0 + for fact in source_facts: + if _text(fact.get("goal_id")) != _text(goal_id): + continue + if _text(fact.get("todo_id")) != _text(todo_id): + continue + usage = max(0, _int(fact.get("usage_units"))) + total += usage + if _text(fact.get("agent_id")): + by_agent[_text(fact.get("agent_id"))] += usage + monetary = _cost(total) + summary: dict[str, Any] = { + "goal_id": _text(goal_id), + "todo_id": _text(todo_id), + "total_usage": total, + "by_agent": dict(sorted(by_agent.items(), key=lambda kv: (-kv[1], kv[0]))), + "usage_units": total, + } + if monetary is not None: + summary["monetary_cost"] = round(monetary, 6) + return summary + + +# --------------------------------------------------------------------------- +# Event loading helpers (read-only) +# --------------------------------------------------------------------------- + + +def _load_run_index_records(runtime_root: Path, goal_id: str) -> list[dict[str, Any]]: + index_path = Path(runtime_root) / "goals" / goal_id / "runs" / "index.jsonl" + if not index_path.exists(): + return [] + records: list[dict[str, Any]] = [] + try: + lines = index_path.read_text(encoding="utf-8").splitlines() + except OSError: + return [] + for line in lines: + if not line.strip(): + continue + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(record, dict): + records.append(record) + return records + + +def load_goal_spend_facts(runtime_root: Path, goal_id: str) -> list[dict[str, Any]]: + """Load normalized spend facts for a Goal from its persisted run index.""" + return spend_facts(_load_run_index_records(runtime_root, goal_id)) + + +def load_all_spend_facts( + runtime_root: Path, + goal_ids: Iterable[str] | None = None, +) -> list[dict[str, Any]]: + """Load normalized spend facts across Goals under a runtime root.""" + root = Path(runtime_root) + goals_dir = root / "goals" + if goal_ids is not None: + goal_list = [str(g).strip() for g in goal_ids if str(g).strip()] + else: + goal_list = sorted( + (p.name for p in goals_dir.iterdir() if p.is_dir()) if goals_dir.exists() else [] + ) + facts: list[dict[str, Any]] = [] + for goal_id in goal_list: + facts.extend(load_goal_spend_facts(root, goal_id)) + return facts + + +def project_usage_summary( + runtime_root: Path, + goal_ids: Iterable[str] | None = None, + *, + as_of: str | None = None, +) -> dict[str, Any]: + """Read-only usage summary across Goals. + + ``as_of`` (ISO date) optionally limits the projection to spend facts + generated on or before that date. + """ + facts = load_all_spend_facts(runtime_root, goal_ids) + if as_of: + as_of_day = _text(as_of)[:10] + facts = [f for f in facts if _text(f.get("day")) <= as_of_day] + by_goal: dict[str, int] = defaultdict(int) + for fact in facts: + by_goal[_text(fact.get("goal_id"))] += max(0, _int(fact.get("usage_units"))) + summary: dict[str, Any] = { + "projected_at": now_utc_iso(), + "total_usage": sum(by_goal.values()), + "by_goal": dict(sorted(by_goal.items(), key=lambda kv: (-kv[1], kv[0]))), + "usage_units": sum(by_goal.values()), + } + monetary = _cost(sum(by_goal.values())) + if monetary is not None: + summary["monetary_cost"] = round(monetary, 6) + return summary diff --git a/loopx/control_plane/quota/live_decision.py b/loopx/control_plane/quota/live_decision.py index 95d0bc9ec..169d8e5ab 100644 --- a/loopx/control_plane/quota/live_decision.py +++ b/loopx/control_plane/quota/live_decision.py @@ -1,10 +1,15 @@ from __future__ import annotations +import os from collections.abc import Callable, Mapping from pathlib import Path from typing import Any from ...quota import build_quota_should_run +from ...rollout_event_log import rollout_event_log_path +from ..new_architecture import master_switch_enabled +from ..policy import PolicyEngine +from ..policy.decision_events import record_policy_decision from ..scheduler.execution_context import ( SchedulerExecutionContextResolution, resolve_scheduler_execution_context, @@ -14,6 +19,18 @@ HostObservationResolver = Callable[..., Mapping[str, Any]] +class PolicyIntegrationError(RuntimeError): + """Raised when the unified ``PolicyEngine`` decision diverges from the + live quota path during the opt-in pilot wiring (RFC §12 Phase 5).""" + + +def _env_flag(name: str, default: bool = False) -> bool: + value = os.environ.get(name, "").strip().lower() + if not value: + return default + return value in {"1", "true", "yes", "on"} + + def bind_scheduler_followup_cli_routes( payload: dict[str, Any], *, @@ -69,8 +86,18 @@ def build_live_quota_should_run_decision( route_source: str = "quota_cli_invocation", scheduler_execution_context: Mapping[str, Any] | SchedulerExecutionContextResolution | None = None, operator_inbox_urgency_projector: Callable[..., dict[str, Any]] | None = None, + use_policy_engine: bool | None = None, + record_policy_decisions: bool | None = None, ) -> dict[str, Any]: - """Build one live CLI decision while keeping host observation injectable.""" + """Build one live CLI decision while keeping host observation injectable. + + RFC §12 Phase 5 pilot wiring: when ``use_policy_engine`` is enabled (or the + ``LOOPX_USE_POLICY_ENGINE`` env flag is set), the decision is additionally + computed through the unified :class:`PolicyEngine`; the unified decision is + verified for consistency with the legacy quota payload and attached under + the ``policy_decision`` key. Optional decision audit events are recorded + when ``record_policy_decisions`` (or ``LOOPX_POLICY_DECISION_RECORD``) is on. + """ resolved_context = resolve_scheduler_execution_context(scheduler_execution_context) codex_app_applicable = ( @@ -103,4 +130,108 @@ def build_live_quota_should_run_decision( runtime_root=runtime_root, source=route_source, ) + if use_policy_engine is None: + use_policy_engine = _env_flag("LOOPX_USE_POLICY_ENGINE", default=master_switch_enabled()) + if record_policy_decisions is None: + record_policy_decisions = _env_flag( + "LOOPX_POLICY_DECISION_RECORD", default=master_switch_enabled() + ) + if use_policy_engine: + _attach_unified_policy_decision( + payload, + status_payload=status_payload, + goal_id=goal_id, + agent_id=agent_id, + available_capabilities=available_capabilities, + include_scheduler_detail=include_scheduler_detail, + observed_rrule=observed_rrule, + resolved_context=resolved_context, + operator_inbox_urgency_projector=operator_inbox_urgency_projector, + runtime_root=runtime_root, + record_policy_decisions=record_policy_decisions, + ) return payload + + +def _attach_unified_policy_decision( + payload: dict[str, Any], + *, + status_payload: dict[str, Any], + goal_id: str, + agent_id: str | None, + available_capabilities: list[str] | None, + include_scheduler_detail: bool, + observed_rrule: str, + resolved_context: SchedulerExecutionContextResolution, + operator_inbox_urgency_projector: Callable[..., dict[str, Any]] | None, + runtime_root: Path, + record_policy_decisions: bool, +) -> None: + """RFC §12 Phase 5 pilot wiring: compute the unified decision via + :class:`PolicyEngine`, verify it matches the legacy quota payload, attach it + as ``payload["policy_decision"]``, and optionally record an audit event.""" + unified = PolicyEngine().decide_live( + status_payload=status_payload, + goal_id=goal_id, + agent_id=agent_id, + available_capabilities=available_capabilities, + include_scheduler_detail=include_scheduler_detail, + codex_app_current_rrule=observed_rrule or None, + scheduler_execution_context=resolved_context, + operator_inbox_urgency_projector=operator_inbox_urgency_projector, + ) + _verify_policy_decision_consistency(payload, unified) + payload["policy_decision"] = unified.to_dict() + if record_policy_decisions: + record_policy_decision( + unified, + goal_id=goal_id, + agent_id=agent_id, + log_path=rollout_event_log_path(runtime_root, goal_id), + state_dir=runtime_root / "goals" / str(goal_id) / "policy-decision-state", + transition_only=True, + ) + + +def _verify_policy_decision_consistency( + payload: dict[str, Any], + unified: Any, +) -> None: + """Verify the unified decision agrees with the legacy quota payload. + + The legacy ``should_run`` flag and the normalized ``outcome`` answer + *different questions* and therefore legitimately diverge in one direction: + + * ``should_run=True`` means "some compute must execute now" — including the + *repair* lanes (``repair_bridge`` / ``workspace_guard`` / ``self_repair`` + set ``capability_repair_allowed`` / ``workspace_repair_allowed``, which + fold into ``should_run`` in ``decision_summary.resolve_quota_run_decision``). + * ``outcome="wait"`` means "this is not a *normal* run delivery" — a repair + bridge is intentionally normalized to ``wait`` (``DECISION_MAP``), because + the agent must first close the capability/workspace gap. + + So ``should_run=True`` with ``outcome="wait"`` is the *intended* repair + semantic, NOT a divergence. Two cases remain genuine bugs: + + * **permissive drift** (any source): ``outcome="run"`` while the quota says + ``should_run=False`` — PolicyEngine authorized normal delivery the quota + layer refused. + * **single-layer deny** (``source == "quota"``): ``outcome="deny"`` while the + quota says ``should_run=True`` — PolicyEngine consumed the same quota layer + yet produced a stricter deny, which cannot happen for the *same* input and + indicates a real mismatch. (A composed ``deny`` from a stricter outer layer + is still valid, as ``test_stricter_composed_deny_over_quota_run_is_accepted`` + asserts.) + """ + should_run = payload.get("should_run") is True + ok = True + if unified.outcome == "run" and not should_run: + ok = False # permissive drift + elif unified.source == "quota" and unified.outcome == "deny" and should_run: + ok = False # single-layer deny over a running quota + if not ok: + raise PolicyIntegrationError( + "PolicyEngine decision diverged from live quota payload: " + f"outcome={unified.outcome!r} source={unified.source!r} " + f"should_run={should_run!r} payload_decision={payload.get('decision')!r}" + ) diff --git a/loopx/control_plane/runtime/checkpoint.py b/loopx/control_plane/runtime/checkpoint.py new file mode 100644 index 000000000..fe45f65ad --- /dev/null +++ b/loopx/control_plane/runtime/checkpoint.py @@ -0,0 +1,214 @@ +"""Task-level checkpoint snapshots (recovery anchor, not a source of truth). + +RFC C4 (Task Checkpoint and Replay) + §10.3 Checkpoint Contract: + +* A checkpoint is a *recovery optimization*, never an authoritative state + store. Events remain the source of truth. +* It stores a reference (``goal_id`` / ``todo_id`` / ``run_id`` / + ``last_event_id``) plus a lightweight state snapshot with a content hash, + so replay can resume exactly where the checkpoint was taken. +* Schema-versioned so incompatible snapshots are never replayed blindly. +* Checkpoints are appended idempotently under the Goal's runtime directory + and coexist with existing migration / legacy mechanisms (RFC §10.5). +""" + +from __future__ import annotations + +import hashlib +import json +import os +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any, Mapping + +from .time import now_utc_iso + +CHECKPOINT_SCHEMA_VERSION = 1 + +#: Identity fields used for idempotent checkpoint appends. +CHECKPOINT_IDENTITY_FIELDS: tuple[str, ...] = ( + "goal_id", + "todo_id", + "last_event_id", + "state_hash", +) + + +@dataclass(frozen=True) +class Checkpoint: + """An immutable Task recovery anchor.""" + + checkpoint_id: str + goal_id: str + todo_id: str + run_id: str + last_event_id: str + schema_version: int = CHECKPOINT_SCHEMA_VERSION + state_snapshot: dict[str, Any] = field(default_factory=dict) + state_hash: str = "" + created_at: str = "" + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + @classmethod + def from_dict(cls, value: Any) -> "Checkpoint": + payload = value if isinstance(value, dict) else {} + return cls( + checkpoint_id=str(payload.get("checkpoint_id") or ""), + goal_id=str(payload.get("goal_id") or ""), + todo_id=str(payload.get("todo_id") or ""), + run_id=str(payload.get("run_id") or ""), + last_event_id=str(payload.get("last_event_id") or ""), + schema_version=int(payload.get("schema_version") or CHECKPOINT_SCHEMA_VERSION), + state_snapshot=dict(payload.get("state_snapshot") or {}), + state_hash=str(payload.get("state_hash") or ""), + created_at=str(payload.get("created_at") or ""), + ) + + +def compute_state_hash(state_snapshot: Mapping[str, Any]) -> str: + """Deterministic content hash of a state snapshot. + + Sort-keys + stable JSON so identical states always hash identically. + """ + encoded = json.dumps( + dict(state_snapshot), + sort_keys=True, + ensure_ascii=True, + separators=(",", ":"), + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def build_checkpoint( + *, + goal_id: str, + todo_id: str, + run_id: str, + last_event_id: str, + state_snapshot: Mapping[str, Any], + schema_version: int = CHECKPOINT_SCHEMA_VERSION, + created_at: str | None = None, +) -> Checkpoint: + """Build a checkpoint; ``state_hash`` is derived automatically.""" + snapshot = dict(state_snapshot) + state_hash = compute_state_hash(snapshot) + checkpoint_id = f"{goal_id}:{todo_id}:{last_event_id}:{state_hash[:12]}" + return Checkpoint( + checkpoint_id=checkpoint_id, + goal_id=str(goal_id), + todo_id=str(todo_id), + run_id=str(run_id), + last_event_id=str(last_event_id), + schema_version=schema_version, + state_snapshot=snapshot, + state_hash=state_hash, + created_at=created_at or now_utc_iso(), + ) + + +def _goal_checkpoint_dir(runtime_root: Path, goal_id: str) -> Path: + return Path(runtime_root) / "goals" / goal_id / "checkpoints" + + +def write_checkpoint( + runtime_root: Path, + checkpoint: Checkpoint, + *, + identity_fields: tuple[str, ...] = CHECKPOINT_IDENTITY_FIELDS, +) -> tuple[Checkpoint, bool]: + """Append a checkpoint idempotently; returns (checkpoint, was_new). + + A checkpoint with the same (goal, todo, last_event_id, state_hash) is + considered an exact duplicate and is not appended twice. + """ + goal_id = str(checkpoint.goal_id or "").strip() + todo_id = str(checkpoint.todo_id or "").strip() + if not goal_id or not todo_id: + raise ValueError("checkpoint requires goal_id and todo_id") + checkpoint_dir = _goal_checkpoint_dir(runtime_root, goal_id) + checkpoint_dir.mkdir(parents=True, exist_ok=True) + checkpoint_path = checkpoint_dir / f"{todo_id}.jsonl" + payload = checkpoint.to_dict() + + if checkpoint_path.exists(): + for line in checkpoint_path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + try: + existing = json.loads(line) + except json.JSONDecodeError: + continue + if all(str(existing.get(f) or "") == str(payload.get(f) or "") for f in identity_fields): + return checkpoint, False + + with checkpoint_path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(payload, sort_keys=True, ensure_ascii=False) + "\n") + return checkpoint, True + + +def load_latest_checkpoint( + runtime_root: Path, + goal_id: str, + todo_id: str, +) -> Checkpoint | None: + """Load the newest checkpoint for a Task, or ``None``.""" + checkpoint_path = _goal_checkpoint_dir(runtime_root, goal_id) / f"{todo_id}.jsonl" + if not checkpoint_path.exists(): + return None + latest: Checkpoint | None = None + for line in checkpoint_path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + try: + payload = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(payload, dict): + continue + checkpoint = Checkpoint.from_dict(payload) + if checkpoint.schema_version != CHECKPOINT_SCHEMA_VERSION: + continue + if latest is None or str(checkpoint.created_at or "") >= str(latest.created_at or ""): + # On timestamp ties, the append-ordered later checkpoint wins. + latest = checkpoint + return latest + + +def load_checkpoints( + runtime_root: Path, + goal_id: str, + todo_id: str, +) -> list[Checkpoint]: + """Load all checkpoints for a Task in append order.""" + checkpoint_path = _goal_checkpoint_dir(runtime_root, goal_id) / f"{todo_id}.jsonl" + if not checkpoint_path.exists(): + return [] + checkpoints: list[Checkpoint] = [] + for line in checkpoint_path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + try: + payload = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(payload, dict): + checkpoints.append(Checkpoint.from_dict(payload)) + return checkpoints + + +def verify_checkpoint_integrity(checkpoint: Checkpoint) -> bool: + """True when the stored state hash matches a recomputed hash.""" + if not checkpoint.state_hash: + return False + return compute_state_hash(checkpoint.state_snapshot) == checkpoint.state_hash + + +def remove_checkpoints(runtime_root: Path, goal_id: str, todo_id: str) -> bool: + """Delete a Task's checkpoint file (rollback helper).""" + checkpoint_path = _goal_checkpoint_dir(runtime_root, goal_id) / f"{todo_id}.jsonl" + if not checkpoint_path.exists(): + return False + os.remove(checkpoint_path) + return True diff --git a/loopx/control_plane/runtime/replay.py b/loopx/control_plane/runtime/replay.py new file mode 100644 index 000000000..1ead6b930 --- /dev/null +++ b/loopx/control_plane/runtime/replay.py @@ -0,0 +1,243 @@ +"""Deterministic, side-effect-free Task state replay. + +RFC C4 (Task Checkpoint and Replay) + §10.4 Replay Requirements: + +* Replay must be deterministic, idempotent, side-effect free, and + schema-version aware. +* It reconstructs *state*, never external side effects (API calls, email, + LLM invocation, file mutation must not run during replay). +* The contract is:: + + replay(events) == replay(events) + + and:: + + checkpoint + events_after_checkpoint == full_replay(events) +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Callable, Iterable, Mapping, Sequence + +from .checkpoint import ( + CHECKPOINT_SCHEMA_VERSION, + Checkpoint, + compute_state_hash, + load_latest_checkpoint, + verify_checkpoint_integrity, +) +from .time import now_utc_iso + +#: Replay context marker: the apply callback may tag derived state. +REPLAY_SCHEMA_VERSION = CHECKPOINT_SCHEMA_VERSION + +#: A state transition function: ``new_state = apply(state, event)``. +StateTransition = Callable[[Mapping[str, Any], Mapping[str, Any]], dict[str, Any]] + +#: An event filter over the raw event stream. +EventFilter = Callable[[Mapping[str, Any]], bool] + + +class ReplayViolationError(RuntimeError): + """Raised when replay invariants are violated (non-determinism, side + effects, or schema mismatch).""" + + +def replay_task( + events: Sequence[Mapping[str, Any]], + *, + initial_state: Mapping[str, Any] | None = None, + apply: StateTransition, + schema_version: int = REPLAY_SCHEMA_VERSION, +) -> dict[str, Any]: + """Deterministically fold ``events`` into state via ``apply``. + + ``apply`` must be a pure transition (no I/O, no mutation of inputs). + """ + state: dict[str, Any] = dict(initial_state or {}) + for event in events: + state = dict(apply(state, event)) + return state + + +def replay_from_checkpoint( + checkpoint: Checkpoint | None, + events_after_checkpoint: Sequence[Mapping[str, Any]], + *, + apply: StateTransition, + schema_version: int = REPLAY_SCHEMA_VERSION, +) -> dict[str, Any]: + """Recover current Task state from a checkpoint plus subsequent events. + + When no checkpoint exists, replay starts from an empty state. + """ + if checkpoint is None: + return replay_task( + events_after_checkpoint, + apply=apply, + schema_version=schema_version, + ) + if checkpoint.schema_version != schema_version: + raise ReplayViolationError( + f"checkpoint schema {checkpoint.schema_version} does not match " + f"replay schema {schema_version}" + ) + if not verify_checkpoint_integrity(checkpoint): + raise ReplayViolationError("checkpoint state hash mismatch; refusing to replay") + return replay_task( + events_after_checkpoint, + initial_state=checkpoint.state_snapshot, + apply=apply, + schema_version=schema_version, + ) + + +def verify_replay_equivalence( + events: Sequence[Mapping[str, Any]], + *, + checkpoint: Checkpoint | None, + events_after_checkpoint: Sequence[Mapping[str, Any]], + apply: StateTransition, + schema_version: int = REPLAY_SCHEMA_VERSION, +) -> tuple[bool, str]: + """Verify that ``checkpoint + events_after`` equals ``full_replay``. + + Returns ``(is_equivalent, explanation)``. + """ + full = replay_task(events, apply=apply, schema_version=schema_version) + recovered = replay_from_checkpoint( + checkpoint, + events_after_checkpoint, + apply=apply, + schema_version=schema_version, + ) + if full == recovered: + return True, "recovered state equals full replay" + return False, "recovered state diverges from full replay" + + +def partition_events_after_checkpoint( + events: Sequence[Mapping[str, Any]], + checkpoint: Checkpoint | None, + *, + event_id_of: Callable[[Mapping[str, Any]], str], +) -> list[dict[str, Any]]: + """Return events strictly after the checkpoint's ``last_event_id``. + + ``event_id_of`` extracts the stable event id from an event record. Events + whose id is missing are treated as after-checkpoint (conservative). + """ + if checkpoint is None: + return [dict(event) for event in events] + last = str(checkpoint.last_event_id or "").strip() + if not last: + return [dict(event) for event in events] + # Locate the checkpoint boundary by event id; keep everything after it. + for index, event in enumerate(events): + if str(event_id_of(event) or "").strip() == last: + return [dict(event) for event in events[index + 1 :]] + # Boundary event not found in the stream: replay conservatively from the + # beginning (checkpoint state is still applied first, so duplicates are + # tolerated by deterministic transitions). + return [dict(event) for event in events] + + +# --------------------------------------------------------------------------- +# Persisted event loading for a single Task (read-only) +# --------------------------------------------------------------------------- + + +def load_task_events( + runtime_root: Path, + goal_id: str, + todo_id: str, + *, + filter_event: EventFilter | None = None, +) -> list[dict[str, Any]]: + """Load run-index records for one Task as a replayable event stream. + + This is a read projection over the existing run index; it never mutates. + """ + index_path = Path(runtime_root) / "goals" / goal_id / "runs" / "index.jsonl" + if not index_path.exists(): + return [] + events: list[dict[str, Any]] = [] + try: + lines = index_path.read_text(encoding="utf-8").splitlines() + except OSError: + return [] + for line in lines: + if not line.strip(): + continue + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(record, dict): + continue + if str(record.get("todo_id") or "").strip() != str(todo_id).strip(): + continue + if filter_event is not None and not filter_event(record): + continue + events.append(record) + return events + + +def recover_task_state( + runtime_root: Path, + *, + goal_id: str, + todo_id: str, + apply: StateTransition, + filter_event: EventFilter | None = None, + schema_version: int = REPLAY_SCHEMA_VERSION, +) -> tuple[dict[str, Any], Checkpoint | None, list[dict[str, Any]]]: + """Full recovery workflow: latest checkpoint + subsequent events. + + Returns ``(state, checkpoint_used, events_replayed)``. + """ + events = load_task_events(runtime_root, goal_id, todo_id, filter_event=filter_event) + checkpoint = load_latest_checkpoint(runtime_root, goal_id, todo_id) + events_after = partition_events_after_checkpoint( + events, + checkpoint, + event_id_of=lambda event: str(event.get("event_id") or event.get("generated_at") or ""), + ) + state = replay_from_checkpoint( + checkpoint, + events_after, + apply=apply, + schema_version=schema_version, + ) + return state, checkpoint, events_after + + +def state_digest(state: Mapping[str, Any]) -> str: + """Content hash of replayed state (for equality checks / persistence).""" + return compute_state_hash(dict(state)) + + +def replay_audit_record( + *, + goal_id: str, + todo_id: str, + state: Mapping[str, Any], + checkpoint: Checkpoint | None, + events_replayed: Sequence[Mapping[str, Any]], + events_total: int, + equivalent: bool, +) -> dict[str, Any]: + """Build a public-safe replay audit record (no task contents).""" + return { + "goal_id": str(goal_id), + "todo_id": str(todo_id), + "schema_version": REPLAY_SCHEMA_VERSION, + "checkpoint_used": checkpoint.to_dict() if checkpoint else None, + "events_replayed": len(events_replayed), + "events_total": int(events_total), + "equivalent": bool(equivalent), + "state_hash": state_digest(state), + "recorded_at": now_utc_iso(), + } diff --git a/loopx/control_plane/scheduler/event_driven_dispatch.py b/loopx/control_plane/scheduler/event_driven_dispatch.py new file mode 100644 index 000000000..465524c5c --- /dev/null +++ b/loopx/control_plane/scheduler/event_driven_dispatch.py @@ -0,0 +1,768 @@ +"""RFC Phase 6 - Event-Driven Scheduling Pilot (narrow path). + +Wires the event-driven narrow path behind an explicit opt-in flag: + + TaskCompleted -> dependency satisfied -> TaskReady -> Queue -> Worker + +Everything in this module is opt-in (``LOOPX_EVENT_DRIVEN_DISPATCH=1`` or an +explicit ``use_event_driven=True``). When disabled every function returns a +``disabled`` marker and no state is written, preserving the legacy heartbeat +behavior (RFC 11.2: heartbeat demoted to a trigger; default unchanged). + +Design constraints (RFC 11.4): +- The rollout event log is *not* an execution event bus. It records public + audit facts (``task_ready`` / ``task_enqueued`` / ``task_dispatched``) only. +- Readiness is recomputed from the projected todo items via + ``handoff_ready_successor_todo_ids`` (handoff gates), never replayed from + rollout events. +- The task queue is a separate append-only JSONL store next to the goal state. +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any, Iterable, Mapping, Sequence + +from ...rollout_event_log import ( + ROLLOUT_EVENT_SCHEMA_VERSION, + append_rollout_event_once, + build_rollout_event, + load_rollout_events, +) +from ..new_architecture import master_switch_enabled +from ..todos.contract import ( + TODO_STATUS_BLOCKED, + TODO_STATUS_DEFERRED, + TODO_STATUS_OPEN, + TODO_TASK_CLASS_ADVANCEMENT, + TODO_TERMINAL_STATUS_VALUES, + normalize_todo_excluded_agents, + normalize_todo_id, + normalize_todo_status, + normalize_todo_task_class, +) +from ..todos.handoff_gate import ( + handoff_ready_successor_todo_ids, + todo_summary_handoff_gates, +) + +EVENT_DRIVEN_DISPATCH_ENV = "LOOPX_EVENT_DRIVEN_DISPATCH" + +TASK_READY_EVENT_KIND = "task_ready" +TASK_ENQUEUED_EVENT_KIND = "task_enqueued" +TASK_DISPATCHED_EVENT_KIND = "task_dispatched" + +TASK_QUEUE_SCHEMA_VERSION = "loopx_scheduler_task_queue_v0" +TASK_QUEUE_ENTRY_SCHEMA_VERSION = "loopx_scheduler_task_queue_entry_v0" + +QUEUE_STATUS_PENDING = "pending" +QUEUE_STATUS_CLAIMED = "claimed" +QUEUE_STATUS_DONE = "done" +QUEUE_STATUSES = { + QUEUE_STATUS_PENDING, + QUEUE_STATUS_CLAIMED, + QUEUE_STATUS_DONE, +} + +DEFAULT_TASK_QUEUE_NAME = "scheduler-task-queue.jsonl" + + +def event_driven_dispatch_enabled(use_event_driven: bool | None = None) -> bool: + """Enable event-driven dispatch. + + An explicit ``use_event_driven`` wins; otherwise the dedicated env var wins; + otherwise the new-architecture master switch decides (on by default). + """ + if use_event_driven is not None: + return bool(use_event_driven) + value = os.environ.get(EVENT_DRIVEN_DISPATCH_ENV, "").strip().lower() + if value: + return value in {"1", "true", "yes", "on"} + return master_switch_enabled() + + +def _safe_segment(value: str | None) -> str: + text = str(value or "").strip() + if not text: + raise ValueError("goal_id is required") + safe = "".join(ch for ch in text if ch.isalnum() or ch in {"-", "_"}) + if not safe: + raise ValueError(f"goal_id must be alphanumeric: {text!r}") + return safe + + +def task_queue_path(runtime_root: Path, *, goal_id: str) -> Path: + """Path to the append-only task queue JSONL for a goal.""" + return ( + Path(runtime_root).expanduser() + / "goals" + / _safe_segment(goal_id) + / DEFAULT_TASK_QUEUE_NAME + ) + + +def _queue_view(entries: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + pending = [e for e in entries if e.get("status") == QUEUE_STATUS_PENDING] + claimed = [e for e in entries if e.get("status") == QUEUE_STATUS_CLAIMED] + done = [e for e in entries if e.get("status") == QUEUE_STATUS_DONE] + return { + "schema_version": TASK_QUEUE_SCHEMA_VERSION, + "entry_count": len(entries), + "pending_count": len(pending), + "claimed_count": len(claimed), + "done_count": len(done), + "pending_todo_ids": [e.get("todo_id") for e in pending], + "claimed_todo_ids": [e.get("todo_id") for e in claimed], + } + + +def load_task_queue(path: Path) -> dict[str, Any]: + """Load the task queue snapshot (empty view when the file does not exist).""" + try: + lines = path.read_text(encoding="utf-8").splitlines() + except OSError: + return _queue_view([]) + entries: list[dict[str, Any]] = [] + for line in lines: + if not line.strip(): + continue + try: + parsed = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(parsed, dict) and parsed.get("schema_version") == TASK_QUEUE_ENTRY_SCHEMA_VERSION: + entries.append(parsed) + return _queue_view(entries) + + +def _enqueued_todo_ids(entries: Iterable[Mapping[str, Any]]) -> set[str]: + return { + str(entry.get("todo_id") or "").strip() + for entry in entries + if str(entry.get("todo_id") or "").strip() + } + + +def _queued_todo_ids(path: Path) -> set[str]: + view = load_task_queue(path) + return set(view.get("pending_todo_ids", [])) | set(view.get("claimed_todo_ids", [])) + + +def enqueue_tasks( + path: Path, + *, + goal_id: str, + todo_ids: Sequence[str], + recorded_at: str, + source: str = "event_driven_dispatch", + use_event_driven: bool | None = None, +) -> dict[str, Any]: + """Append new pending queue entries, idempotent per todo_id. + + Returns a summary with newly enqueued todo ids and skipped duplicates. + """ + if not event_driven_dispatch_enabled(use_event_driven): + return { + "ok": True, + "disabled": True, + "reason": f"{EVENT_DRIVEN_DISPATCH_ENV} not enabled", + } + goal = _safe_segment(goal_id) + path.parent.mkdir(parents=True, exist_ok=True) + existing: list[dict[str, Any]] = [] + try: + existing = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + except OSError: + existing = [] + existing = [e for e in existing if isinstance(e, dict) and e.get("schema_version") == TASK_QUEUE_ENTRY_SCHEMA_VERSION] + known = _enqueued_todo_ids(existing) + + newly: list[str] = [] + skipped: list[str] = [] + with path.open("a", encoding="utf-8") as handle: + for todo_id in todo_ids: + normalized = str(todo_id or "").strip() + if not normalized: + continue + if normalized in known: + skipped.append(normalized) + continue + entry: dict[str, Any] = { + "schema_version": TASK_QUEUE_ENTRY_SCHEMA_VERSION, + "goal_id": goal, + "todo_id": normalized, + "status": QUEUE_STATUS_PENDING, + "enqueued_at": recorded_at, + "enqueued_by": source, + } + handle.write(json.dumps(entry, sort_keys=True, ensure_ascii=False) + "\n") + known.add(normalized) + newly.append(normalized) + return { + "ok": True, + "goal_id": goal, + "newly_enqueued": newly, + "skipped_duplicates": skipped, + } + + +def claim_next_task( + path: Path, + *, + worker_id: str, + use_event_driven: bool | None = None, + capabilities: Sequence[str] | None = None, + lease_seconds: int | float | None = None, +) -> dict[str, Any] | None: + """Claim the oldest pending task for a worker (Worker Pool acquire). + + Mutates the queue in place: rewrites the JSONL with the claimed status. + Returns the claimed entry, or None when the queue is empty. + + ``capabilities`` (optional) enables capability matching: only tasks whose + ``required_capabilities`` are all present in the worker's capability set are + claimable. Tasks carrying a legacy ``capability_binding_ref`` additionally + require the worker to declare the bound pack token, and the pack must be + ``ready`` in the capability registry (fail closed) — this mirrors the + legacy capability-pack eligibility contract. ``lease_seconds`` (optional) + attaches a ``lease_until`` expiry so a crashed worker's task can be + reclaimed (zombie recovery). Both are opt-in and leave the default FIFO + claim behavior unchanged when omitted. + """ + if not event_driven_dispatch_enabled(use_event_driven): + return None + if not path.exists(): + return None + if capabilities is not None or lease_seconds is not None: + from .task_lifecycle import claim_next_eligible_task + from ...capabilities.catalog import build_capability_registry + + claimed = claim_next_eligible_task( + path, + worker_id=worker_id, + capabilities=capabilities, + lease_seconds=lease_seconds, + registry=build_capability_registry(), + ) + return claimed + lines = path.read_text(encoding="utf-8").splitlines() + entries: list[dict[str, Any]] = [] + for line in lines: + if not line.strip(): + continue + try: + parsed = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(parsed, dict) and parsed.get("schema_version") == TASK_QUEUE_ENTRY_SCHEMA_VERSION: + entries.append(parsed) + claimed: dict[str, Any] | None = None + for entry in entries: + if entry.get("status") == QUEUE_STATUS_PENDING: + entry["status"] = QUEUE_STATUS_CLAIMED + entry["claimed_by"] = str(worker_id).strip() + # Always attach a lease (default TTL) so the task is never mistaken + # for a legacy un-leased zombie by is_expired/reconcile. A claimed + # task without lease_until would be reclaimed on the very next + # reconcile tick, causing a claim->reclaim->claim livelock. + from .task_lifecycle import lease_expiry + + entry["lease_until"] = lease_expiry(lease_seconds) + claimed = entry + break + if claimed is None: + return None + path.write_text( + "".join(json.dumps(e, sort_keys=True, ensure_ascii=False) + "\n" for e in entries), + encoding="utf-8", + ) + return claimed + + +def record_task_event( + event_log_path: Path, + *, + goal_id: str, + event_kind: str, + todo_id: str, + agent_id: str | None = None, + source_event_id: str | None = None, + details: Mapping[str, Any] | None = None, + recorded_at: str | None = None, + use_event_driven: bool | None = None, +) -> dict[str, Any]: + """Record one public audit event, idempotent by (event_kind, todo_id).""" + if not event_driven_dispatch_enabled(use_event_driven): + return { + "ok": True, + "disabled": True, + "reason": f"{EVENT_DRIVEN_DISPATCH_ENV} not enabled", + } + event = build_rollout_event( + goal_id=goal_id, + event_kind=event_kind, + agent_id=agent_id, + todo_id=todo_id, + source_event_id=source_event_id, + details=details, + recorded_at=recorded_at, + ) + appended, is_new = append_rollout_event_once( + Path(event_log_path), + event, + identity_fields=("goal_id", "event_kind", "todo_id"), + ) + return { + "ok": True, + "event": appended, + "new": is_new, + } + + +def _outstanding_acceptance_pending( + event_log_path: Path | None, +) -> list[dict[str, Any]]: + """Return the unresolved acceptance gaps recorded for a goal, if any. + + A ``goal_acceptance_pending`` fact means the goal previously declared + acceptance criteria that are still unsatisfied. A later criteria-less + dispatch tick (e.g. heartbeat-driven) must treat those gaps as still open + and hold the goal in WAIT rather than closing it and silently dropping the + acceptance gate. The pending fact is only cleared once a subsequent + ``goal_acceptance_satisfied`` or ``goal_closed`` fact is recorded. + """ + if event_log_path is None: + return [] + events = load_rollout_events(event_log_path) + pending: dict[str, dict[str, Any]] = {} + for event in events: + kind = event.get("event_kind") + if kind == "goal_acceptance_pending": + gaps = event.get("acceptance_gaps") or [] + if not isinstance(gaps, list): + gaps = [] + for gap in gaps: + if isinstance(gap, dict) and gap.get("criterion_id"): + pending[str(gap["criterion_id"])] = gap + elif kind in ("goal_acceptance_satisfied", "goal_closed"): + # A satisfied/closed fact clears any prior pending gate. + pending.clear() + return list(pending.values()) + + +def build_event_driven_dispatch( + *, + runtime_root: Path, + goal_id: str, + items: Sequence[Mapping[str, Any]], + completed_todo_id: str | None = None, + event_log_path: Path | None = None, + worker_id: str | None = None, + agent_id: str | None = None, + recorded_at: str | None = None, + use_event_driven: bool | None = None, + reconcile: bool = False, + worker_capabilities: Sequence[str] | None = None, + lease_seconds: int | float | None = None, + acceptance_criteria: Mapping[str, Any] | None = None, + evidence: Mapping[str, Any] | None = None, + acceptance_base_dir: Path | None = None, +) -> dict[str, Any]: + """Advance READY successors, enqueue them, and (optionally) claim for a worker. + + Narrow RFC Phase 6 path: + + TaskCompleted -> dependency satisfied -> TaskReady -> Queue -> Worker + + Steps: + 0. Optional reconciliation (``reconcile=True``): expire stale leases (zombie + recovery) and promote ready retries, per ``plan/new_plan.md`` P0. + 1. Recompute READY successors from handoff gates (pure function). + 2. Record a ``task_ready`` rollout audit event per successor (idempotent). + 3. Append each successor to the task queue (idempotent). + 4. Optionally claim the next task for a worker (Worker Pool acquire). + + ``worker_id`` names the claimer (written to the queue entry's ``claimed_by``); + ``agent_id`` names the registered LoopX agent identity recorded on the + ``task_dispatched`` audit event. When ``agent_id`` is omitted it falls back + to ``worker_id`` so existing callers keep recording the claimer identity. + + ``worker_capabilities`` and ``lease_seconds`` (optional) are forwarded to + :func:`claim_next_task` for capability-matched, lease-bound claiming. + + When the opt-in flag is off, returns a ``disabled`` marker and writes nothing. + """ + from ..runtime.time import now_utc_iso + + if not event_driven_dispatch_enabled(use_event_driven): + return { + "ok": True, + "disabled": True, + "reason": f"{EVENT_DRIVEN_DISPATCH_ENV} not enabled", + "goal_id": goal_id, + } + stamp = recorded_at or now_utc_iso() + queue_path = task_queue_path(runtime_root, goal_id=goal_id) + + # 0a. Materialize the completed todo as a terminal fact. The ``--completed-todo-id`` + # flag carries the semantics "this todo is DONE, advance from here", but the + # legacy path only wrote it into task_ready audit details and never marked the + # todo complete — so the todo stayed open, readiness kept recomputing it as + # READY, and closure reported ``ready_work_remaining`` forever. Record a + # ``todo_complete`` rollout fact so ``load_todo_items_from_rollout_log`` (and + # the closure evaluator) see the terminal status on this and future ticks. + if completed_todo_id and event_log_path is not None: + record_task_event( + event_log_path, + goal_id=goal_id, + event_kind="todo_complete", + todo_id=completed_todo_id, + source_event_id=None, + details={"cause": "event_driven_dispatch_completed_todo_id"}, + recorded_at=stamp, + use_event_driven=use_event_driven, + ) + # Reflect the completion on the in-memory item set for THIS tick as well, so + # readiness and closure evaluate against the terminal state immediately + # (rather than only on the next tick after the rollout log is replayed). + if completed_todo_id: + normalized_completed = normalize_todo_id(completed_todo_id) + items = [ + dict(item, status="done") + if isinstance(item, dict) + and normalize_todo_id(item.get("todo_id")) == normalized_completed + else item + for item in items + ] + + # 0. Optional reconciliation: expire zombie leases and promote ready retries. + reconcile_result: dict[str, Any] | None = None + if reconcile: + from .task_lifecycle import reconcile_queue + + reconcile_result = reconcile_queue( + queue_path, + worker_id=worker_id, + recorded_at=stamp, + ) + + # 1. READY successors (pure; reuses handoff gate readiness). + ready_successors = advance_ready_todo_ids(items) + enqueued = _queued_todo_ids(queue_path) + ready_successors = [todo_id for todo_id in ready_successors if todo_id not in enqueued] + + # 2. Record task_ready audit events. + ready_events: list[dict[str, Any]] = [] + if event_log_path is not None: + for todo_id in ready_successors: + recorded = record_task_event( + event_log_path, + goal_id=goal_id, + event_kind=TASK_READY_EVENT_KIND, + todo_id=todo_id, + source_event_id=None, + details={ + "completed_todo_id": completed_todo_id, + "cause": "handoff_gate_cleared", + }, + recorded_at=stamp, + use_event_driven=use_event_driven, + ) + if not recorded.get("disabled"): + ready_events.append(recorded["event"]) + + # 3. Enqueue. + enqueued_result = enqueue_tasks( + queue_path, + goal_id=goal_id, + todo_ids=ready_successors, + recorded_at=stamp, + use_event_driven=use_event_driven, + ) + enqueued_events: list[dict[str, Any]] = [] + if event_log_path is not None: + for todo_id in enqueued_result.get("newly_enqueued", []): + recorded = record_task_event( + event_log_path, + goal_id=goal_id, + event_kind=TASK_ENQUEUED_EVENT_KIND, + todo_id=todo_id, + source_event_id=None, + details={"queue_position": None}, + recorded_at=stamp, + use_event_driven=use_event_driven, + ) + if not recorded.get("disabled"): + enqueued_events.append(recorded["event"]) + + # 4. Worker acquire (optional). + dispatched: dict[str, Any] | None = None + if worker_id: + claimed = claim_next_task( + queue_path, + worker_id=worker_id, + use_event_driven=use_event_driven, + capabilities=worker_capabilities, + lease_seconds=lease_seconds, + ) + if claimed is not None: + if event_log_path is not None: + # Prefer the registered LoopX agent identity; fall back to the + # claimer (worker_id) so legacy callers stay unchanged. + dispatch_agent_id = (agent_id or "").strip() or worker_id + record_task_event( + event_log_path, + goal_id=goal_id, + event_kind=TASK_DISPATCHED_EVENT_KIND, + todo_id=str(claimed.get("todo_id") or ""), + agent_id=dispatch_agent_id, + source_event_id=None, + details={"status": claimed.get("status")}, + recorded_at=stamp, + use_event_driven=use_event_driven, + ) + dispatched = { + "todo_id": claimed.get("todo_id"), + "claimed_by": claimed.get("claimed_by"), + "status": claimed.get("status"), + } + + # 5. Closure evaluation (event-driven, derived): when no ready successors + # remain and the queue holds no pending/claimed work, the Closure Evaluator + # decides whether the goal is done. This decouples Goal closure from Todo + # lifecycle: we do NOT require every todo to carry an explicit no_followup + # intent — empty ready + empty queue + no replan is enough to emit + # goal_closure_ready (plan/new_plan.md elegant-close design). + closure: dict[str, Any] | None = None + if event_log_path is not None and not ready_successors: + from ..goals.goal_closure import ( + build_goal_closure_state, + evaluate_goal_closure, + maybe_close_goal, + ) + from ..goals.goal_acceptance import evaluate_goal_acceptance + + queue_view = load_task_queue(queue_path) + # Derive remaining (non-terminal) work directly from the projected items, + # NOT hardcoded empty lists. ``advance_ready_todo_ids`` only reports OPEN + # advancement todos as READY; ``blocked`` / ``deferred`` todos are NOT + # "ready" but ARE still unfinished work that must block goal closure. + # Without this, a goal with a blocked (or deferred) todo would be wrongly + # ``goal_closed`` the moment no *ready* successors remain — e.g. another + # todo submitted mid-run that is blocked on a dependency, or deferred work + # that has not been scheduled yet. + blocked_todo_ids: list[str] = [] + deferred_todo_ids: list[str] = [] + # Remaining *executable advancement* work. Reuse the same readiness rules + # as ``advance_ready_todo_ids`` (unfiltered by queue membership) so that + # non-advancement todos — continuous_monitor, user_gate, user_action, + # blocker — that are still "open" do NOT falsely block goal closure. + # These todos live on their own lifecycles (background monitor, user + # action) and are never claimed by the advancement worker pool; counting + # them as "ready work" would wedge the goal in RUN forever. + open_todo_ids: list[str] = [ + tid for tid in advance_ready_todo_ids(items) + ] + for item in items: + if not isinstance(item, dict): + continue + tid = normalize_todo_id(item.get("todo_id")) + if not tid: + continue + status = normalize_todo_status(item.get("status")) + if status == TODO_STATUS_BLOCKED: + blocked_todo_ids.append(tid) + elif status == TODO_STATUS_DEFERRED: + deferred_todo_ids.append(tid) + # Run the Goal Acceptance / Evidence Verification layer up-front. When + # acceptance criteria + evidence are supplied, the closure state carries + # them so the Closure Evaluator can distinguish RUN (more work) from WAIT + # (evidence gaps) from CLOSE (done + verified). This is what lets a single + # dispatch call drive the full acceptance -> goal_closed loop instead of + # forcing the agent to hand-run goal-closure --verify --apply (the + # long refresh-state retry loop seen in the website1 color session). + acceptance_eval = None + if acceptance_criteria is not None or evidence is not None: + acceptance_eval = evaluate_goal_acceptance( + acceptance_criteria=acceptance_criteria, + evidence=evidence, + base_dir=acceptance_base_dir, + ) + else: + # No criteria/evidence on this tick. Do NOT silently skip acceptance: + # if a prior dispatch already recorded a ``goal_acceptance_pending`` + # fact (acceptance criteria were declared earlier and remain + # unsatisfied), a criteria-less dispatch (e.g. a heartbeat-driven + # tick) must NOT close the goal and erase the outstanding acceptance + # gate. Synthesize an unsatisfied acceptance evaluation so the + # Closure Evaluator holds the goal in WAIT until evidence arrives. + pending = _outstanding_acceptance_pending(event_log_path) + if pending: + acceptance_eval = { + "satisfied": False, + "acceptance_gaps": pending, + "criteria_results": pending, + "evidence_count": 0, + "verified_count": 0, + "criteria_count": len(pending), + } + closure_state = build_goal_closure_state( + ready_todo_ids=open_todo_ids, + pending_dependency_ids=[], + blocked_todo_ids=blocked_todo_ids, + deferred_todo_ids=deferred_todo_ids, + replan_required=False, + external_followup_required=False, + open_todo_count=max(queue_view.get("pending_count", 0), len(open_todo_ids)), + claimed_advancement_count=queue_view.get("claimed_count", 0), + acceptance=acceptance_eval, + ) + closure = evaluate_goal_closure(closure_state) + if closure["ready"]: + # Atomic close: emits goal_closure_ready AND goal_closed, including + # the acceptance-satisfied check. No separate manual step required. + maybe_close_goal( + log_path=event_log_path, + goal_id=goal_id, + state=closure_state, + agent_id=agent_id, + ) + + return { + "ok": True, + "goal_id": _safe_segment(goal_id), + "event_driven_dispatch": { + "enabled": True, + "ready_successors": ready_successors, + "newly_enqueued": enqueued_result.get("newly_enqueued", []), + "skipped_duplicates": enqueued_result.get("skipped_duplicates", []), + "dispatched": dispatched, + "reconcile": reconcile_result, + "closure": closure, + "queue": load_task_queue(queue_path), + }, + "recorded_events": { + "task_ready": ready_events, + "task_enqueued": enqueued_events, + }, + } + + +def _has_excluded_agents(item: Mapping[str, Any]) -> bool: + """True when the item declares excluded_agents (i.e. it is a handoff gate).""" + return bool(normalize_todo_excluded_agents(item.get("excluded_agents"))) + + +def advance_ready_todo_ids( + items: Sequence[Mapping[str, Any]], +) -> list[str]: + """Return the READY todo ids for the given projected items. + + Pure function: recomputes handoff gate readiness from the current item set. + + A todo is READY when it is: + 1. A CLEARED_WITH_SUCCESSOR gate's successor (handoff gate chain); or + 2. An unconstrained open advancement todo that is not gated by any handoff + gate successor edge and not yet done. This covers the common "initial + READY todos" shape (RFC Phase 5/6), where an independent agent task has + no handoff gate dependency and must still enter the Task Queue so a + resident Worker can claim it. + + Callers filter against already-enqueued ids for idempotency. + """ + item_list = list(items) + ready: set[str] = set() + + # Authoritative terminal statuses (done/deferred/closed/...). A todo in a + # terminal state must NEVER be (re)enqueued or claimed, even when a handoff + # gate still lists it as a successor. Without this filter, stale successors + # left over from a previous (font) task are re-enqueued and claimed by the + # event-driven scheduler even though the markdown active state already + # considers them done — exactly the drift seen in the website1 color session. + terminal_todo_ids: set[str] = set() + for item in item_list: + if not isinstance(item, dict): + continue + normalized = normalize_todo_id(item.get("todo_id")) + if not normalized: + continue + status = str(item.get("status") or "").strip() + if status in TODO_TERMINAL_STATUS_VALUES or ( + status and status not in {TODO_STATUS_OPEN, "blocked", "in_progress", "active", "pending"} + ): + terminal_todo_ids.add(normalized) + + # 1. Handoff gate successors (existing semantics), excluding terminal todos. + for todo_id in handoff_ready_successor_todo_ids({"items": item_list}): + normalized = str(todo_id or "").strip() + if normalized and normalized not in terminal_todo_ids: + ready.add(normalized) + + # 2. Unconstrained open advancement todos not referenced by any handoff gate. + # An item is "gated" when some gate either directly unblocks it + # (gate.unblocks_todo_id) or lists it as a successor edge. + gated_todo_ids: set[str] = set() + for gate in todo_summary_handoff_gates({"items": item_list}): + if not isinstance(gate, dict): + continue + direct = normalize_todo_id(gate.get("unblocks_todo_id")) + if direct: + gated_todo_ids.add(direct) + successor_ids = gate.get("successor_todo_ids") + if isinstance(successor_ids, list): + for todo_id in successor_ids: + normalized = normalize_todo_id(todo_id) + if normalized: + gated_todo_ids.add(normalized) + for item in item_list: + if not isinstance(item, dict): + continue + if str(item.get("status") or "").strip() != TODO_STATUS_OPEN: + continue + todo_id = normalize_todo_id(item.get("todo_id")) + if not todo_id or todo_id in ready: + continue + if normalize_todo_task_class( + item.get("task_class"), + text=str(item.get("text") or "").strip(), + action_kind=item.get("action_kind"), + ) != TODO_TASK_CLASS_ADVANCEMENT: + continue + # Skip handoff gates themselves (excluded_agents) — gate readiness is + # driven by the gate state machine, not the free-advancement rule. + if _has_excluded_agents(item): + continue + # Skip items that are successors of any gate (gate drives their readiness). + if todo_id in gated_todo_ids: + continue + # Skip items whose readiness is delegated to a "resume_when" / supersede edge. + if normalize_todo_id(item.get("superseded_by")): + continue + ready.add(todo_id) + + return sorted(ready) + + +__all__ = [ + "EVENT_DRIVEN_DISPATCH_ENV", + "TASK_READY_EVENT_KIND", + "TASK_ENQUEUED_EVENT_KIND", + "TASK_DISPATCHED_EVENT_KIND", + "TASK_QUEUE_SCHEMA_VERSION", + "TASK_QUEUE_ENTRY_SCHEMA_VERSION", + "QUEUE_STATUS_PENDING", + "QUEUE_STATUS_CLAIMED", + "QUEUE_STATUS_DONE", + "QUEUE_STATUSES", + "DEFAULT_TASK_QUEUE_NAME", + "event_driven_dispatch_enabled", + "task_queue_path", + "load_task_queue", + "enqueue_tasks", + "claim_next_task", + "record_task_event", + "build_event_driven_dispatch", + "advance_ready_todo_ids", +] diff --git a/loopx/control_plane/scheduler/merge.py b/loopx/control_plane/scheduler/merge.py new file mode 100644 index 000000000..db8dd7d39 --- /dev/null +++ b/loopx/control_plane/scheduler/merge.py @@ -0,0 +1,327 @@ +"""Opt-in merge of event-driven scheduling with the heartbeat polling path. + +RFC Phase 5 comprehensive eventing: the heartbeat polling path and the +event-driven dispatch path are formally merged behind an explicit opt-in flag. +When enabled, one heartbeat tick: + + 1. records a ``heartbeat_observed`` event fact (heartbeat = event source); + 2. computes the unified decision through :class:`PolicyEngine`; + 3. advances READY successors and enqueues them (event-driven dispatch); + 4. lets a worker acquire the next task (Worker Pool). + +When the opt-in flag is off, the legacy heartbeat polling path is untouched and +no event facts, queue writes, or claims happen here. + +This merge is *composition*, not a rewrite: it delegates to the existing +``policy/engine``, ``heartbeat/event_source``, and +``scheduler/event_driven_dispatch`` modules. +""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import Any, Callable, Mapping, Sequence + +from ...event_sourced_state import AppendOnlyStateEventStore, build_state_projection +from ...rollout_event_log import load_rollout_events, rollout_event_log_path +from ..heartbeat.event_source import ( + HEARTBEAT_OBSERVED_EVENT_KIND, + heartbeat_event_source_enabled, + record_heartbeat_observation, +) +from ..new_architecture import master_switch_enabled +from ..policy import PolicyEngine +from ..policy.decision_events import record_policy_decision +from ..runtime.time import now_utc_iso +from .event_driven_dispatch import ( + EVENT_DRIVEN_DISPATCH_ENV, + build_event_driven_dispatch, + event_driven_dispatch_enabled, + load_task_queue, + task_queue_path, +) + +MERGE_PATH_ENV = "LOOPX_MERGE_EVENT_DRIVEN_AND_HEARTBEAT" + +MERGE_SCHEMA_VERSION = "loopx_event_driven_heartbeat_merge_v0" + +DecisionFactory = Callable[..., Any] + + +def merge_enabled( + *, + use_event_driven: bool | None = None, + use_event_source: bool | None = None, + use_merge: bool | None = None, +) -> bool: + """The merge is on when eventing, event-source, and merge flags allow it. + + An explicit ``use_merge`` wins; otherwise the dedicated env var wins; + otherwise the new-architecture master switch decides (on by default). + """ + if use_merge is not None: + return bool(use_merge) + value = os.environ.get(MERGE_PATH_ENV, "").strip().lower() + if value: + if value not in {"1", "true", "yes", "on"}: + return False + elif not master_switch_enabled(): + return False + return ( + event_driven_dispatch_enabled(use_event_driven) + and heartbeat_event_source_enabled(use_event_source) + ) + + +def load_todo_items_from_rollout_log( + runtime_root: Path, + goal_id: str, + event_log_path: Path | None = None, +) -> list[dict[str, Any]]: + """Build todo items from ``todo_add`` / ``todo_complete`` rollout events. + + Real goals record task mutations in the rollout event log (``todo_add`` / + ``todo_complete``) rather than a separate ``events.jsonl`` state store. This + reconstructs the latest status per todo_id so the event-driven dispatch can + recompute readiness for real goal data. + + Only public-safe fields are used (todo_id, status, role); raw task text is + intentionally not reconstructed here. + """ + resolved_goal = str(goal_id or "").strip() + log_path = ( + Path(event_log_path) + if event_log_path is not None + else rollout_event_log_path(runtime_root, resolved_goal) + ) + statuses: dict[str, str] = {} + roles: dict[str, str] = {} + unblocks: dict[str, str] = {} + excluded: dict[str, list[str]] = {} + task_classes: dict[str, str] = {} + try: + events = load_rollout_events(log_path) + except OSError: + return [] + for event in events: + kind = event.get("event_kind") + if kind not in {"todo_add", "todo_complete"}: + continue + todo_id = str(event.get("todo_id") or "").strip() + if not todo_id: + continue + # A todo_complete event is authoritative: the todo has reached a + # terminal state and must never be recomputed as READY again. Real + # goal logs replay every todo_add/todo_complete in order, so the + # *last* event for a todo_id wins; when the last event is + # todo_complete we normalize the status to the canonical terminal + # value. This prevents already-done todos (including stale + # cross-goal zombies like `todo_change_font`) from being re-admitted + # to the Task Queue by advance_ready_todo_ids — the drift seen in + # the website1 color session where goal-closure reported ready=6. + if kind == "todo_complete": + statuses[todo_id] = "done" + continue + # todo_add wins only when not already finalized by a completion. + statuses.setdefault(todo_id, str(event.get("status") or "open")) + details = event.get("details") or {} + if not isinstance(details, dict): + details = {} + role = str(details.get("role")) if details.get("role") else "agent" + roles.setdefault(todo_id, role) + dep = details.get("unblocks_todo_id") + if not dep: + causality = event.get("causality") or {} + dep_list = causality.get("unblocks") if isinstance(causality, dict) else None + if isinstance(dep_list, list) and dep_list: + dep = str(dep_list[0]) + if dep: + unblocks[todo_id] = str(dep) + excl = details.get("excluded_agents") + if isinstance(excl, list): + excluded[todo_id] = [str(a) for a in excl] + elif isinstance(excl, str) and excl: + # Accept both "a,b" and stringified repr lists like "['agent_worker']". + cleaned = excl.strip() + if cleaned.startswith("[") and cleaned.endswith("]"): + cleaned = cleaned[1:-1].replace("'", "").replace('"', "") + excluded[todo_id] = [a.strip() for a in cleaned.split(",") if a.strip()] + tc = details.get("task_class") + if tc: + task_classes[todo_id] = str(tc) + items = [ + { + "todo_id": todo_id, + "status": status, + "role": roles.get(todo_id, "agent"), + "task_class": task_classes.get(todo_id, "advancement_task"), + "unblocks_todo_id": unblocks.get(todo_id), + "excluded_agents": excluded.get(todo_id, []), + } + for todo_id, status in statuses.items() + ] + return items + + +def _loaded_items( + runtime_root: Path, + goal_id: str, +) -> tuple[list[dict[str, Any]], Path]: + """Load projected user+agent todo items and the rollout event log path. + + Prefers the dedicated ``events.jsonl`` state store; falls back to the + ``todo_add`` / ``todo_complete`` rollout events when the store is absent + (the common shape for real goals). + """ + log_path = rollout_event_log_path(runtime_root, goal_id) + state_log_path = runtime_root / "goals" / str(goal_id) / "events.jsonl" + state_events = AppendOnlyStateEventStore(state_log_path).load() + projection = build_state_projection(state_events, goal_id=goal_id) + items = [ + *projection.get("user_todos", {}).get("items", []), + *projection.get("agent_todos", {}).get("items", []), + ] + if not items: + items = load_todo_items_from_rollout_log(runtime_root, goal_id, log_path) + return items, log_path + + +def merge_event_driven_and_heartbeat( + *, + runtime_root: Path, + goal_id: str, + agent_id: str | None = None, + completed_todo_id: str | None = None, + worker_id: str | None = None, + status_payload: Mapping[str, Any] | None = None, + items: Sequence[Mapping[str, Any]] | None = None, + event_log_path: Path | None = None, + recorded_at: str | None = None, + tick_id: str | None = None, + use_event_driven: bool | None = None, + use_event_source: bool | None = None, + use_merge: bool | None = None, + record_policy_decisions: bool | None = None, +) -> dict[str, Any]: + """Run the merged heartbeat + event-driven path behind the opt-in flag. + + Returns a merged payload with four sections: + + * ``heartbeat`` — the recorded ``heartbeat_observed`` event fact; + * ``policy_decision`` — the unified :class:`PolicyEngine` decision; + * ``event_driven_dispatch`` — READY advancement, enqueue, worker claim; + * ``queue`` — the resulting task queue view. + + When the merge (or any required sub-flag) is off, returns a ``disabled`` + marker and writes nothing. + """ + resolved_goal_id = str(goal_id or "").strip() + stamp = recorded_at or now_utc_iso() + log_path = ( + Path(event_log_path) + if event_log_path is not None + else rollout_event_log_path(runtime_root, resolved_goal_id) + ) + enabled = merge_enabled( + use_event_driven=use_event_driven, + use_event_source=use_event_source, + use_merge=use_merge, + ) + if not enabled: + return { + "ok": True, + "disabled": True, + "reason": f"{MERGE_PATH_ENV} not enabled (eventing + event-source + merge all required)", + "goal_id": resolved_goal_id, + } + + # 1. Heartbeat = event source: record the observation fact. + heartbeat = record_heartbeat_observation( + runtime_root=runtime_root, + goal_id=resolved_goal_id, + agent_id=agent_id, + event_log_path=log_path, + source="heartbeat_poll", + tick_id=tick_id, + status=str((status_payload or {}).get("decision") or None) + if status_payload + else None, + details={ + "cause": "merged_event_driven_and_heartbeat", + "completed_todo_id": completed_todo_id, + }, + recorded_at=stamp, + use_event_source=use_event_source, + ) + + # 2. Decision through the unified PolicyEngine. + # Forward scheduler_execution_context (if present in the status payload) + # so the policy composition can validate the scheduler context instead of + # short-circuiting with "missing required field". + decision: Any = None + if status_payload is not None: + engine = PolicyEngine() + supplied = dict(status_payload) + scheduler_ctx = supplied.get("scheduler_execution_context") + if scheduler_ctx is None and "scheduler_execution_context" in supplied: + scheduler_ctx = supplied["scheduler_execution_context"] + decision = engine.decide( + status_payload=supplied, + goal_id=resolved_goal_id, + agent_id=agent_id, + scheduler_execution_context=scheduler_ctx, + ) + if record_policy_decisions is None: + value = os.environ.get("LOOPX_POLICY_DECISION_RECORD", "").strip().lower() + record_policy_decisions = ( + value in {"1", "true", "yes", "on"} if value else master_switch_enabled() + ) + if record_policy_decisions: + record_policy_decision( + decision, + goal_id=resolved_goal_id, + agent_id=agent_id, + log_path=log_path, + state_dir=runtime_root / "goals" / resolved_goal_id / "policy-decision-state", + transition_only=True, + ) + + # 3. Event-driven dispatch advancement. + if items is None: + projected_items, _ = _loaded_items(runtime_root, resolved_goal_id) + else: + projected_items = [dict(item) for item in items] + dispatch = build_event_driven_dispatch( + runtime_root=runtime_root, + goal_id=resolved_goal_id, + items=projected_items, + completed_todo_id=completed_todo_id, + event_log_path=log_path, + worker_id=worker_id, + recorded_at=stamp, + use_event_driven=use_event_driven, + ) + + queue = load_task_queue(task_queue_path(runtime_root, goal_id=resolved_goal_id)) + + return { + "ok": True, + "schema_version": MERGE_SCHEMA_VERSION, + "goal_id": resolved_goal_id, + "agent_id": agent_id, + "enabled": True, + "heartbeat": heartbeat, + "policy_decision": decision.to_dict() if decision is not None else None, + "event_driven_dispatch": dispatch.get("event_driven_dispatch"), + "recorded_events": dispatch.get("recorded_events"), + "queue": queue, + } + + +__all__ = [ + "MERGE_PATH_ENV", + "MERGE_SCHEMA_VERSION", + "merge_enabled", + "merge_event_driven_and_heartbeat", +] diff --git a/loopx/control_plane/scheduler/resident.py b/loopx/control_plane/scheduler/resident.py new file mode 100644 index 000000000..ab9226cac --- /dev/null +++ b/loopx/control_plane/scheduler/resident.py @@ -0,0 +1,811 @@ +"""Scheduler converged into a resident Task Queue + Worker Pool (RFC Phase 5). + +The Scheduler no longer owns business decisions: it is a resident process that +only answers *timing* and *which task may execute now*. The pipeline is: + + Trigger -> Scheduler -> Policy -> Queue -> Worker -> Agent -> Events + +This module provides the resident execution machinery: + +* :class:`WorkerPool` — a bounded set of workers that acquire the next pending + task from the append-only JSONL queue via ``claim_next_task``. +* :class:`ResidentScheduler` — a resident loop that recomputes READY successors + from handoff gates, enqueues them, and hands them to idle workers. +* ``run_resident_scheduler_loop`` / ``run_resident_scheduler_bounded`` — loop + drivers (the latter bounded for tests and one-shot cron use). + +Design constraints (RFC §11.4): + +* The rollout event log is *not* an execution bus; it records public audit + facts only (``task_ready`` / ``task_enqueued`` / ``task_dispatched``). +* Readiness is recomputed from the projected todo items, never replayed. +* The task queue is a separate append-only JSONL store next to goal state. +* Everything is opt-in behind ``LOOPX_EVENT_DRIVEN_DISPATCH`` (reused) or an + explicit ``use_event_driven=True``; the legacy heartbeat path is unchanged. +""" + +from __future__ import annotations + +import os +import shlex +import subprocess +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable, Iterable, Mapping, Sequence + +from ...rollout_event_log import rollout_event_log_path +from ..runtime.time import now_utc_iso +from .event_driven_dispatch import ( + EVENT_DRIVEN_DISPATCH_ENV, + QUEUE_STATUS_PENDING, + TASK_DISPATCHED_EVENT_KIND, + build_event_driven_dispatch, + claim_next_task, + event_driven_dispatch_enabled, + task_queue_path, +) + +RESIDENT_SCHEDULER_SCHEMA_VERSION = "loopx_resident_scheduler_v0" + + +def _env_flag(name: str, default: bool = False) -> bool: + value = os.environ.get(name, "").strip().lower() + if not value: + return default + return value in {"1", "true", "yes", "on"} + + +def load_todo_items_from_rollout_log( + runtime_root: Path, + goal_id: str, + event_log_path: Path | None = None, +) -> list[dict[str, Any]]: + """Build todo items from ``todo_add`` / ``todo_complete`` rollout events. + + Real goals record task mutations in the rollout event log rather than a + separate ``events.jsonl`` state store. This reconstructs the latest status + per todo_id so event-driven dispatch can recompute readiness for real goal + data. Only public-safe fields are used. + """ + resolved_goal = str(goal_id or "").strip() + log_path = ( + Path(event_log_path) + if event_log_path is not None + else rollout_event_log_path(runtime_root, resolved_goal) + ) + # Reuse the canonical implementation in ``merge`` to avoid divergent copies. + # (The duplicate here previously lacked the ``todo_complete -> done`` + # authoritative-normalization, so a completed todo whose event carried a + # non-"done" status field was silently re-admitted as open.) + from .merge import load_todo_items_from_rollout_log as _canonical + + return _canonical(runtime_root, goal_id, event_log_path) + + +def _loaded_items( + runtime_root: Path, + goal_id: str, +) -> tuple[list[dict[str, Any]], Path]: + """Load projected user+agent todo items and the rollout event log path. + + Delegates to the canonical ``merge._loaded_items`` to avoid a divergent + duplicate (single source of truth for events.jsonl vs rollout-log fallback). + """ + from .merge import _loaded_items as _canonical + + return _canonical(runtime_root, goal_id) + + +WorkerTaskRunner = Callable[..., dict[str, Any]] + + +def _command_matches_worker_prefix(command: str | None, prefixes: Sequence[str]) -> bool: + """Match a worker command against an allow-list of command prefixes. + + Mirrors the original scheduler executor gate (``_command_matches_allowed_prefix``) + so the new-architecture worker executes only whitelisted commands. + """ + if not command or not prefixes: + return False + try: + command_parts = shlex.split(command) + except ValueError: + command_parts = [] + first = command_parts[0] if command_parts else command.strip().split(None, 1)[0] + for prefix in prefixes: + prefix = str(prefix).strip() + if not prefix: + continue + if first == prefix or first == prefix.split(None, 1)[0]: + return True + if command.strip() == prefix or command.strip().startswith(f"{prefix} "): + return True + return False + + +def _run_worker_shell_command( + command: str, + *, + timeout_seconds: float = 60.0, + capture_output: bool = False, +) -> dict[str, Any]: + """Run a worker task command (default runner for :data:`WorkerTaskRunner`).""" + import subprocess as _subprocess + + stdout = _subprocess.PIPE if capture_output else _subprocess.DEVNULL + stderr = _subprocess.STDOUT if capture_output else _subprocess.DEVNULL + try: + completed = _subprocess.run( + command, + shell=True, + timeout=timeout_seconds, + stdout=stdout, + stderr=stderr, + ) + output = "" + if capture_output: + output = (completed.stdout or b"").decode("utf-8", errors="replace").strip() + return { + "ok": completed.returncode == 0, + "returncode": completed.returncode, + "timed_out": False, + "output_captured": capture_output, + "output": output, + } + except _subprocess.TimeoutExpired: + return {"ok": False, "returncode": None, "timed_out": True, "output_captured": capture_output, "output": ""} + + +def execute_claimed_task( + claimed_entry: Mapping[str, Any], + *, + worker_command: str | None = None, + worker_command_prefixes: Sequence[str] | None = None, + guard_checked: bool = False, + runner: WorkerTaskRunner | None = None, +) -> dict[str, Any]: + """Optionally execute a claimed task behind explicit opt-in gates. + + Mirrors the original scheduler executor gates (``codex_cli_scheduler``): + + * ``worker_command`` + ``worker_command_prefixes`` must both be set, and the + command must match an allowed prefix. + * ``guard_checked`` must be True (fresh quota guard confirmation). + * The ``runner`` is injectable for tests (default runs the shell command). + + A task is executed only when all gates pass; otherwise the execution is + skipped with a ``reason`` (never auto-run). + """ + worker_command_prefixes = list(worker_command_prefixes or []) + runner = runner or _run_worker_shell_command + command = str(worker_command).strip() if worker_command else None + result: dict[str, Any] = { + "todo_id": str(claimed_entry.get("todo_id") or "").strip(), + "claimed_by": str(claimed_entry.get("claimed_by") or "").strip(), + "executed": False, + "reason": None, + "output_captured": False, + } + if not command: + result["reason"] = "worker_command_missing" + return result + if not guard_checked: + result["reason"] = "fresh_quota_guard_confirmation_required" + return result + if not worker_command_prefixes: + result["reason"] = "worker_command_prefix_required" + return result + if not _command_matches_worker_prefix(command, worker_command_prefixes): + result["reason"] = "worker_command_prefix_mismatch" + return result + run_result = runner(command) + result.update( + executed=run_result.get("ok") is True, + ok=run_result.get("ok") is True, + reason="executed" if run_result.get("ok") is True else "execution_failed", + output_captured=run_result.get("output_captured") is True, + returncode=run_result.get("returncode"), + timed_out=run_result.get("timed_out") is False or run_result.get("timed_out"), + ) + return result + + +def _sync_registry_goal_closed_from_runtime(runtime_root: Path, goal_id: str) -> bool: + """Best-effort registry sync for a closed goal, given only ``runtime_root``. + + The resident scheduler does not carry the project directory; it learns the + project repo from the global registry (``/registry.global.json``) + and then syncs the project-local ``.loopx/registry.json``. A missing/partial + registry is a silent no-op. + """ + try: + import json + + global_path = Path(runtime_root) / "registry.global.json" + global_registry = json.loads(global_path.read_text()) + except Exception: + return False + repo = None + for entry in global_registry.get("goals", []) or []: + if str(entry.get("id") or "") == goal_id: + repo = entry.get("repo") + break + if not repo: + return False + try: + from ...registry import sync_registry_goal_closed + + return sync_registry_goal_closed(Path(repo) / ".loopx" / "registry.json", goal_id) + except Exception: + return False + + +def finalize_resident_execution( + *, + runtime_root: Path, + goal_id: str, + event_log_path: Path | None, + executed: Sequence[Mapping[str, Any]], + worker_id: str | None = None, + recorded_at: str | None = None, + acceptance_criteria: Sequence[Mapping[str, Any]] | None = None, + evidence: Sequence[Mapping[str, Any]] | None = None, + acceptance_base_dir: Path | None = None, +) -> dict[str, Any]: + """Close the resident execution loop (task_completed -> acceptance -> closure). + + This is the missing link that turns a claimed+executed task into a fully + closed goal lifecycle: + + 1. For each executed task, mark it ``done`` (``complete_task``) and emit a + ``task_completed`` audit event; failed executions emit ``task_failed``. + 2. Run the Goal Acceptance evaluator against the declared criteria/evidence. + 3. Run the Closure Evaluator; if evidence is sufficient and no work remains, + emit ``goal_closure_ready`` + ``goal_closed``. If acceptance has gaps, + emit ``goal_acceptance_pending`` (goal stays open). + + Returns a summary with per-task results, the acceptance evaluation, and the + closure evaluation (``closed`` tells whether the goal was closed). + """ + from ...rollout_event_log import append_rollout_event_once, build_rollout_event + from ..goals.goal_acceptance import evaluate_goal_acceptance + from ..goals.goal_closure import ( + build_goal_closure_state, + maybe_close_goal, + ) + from .event_driven_dispatch import load_task_queue, task_queue_path as _tqp + from .task_lifecycle import complete_task, fail_task + + stamp = recorded_at or now_utc_iso() + queue_path = _tqp(runtime_root, goal_id=goal_id) + log_path = Path(event_log_path) if event_log_path is not None else rollout_event_log_path( + runtime_root, goal_id + ) + + task_results: list[dict[str, Any]] = [] + for entry in executed: + todo_id = str(entry.get("todo_id") or "").strip() + executed_ok = entry.get("executed") is True or entry.get("ok") is True + if not todo_id: + continue + if executed_ok: + completed = complete_task( + queue_path, + task_id=todo_id, + worker_id=worker_id or str(entry.get("claimed_by") or "").strip(), + recorded_at=stamp, + ) + event = build_rollout_event( + goal_id=goal_id, + event_kind="task_completed", + agent_id=worker_id, + recorded_at=stamp, + ) + event["todo_id"] = todo_id + append_rollout_event_once( + log_path, event, identity_fields=("goal_id", "event_kind", "todo_id") + ) + task_results.append( + {"todo_id": todo_id, "executed": True, "completed": completed is not None} + ) + else: + fail_task( + queue_path, + task_id=todo_id, + worker_id=worker_id or str(entry.get("claimed_by") or "").strip(), + error=str(entry.get("reason") or "execution_failed"), + transient=False, + recorded_at=stamp, + ) + event = build_rollout_event( + goal_id=goal_id, + event_kind="task_failed", + agent_id=worker_id, + recorded_at=stamp, + ) + event["todo_id"] = todo_id + append_rollout_event_once( + log_path, event, identity_fields=("goal_id", "event_kind", "todo_id") + ) + task_results.append({"todo_id": todo_id, "executed": False, "completed": False}) + + # Acceptance evaluation (evidence verification before closure). + acceptance = evaluate_goal_acceptance( + acceptance_criteria=acceptance_criteria, + evidence=evidence, + base_dir=acceptance_base_dir, + ) + + # Closure evaluation: acceptance + no-work -> close. + queue_view = load_task_queue(queue_path) + state = build_goal_closure_state( + ready_todo_ids=list(queue_view.get("pending_todo_ids", [])), + open_todo_count=queue_view.get("pending_count", 0), + claimed_advancement_count=queue_view.get("claimed_count", 0), + acceptance=acceptance, + ) + closure = maybe_close_goal(log_path=log_path, goal_id=goal_id, state=state) + + # When closure derived (goal_closed emitted), keep the registry goal entry's + # status in lockstep with the rollout log so `status`/registry and + # start-goal's guided packet agree the goal is closed. The resident loop only + # knows ``runtime_root``, so resolve the project registry via the global + # registry's ``repo`` field (best-effort; a missing/partial registry is a + # silent no-op). + if closure.get("closed") is True: + try: + from ...registry import sync_registry_goal_closed + + _sync_registry_goal_closed_from_runtime(runtime_root, goal_id) + except Exception: + pass + + return { + "ok": True, + "goal_id": goal_id, + "task_results": task_results, + "acceptance": acceptance, + "closure": closure, + "closed": closure.get("closed") is True, + "queue": queue_view, + } + + +class WorkerPool: + """A bounded set of workers claiming tasks from the scheduler queue. + + ``acquire()`` claims the oldest pending task for a worker (Worker Pool + acquire via ``claim_next_task``), rewriting the queue in place. When the + queue is empty it returns ``None``. Claiming is opt-in behind the shared + event-driven flag. + """ + + def __init__( + self, + *, + worker_ids: Sequence[str] = (), + runtime_root: Path, + goal_id: str, + use_event_driven: bool | None = None, + capabilities_map: Mapping[str, Sequence[str]] | None = None, + lease_seconds: int | float | None = None, + ) -> None: + self._worker_ids = [str(w).strip() for w in worker_ids if str(w).strip()] + self._runtime_root = Path(runtime_root) + self._goal_id = str(goal_id or "").strip() + self._use_event_driven = use_event_driven + self._capabilities_map = { + str(worker): [str(c) for c in caps] + for worker, caps in (capabilities_map or {}).items() + } + self._lease_seconds = lease_seconds + self._acquired: list[dict[str, Any]] = [] + + @property + def worker_ids(self) -> list[str]: + return list(self._worker_ids) + + @property + def acquired(self) -> list[dict[str, Any]]: + return [dict(entry) for entry in self._acquired] + + @property + def idle_worker_count(self) -> int: + claimed = {str(e.get("claimed_by") or "") for e in self._acquired} + return len([w for w in self._worker_ids if w not in claimed]) + + def worker_capabilities(self, worker_id: str) -> list[str] | None: + """Return the declared capabilities for a worker (None when undeclared).""" + caps = self._capabilities_map.get(str(worker_id).strip()) + return list(caps) if caps is not None else None + + def acquire(self, worker_id: str) -> dict[str, Any] | None: + """Claim the next pending task for ``worker_id``, or None when empty. + + When capabilities are declared for the worker, only tasks whose required + capabilities are satisfied are claimable (capability matching, P1). When + ``lease_seconds`` is set, the claimed entry carries a ``lease_until`` so a + crashed worker's task can be reclaimed by reconciliation. + """ + if not event_driven_dispatch_enabled(self._use_event_driven): + return None + claimed = claim_next_task( + task_queue_path(self._runtime_root, goal_id=self._goal_id), + worker_id=str(worker_id).strip(), + use_event_driven=self._use_event_driven, + capabilities=self.worker_capabilities(worker_id), + lease_seconds=self._lease_seconds, + ) + if claimed is not None: + self._acquired.append(dict(claimed)) + return claimed + + def drain( + self, + *, + worker_ids: Sequence[str] | None = None, + limit: int | None = None, + capabilities_map: Mapping[str, Sequence[str]] | None = None, + lease_seconds: int | float | None = None, + ) -> list[dict[str, Any]]: + """Claim tasks for each idle worker until the queue is empty or ``limit``. + + ``capabilities_map`` / ``lease_seconds`` (optional) let a caller override + the pool-level capability and lease settings for a single drain pass. + """ + caps_map = { + str(worker): [str(c) for c in caps] + for worker, caps in (capabilities_map or {}).items() + } or self._capabilities_map + lease = lease_seconds if lease_seconds is not None else self._lease_seconds + pool = [str(w).strip() for w in (worker_ids if worker_ids is not None else self._worker_ids)] + acquired: list[dict[str, Any]] = [] + claimed = {str(e.get("claimed_by") or "") for e in self._acquired} + for worker in pool: + if limit is not None and len(acquired) >= limit: + break + if worker in claimed: + continue + caps = caps_map.get(worker) + if not event_driven_dispatch_enabled(self._use_event_driven): + entry = None + else: + entry = claim_next_task( + task_queue_path(self._runtime_root, goal_id=self._goal_id), + worker_id=str(worker).strip(), + use_event_driven=self._use_event_driven, + capabilities=caps, + lease_seconds=lease, + ) + if entry is not None: + self._acquired.append(dict(entry)) + acquired.append(entry) + claimed.add(worker) + else: + # A worker with no eligible task does not stop the drain for the + # other workers; only stop when the queue is genuinely exhausted. + if caps is None and self._queue_is_empty(): + break + return acquired + + def _queue_is_empty(self) -> bool: + from .event_driven_dispatch import load_task_queue + + view = load_task_queue(task_queue_path(self._runtime_root, goal_id=self._goal_id)) + return view.get("pending_count", 0) == 0 + + +class ResidentScheduler: + """A resident scheduler process: Task Queue + Worker Pool, policy-gated. + + One ``tick()`` advances the event-driven narrow path: + + TaskCompleted -> dependency satisfied -> TaskReady -> Queue -> Worker + + The scheduler stays business-agnostic: readiness comes from handoff gates + (pure projection) and the decision to run is delegated to PolicyEngine by + the caller via ``build_event_driven_dispatch``. This class only manages + queue advancement and worker acquisition. + """ + + def __init__( + self, + *, + runtime_root: Path, + goal_id: str, + worker_ids: Sequence[str] = (), + agent_id: str | None = None, + event_log_path: Path | None = None, + use_event_driven: bool | None = None, + reconcile: bool = True, + worker_capabilities: Mapping[str, Sequence[str]] | None = None, + lease_seconds: int | float | None = None, + ) -> None: + self._runtime_root = Path(runtime_root) + self._goal_id = str(goal_id or "").strip() + self._agent_id = str(agent_id or "").strip() or None + self._event_log_path = ( + Path(event_log_path) + if event_log_path is not None + else rollout_event_log_path(self._runtime_root, self._goal_id) + ) + self._pool = WorkerPool( + worker_ids=worker_ids, + runtime_root=self._runtime_root, + goal_id=self._goal_id, + use_event_driven=use_event_driven, + ) + self._use_event_driven = use_event_driven + self._reconcile = bool(reconcile) + self._worker_capabilities = { + str(worker): [str(c) for c in caps] + for worker, caps in (worker_capabilities or {}).items() + } + self._lease_seconds = lease_seconds + self._tick_count = 0 + + @property + def goal_id(self) -> str: + return self._goal_id + + @property + def tick_count(self) -> int: + return self._tick_count + + @property + def pool(self) -> WorkerPool: + return self._pool + + def tick( + self, + *, + completed_todo_id: str | None = None, + recorded_at: str | None = None, + claim_workers: bool = True, + worker_exec: Callable[[Mapping[str, Any]], dict[str, Any]] | None = None, + ) -> dict[str, Any]: + """Advance READY successors, enqueue them, and claim for idle workers. + + ``worker_exec`` is an optional per-claimed-task executor hook invoked + after a worker claims a task (e.g. ``execute_claimed_task``). It runs + only when supplied; otherwise claimed tasks are not executed (the + scheduler stays business-agnostic). + """ + self._tick_count += 1 + stamp = recorded_at or now_utc_iso() + items, _ = _loaded_items(self._runtime_root, self._goal_id) + # Claimer capabilities for the primary dispatch worker (if declared). + claim_worker_caps = None + primary_worker = self._pool.worker_ids[0] if self._pool.worker_ids else None + if primary_worker and primary_worker in self._worker_capabilities: + claim_worker_caps = self._worker_capabilities[primary_worker] + payload = build_event_driven_dispatch( + runtime_root=self._runtime_root, + goal_id=self._goal_id, + items=items, + completed_todo_id=completed_todo_id, + event_log_path=self._event_log_path, + worker_id=primary_worker, + agent_id=self._agent_id, + recorded_at=stamp, + use_event_driven=self._use_event_driven, + reconcile=self._reconcile, + worker_capabilities=claim_worker_caps, + lease_seconds=self._lease_seconds, + ) + acquired: list[dict[str, Any]] = [] + if claim_workers: + acquired = self._pool.drain(capabilities_map=self._worker_capabilities, lease_seconds=self._lease_seconds) + # Worker execution targets: the task the dispatch claimed this tick + # (``dispatched``), plus any additional tasks drained by the pool. + dispatch_summary = payload.get("event_driven_dispatch") or {} + dispatched = dispatch_summary.get("dispatched") + exec_targets: list[dict[str, Any]] = [] + if isinstance(dispatched, dict) and dispatched.get("todo_id"): + exec_targets.append(dispatched) + exec_target_ids = {str(e.get("todo_id")) for e in exec_targets} + for entry in acquired: + if str(entry.get("todo_id")) not in exec_target_ids: + exec_targets.append(entry) + exec_target_ids.add(str(entry.get("todo_id"))) + executed: list[dict[str, Any]] = [] + if worker_exec is not None: + for entry in exec_targets: + executed.append(worker_exec(entry)) + payload["resident_scheduler"] = { + "schema_version": RESIDENT_SCHEDULER_SCHEMA_VERSION, + "goal_id": self._goal_id, + "tick_count": self._tick_count, + "reconcile": ((payload.get("event_driven_dispatch") or {}).get("reconcile")), + "worker_pool": { + "worker_ids": self._pool.worker_ids, + "idle_worker_count": self._pool.idle_worker_count, + "acquired": acquired, + }, + "worker_executions": executed, + } + return payload + + +def run_resident_scheduler_bounded( + *, + runtime_root: Path, + goal_id: str, + worker_ids: Sequence[str] = (), + agent_id: str | None = None, + max_iterations: int = 1, + interval_seconds: float = 0.0, + completed_todo_id: str | None = None, + use_event_driven: bool | None = None, + sleep: Callable[[float], None] = time.sleep, + worker_exec_command: str | None = None, + worker_exec_command_prefixes: Sequence[str] | None = None, + guard_checked: bool = False, + runner: WorkerTaskRunner | None = None, + reconcile: bool = True, + worker_capabilities: Mapping[str, Sequence[str]] | None = None, + lease_seconds: int | float | None = None, + acceptance_criteria: Sequence[Mapping[str, Any]] | None = None, + evidence: Sequence[Mapping[str, Any]] | None = None, + acceptance_base_dir: Path | None = None, +) -> dict[str, Any]: + """Run the resident scheduler for a bounded number of ticks (test / cron). + + Returns an aggregate summary with per-tick results and a final queue view. + + ``agent_id`` names the registered LoopX agent identity recorded on + ``task_dispatched`` audit events (falls back to the claimer when omitted). + + Optional worker execution (mirrors the original scheduler executor gates): + when ``worker_exec_command`` + ``worker_exec_command_prefixes`` are supplied + and ``guard_checked`` is True, each claimed task is executed after claim via + :func:`execute_claimed_task`. This is opt-in; without it tasks are only + claimed, never executed. + + ``reconcile`` (default True) enables lease-expiry (zombie recovery) and + retry promotion each tick; ``worker_capabilities`` enables capability-matched + claiming; ``lease_seconds`` bounds how long a claim is held. + + ``acceptance_criteria`` + ``evidence`` (optional) close the full loop: after + worker execution, tasks are completed, acceptance is verified, and the goal + is closed (or held pending) via :func:`finalize_resident_execution`. The + final summary includes ``finalize`` (task_results / acceptance / closure). + """ + scheduler = ResidentScheduler( + runtime_root=runtime_root, + goal_id=goal_id, + worker_ids=worker_ids, + agent_id=agent_id, + use_event_driven=use_event_driven, + reconcile=reconcile, + worker_capabilities=worker_capabilities, + lease_seconds=lease_seconds, + ) + iterations = max(0, int(max_iterations)) + + def _worker_exec(claimed: Mapping[str, Any]) -> dict[str, Any]: + return execute_claimed_task( + claimed, + worker_command=worker_exec_command, + worker_command_prefixes=worker_exec_command_prefixes, + guard_checked=guard_checked, + runner=runner, + ) + + ticks: list[dict[str, Any]] = [] + for _ in range(iterations): + tick_payload = scheduler.tick( + completed_todo_id=completed_todo_id, + worker_exec=_worker_exec if worker_exec_command else None, + ) + ticks.append(tick_payload) + if interval_seconds > 0 and _ < iterations - 1: + sleep(interval_seconds) + from .event_driven_dispatch import load_task_queue + + # Close the loop: task_completed -> acceptance -> closure (goal_closed). + finalize: dict[str, Any] | None = None + if worker_exec_command and (acceptance_criteria is not None or evidence is not None): + executed_entries: list[dict[str, Any]] = [] + for tick in ticks: + executed_entries.extend( + (tick.get("resident_scheduler") or {}).get("worker_executions") or [] + ) + finalize = finalize_resident_execution( + runtime_root=runtime_root, + goal_id=goal_id, + event_log_path=rollout_event_log_path(runtime_root, goal_id), + executed=executed_entries, + worker_id=agent_id, + acceptance_criteria=acceptance_criteria, + evidence=evidence, + acceptance_base_dir=acceptance_base_dir, + ) + + return { + "ok": True, + "goal_id": goal_id, + "schema_version": RESIDENT_SCHEDULER_SCHEMA_VERSION, + "enabled": event_driven_dispatch_enabled(use_event_driven), + "max_iterations": iterations, + "tick_count": scheduler.tick_count, + "worker_exec_command": worker_exec_command, + "ticks": ticks, + "queue": load_task_queue(task_queue_path(runtime_root, goal_id=goal_id)), + "finalize": finalize, + } + + +def run_resident_scheduler_loop( + *, + runtime_root: Path, + goal_id: str, + worker_ids: Sequence[str] = (), + agent_id: str | None = None, + interval_seconds: float = 10.0, + completed_todo_id: str | None = None, + use_event_driven: bool | None = None, + sleep: Callable[[float], None] = time.sleep, +) -> dict[str, Any]: + """Run the resident scheduler as a long-lived polling loop. + + Intended for a launchd / systemd / tmux resident worker. The loop keeps + running until interrupted (``KeyboardInterrupt``); the interval applies + between ticks. Use ``run_resident_scheduler_bounded`` for bounded runs. + + ``agent_id`` names the registered LoopX agent identity recorded on + ``task_dispatched`` audit events (falls back to the claimer when omitted). + """ + scheduler = ResidentScheduler( + runtime_root=runtime_root, + goal_id=goal_id, + worker_ids=worker_ids, + agent_id=agent_id, + use_event_driven=use_event_driven, + ) + interval = max(0.0, float(interval_seconds)) + started_at = now_utc_iso() + ticks: list[dict[str, Any]] = [] + try: + while True: + tick_payload = scheduler.tick(completed_todo_id=completed_todo_id) + ticks.append(tick_payload) + if interval > 0: + sleep(interval) + except KeyboardInterrupt: + pass + from .event_driven_dispatch import load_task_queue + + return { + "ok": True, + "goal_id": goal_id, + "schema_version": RESIDENT_SCHEDULER_SCHEMA_VERSION, + "enabled": event_driven_dispatch_enabled(use_event_driven), + "started_at": started_at, + "ended_at": now_utc_iso(), + "tick_count": scheduler.tick_count, + "tick_summaries": [ + { + "tick": index + 1, + "disabled": payload.get("disabled") is True, + "newly_enqueued": ( + (payload.get("event_driven_dispatch") or {}).get("newly_enqueued", []) + if isinstance(payload.get("event_driven_dispatch"), dict) + else [] + ), + } + for index, payload in enumerate(ticks) + ], + "queue": load_task_queue(task_queue_path(runtime_root, goal_id=goal_id)), + } + + +__all__ = [ + "RESIDENT_SCHEDULER_SCHEMA_VERSION", + "WorkerPool", + "ResidentScheduler", + "execute_claimed_task", + "run_resident_scheduler_bounded", + "run_resident_scheduler_loop", +] diff --git a/loopx/control_plane/scheduler/task_lifecycle.py b/loopx/control_plane/scheduler/task_lifecycle.py new file mode 100644 index 000000000..11fbe3439 --- /dev/null +++ b/loopx/control_plane/scheduler/task_lifecycle.py @@ -0,0 +1,677 @@ +"""Task lifecycle: lease, retry, idempotency and capability matching. + +This module closes the automation loop for the resident scheduler by giving the +append-only task queue a real lifecycle, per ``plan/new_plan.md`` P0/P1: + +* **Lease** — a claimed task carries a ``lease_until`` timestamp and an + ``attempt`` counter. A worker that crashes or is killed after ``claimed`` + becomes a *zombie*; the scheduler expires its lease and re-enqueues the task + (``claimed -> expired -> pending``) so it is not stuck forever. +* **Retry** — a failed task transitions ``claimed -> failed -> retry_wait`` + (with a backoff window) and then ``retry_wait -> pending`` when the retry + delay elapses, up to ``max_attempts``; exhausting attempts leaves the task in + ``failed`` (promotable to ``dead_letter`` via :func:`dead_letter_exhausted` + for explicit operator attention). +* **Idempotency** — tasks are keyed by a ``task_id`` that encodes the todo and a + ``generation`` (``todo_id:generation:N``). A duplicate event or a re-execution + of the same todo at a different generation never collides with the same + logical work item, while within one generation enqueue stays idempotent. +* **Capability matching** — ``eligible(worker, task)`` matches a worker's + declared capabilities against a task's ``required_capabilities``, so a claim + prefers the oldest pending task a given worker is actually able to run. + +Every mutation is idempotent and file-backed; the store remains an append-only +JSONL (rewritten in place for status transitions, same as ``claim_next_task``). +All entry statuses are backward compatible with the existing +``pending`` / ``claimed`` / ``done`` lifecycle. +""" + +from __future__ import annotations + +import json +import time +from pathlib import Path +from typing import Any, Iterable, Mapping, Sequence + +from ..runtime.time import now_utc_iso +from .event_driven_dispatch import ( + QUEUE_STATUS_CLAIMED, + QUEUE_STATUS_DONE, + QUEUE_STATUS_PENDING, + QUEUE_STATUSES, + TASK_QUEUE_ENTRY_SCHEMA_VERSION, + load_task_queue, +) + +# Extended lifecycle states (superset of pending/claimed/done). +QUEUE_STATUS_RUNNING = "running" +QUEUE_STATUS_EXPIRED = "expired" +QUEUE_STATUS_FAILED = "failed" +QUEUE_STATUS_RETRY_WAIT = "retry_wait" +QUEUE_STATUS_CANCELLED = "cancelled" +QUEUE_STATUS_DEAD_LETTER = "dead_letter" + +EXTENDED_QUEUE_STATUSES = QUEUE_STATUSES | { + QUEUE_STATUS_RUNNING, + QUEUE_STATUS_EXPIRED, + QUEUE_STATUS_FAILED, + QUEUE_STATUS_RETRY_WAIT, + QUEUE_STATUS_CANCELLED, + QUEUE_STATUS_DEAD_LETTER, +} + +# Statuses that still occupy their logical todo (idempotency domain). +_ACTIVE_STATUSES = { + QUEUE_STATUS_PENDING, + QUEUE_STATUS_CLAIMED, + QUEUE_STATUS_RUNNING, + QUEUE_STATUS_RETRY_WAIT, +} + +DEFAULT_LEASE_SECONDS = 600 +DEFAULT_MAX_ATTEMPTS = 3 +DEFAULT_RETRY_BACKOFF_SECONDS = 30 + +TASK_ID_SEPARATOR = ":generation:" + + +# --------------------------------------------------------------------------- +# Idempotency: generation-aware task id +# --------------------------------------------------------------------------- + + +def build_task_id(todo_id: str, generation: int = 0) -> str: + """Return a generation-aware task id: ``todo_id:generation:N``.""" + normalized = str(todo_id or "").strip() + try: + gen = int(generation) + except (TypeError, ValueError): + gen = 0 + return f"{normalized}{TASK_ID_SEPARATOR}{max(0, gen)}" + + +def parse_task_id(task_id: str) -> tuple[str, int] | None: + """Split a task id back into ``(todo_id, generation)`` or None.""" + text = str(task_id or "").strip() + if not text or TASK_ID_SEPARATOR not in text: + return None + todo_id, _, raw_gen = text.rpartition(TASK_ID_SEPARATOR) + try: + generation = int(raw_gen) + except ValueError: + return None + return todo_id, generation + + +def task_generation(task_id: str | None) -> int: + """Extract the generation from a task id (0 when absent/undeterminable).""" + parsed = parse_task_id(task_id or "") + return parsed[1] if parsed else 0 + + +# --------------------------------------------------------------------------- +# Worker capability matching +# --------------------------------------------------------------------------- + + +def normalize_worker_capabilities(capabilities: Sequence[str] | None) -> set[str]: + """Normalize a worker's declared capabilities to a lowercase token set. + + Accepts a list, tuple, or a comma/space-separated string. + """ + result: set[str] = set() + values: Sequence[str] + if isinstance(capabilities, str): + values = (capabilities,) + else: + values = capabilities or () + for value in values: + for token in str(value or "").replace(",", " ").split(): + token = token.strip().lower() + if token: + result.add(token) + return result + + +def normalize_required_capabilities(value: Any) -> list[str]: + """Normalize a task's required capabilities to a lowercase token list.""" + tokens: list[str] = [] + if isinstance(value, str): + raw = [value] + elif isinstance(value, (list, tuple)): + raw = value + else: + return tokens + for item in raw: + for token in str(item or "").replace(",", " ").split(): + token = token.strip().lower() + if token and token not in tokens: + tokens.append(token) + return tokens + + +def task_required_capabilities(entry: Mapping[str, Any]) -> list[str]: + """Read the normalized required capabilities from a task entry.""" + return normalize_required_capabilities(entry.get("required_capabilities")) + + +def eligible( + worker: Mapping[str, Any], + task: Mapping[str, Any], + *, + registry: Any = None, +) -> bool: + """Whether ``worker`` may run ``task`` based on required capabilities. + + A task with no required capabilities is eligible for any worker. A task with + required capabilities is eligible only for a worker that declares all of + them. Workers declare capabilities via ``worker["capabilities"]`` (list or + comma-separated string) or the ``capabilities`` keyword on claim. + + When ``registry`` (a ``CapabilityRegistry``) is supplied, a task that carries + a ``capability_binding_ref`` (``:``) is additionally + gated on that capability pack being ``ready`` in its provider lifecycle and + on the worker declaring the pack id as a capability token. This is the + bridge to the legacy capability-pack system; without a registry the function + is identical to the original token-only matching. + """ + binding = task.get("capability_binding_ref") + if registry is not None or binding: + from ..capabilities_bridge import eligible_bridged + + return eligible_bridged(worker, task, registry=registry) + required = task_required_capabilities(task) + if not required: + return True + worker_caps = normalize_worker_capabilities(worker.get("capabilities")) + return all(cap in worker_caps for cap in required) + + +def worker_satisfies_capabilities( + capabilities: Sequence[str] | None, + required_capabilities: Sequence[str] | None, +) -> bool: + """Convenience form of :func:`eligible` given raw capability lists.""" + worker_caps = normalize_worker_capabilities(capabilities) + required = normalize_required_capabilities(required_capabilities) + return all(cap in worker_caps for cap in required) + + +# --------------------------------------------------------------------------- +# File helpers +# --------------------------------------------------------------------------- + + +def _read_entries(path: Path) -> list[dict[str, Any]]: + """Read all valid queue entries from a JSONL path (empty when absent).""" + try: + lines = path.read_text(encoding="utf-8").splitlines() + except OSError: + return [] + entries: list[dict[str, Any]] = [] + for line in lines: + if not line.strip(): + continue + try: + parsed = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(parsed, dict) and parsed.get("schema_version") == TASK_QUEUE_ENTRY_SCHEMA_VERSION: + entries.append(parsed) + return entries + + +def _write_entries(path: Path, entries: Iterable[Mapping[str, Any]]) -> None: + """Atomically rewrite the queue JSONL from an entry list.""" + path.parent.mkdir(parents=True, exist_ok=True) + tmp_path = path.with_suffix(path.suffix + ".tmp") + tmp_path.write_text( + "".join(json.dumps(dict(e), sort_keys=True, ensure_ascii=False) + "\n" for e in entries), + encoding="utf-8", + ) + tmp_path.replace(path) + + +# --------------------------------------------------------------------------- +# Lease +# --------------------------------------------------------------------------- + + +def lease_expiry( + lease_seconds: int | float | None = None, + *, + recorded_at: str | None = None, + now: float | None = None, +) -> float: + """Compute the absolute epoch-second lease expiry for a claimed task.""" + lease = DEFAULT_LEASE_SECONDS if lease_seconds is None else max(1, float(lease_seconds)) + base = now if now is not None else time.time() + return base + lease + + +def is_expired(entry: Mapping[str, Any], *, now: float | None = None) -> bool: + """Whether a claimed/running task has outlived its lease. + + A claimed/running task **without a ``lease_until``** is treated as an expired + zombie: it was claimed by older code (or a crash before the lease was + written) and there is no active worker holding it, so it must be reclaimed. + This recovers legacy ``claimed`` entries (e.g. ``claimed: 4`` stale tasks in + the website1 session) that would otherwise be stuck forever. + """ + status = str(entry.get("status") or "") + if status not in {QUEUE_STATUS_CLAIMED, QUEUE_STATUS_RUNNING}: + return False + raw = entry.get("lease_until") + if raw is None: + return True # claimed/running without a lease = zombie + try: + lease_until = float(raw) + except (TypeError, ValueError): + return True + current = now if now is not None else time.time() + return current >= lease_until + + +def expire_stale_leases( + path: Path, + *, + worker_id: str | None = None, + recorded_at: str | None = None, + now: float | None = None, +) -> list[dict[str, Any]]: + """Re-enqueue claimed/running tasks whose lease has expired (zombie recovery). + + Transitions ``claimed|running -> expired -> pending`` so a crashed worker's + task is reclaimed. Returns the list of re-enqueued (expired) entries. + """ + entries = _read_entries(path) + reenqueued: list[dict[str, Any]] = [] + changed = False + stamp = recorded_at or now_utc_iso() + for entry in entries: + if is_expired(entry, now=now): + entry["status"] = QUEUE_STATUS_PENDING + entry["lease_until"] = None + entry["expired_at"] = stamp + entry["expired_by"] = worker_id + entry.pop("claimed_by", None) + entry.pop("claimed_at", None) + reenqueued.append(dict(entry)) + changed = True + if changed: + _write_entries(path, entries) + return reenqueued + + +# --------------------------------------------------------------------------- +# Claim with capability matching + lease +# --------------------------------------------------------------------------- + + +def claim_next_eligible_task( + path: Path, + *, + worker_id: str, + capabilities: Sequence[str] | None = None, + lease_seconds: int | float | None = None, + recorded_at: str | None = None, + now: float | None = None, + registry: Any = None, +) -> dict[str, Any] | None: + """Claim the oldest pending task the worker can run, with a fresh lease. + + Returns the claimed entry (with ``claimed_by``, ``claimed_at``, + ``lease_until``, ``attempt``) or None when no eligible task is available. + + When ``registry`` is supplied, capability-pack bindings + (``capability_binding_ref``) are resolved through the registry (see + :func:`eligible`). + """ + entries = _read_entries(path) + claimed: dict[str, Any] | None = None + for entry in entries: + if entry.get("status") != QUEUE_STATUS_PENDING: + continue + if not eligible({"capabilities": capabilities}, entry, registry=registry): + continue + entry["status"] = QUEUE_STATUS_CLAIMED + entry["claimed_by"] = str(worker_id).strip() + entry["claimed_at"] = recorded_at or now_utc_iso() + entry["lease_until"] = lease_expiry( + lease_seconds, recorded_at=recorded_at, now=now + ) + try: + entry["attempt"] = int(entry.get("attempt") or 0) + 1 + except (TypeError, ValueError): + entry["attempt"] = 1 + claimed = entry + break + if claimed is None: + return None + _write_entries(path, entries) + return claimed + + +# --------------------------------------------------------------------------- +# Completion / failure / retry / dead-letter +# --------------------------------------------------------------------------- + + +def complete_task( + path: Path, + *, + task_id: str, + worker_id: str | None = None, + recorded_at: str | None = None, +) -> dict[str, Any] | None: + """Transition a claimed/running task to ``done`` (idempotent). + + Returns the updated entry, or None if no matching task was found. A task is + matched by ``task_id`` (falling back to ``todo_id`` when the task_id has no + generation marker) and the claiming worker. + """ + entries = _read_entries(path) + stamp = recorded_at or now_utc_iso() + target: dict[str, Any] | None = None + for entry in entries: + if _entry_matches(entry, task_id) and entry.get("status") in { + QUEUE_STATUS_CLAIMED, + QUEUE_STATUS_RUNNING, + }: + if worker_id and str(entry.get("claimed_by") or "") not in { + "", + str(worker_id), + }: + continue + target = entry + break + if target is None: + return None + target["status"] = QUEUE_STATUS_DONE + target["completed_at"] = stamp + target["lease_until"] = None + target["completed_by"] = worker_id + _write_entries(path, entries) + return target + + +def _entry_matches(entry: Mapping[str, Any], task_id: str) -> bool: + """Match an entry by task_id, or by todo_id when the task_id is plain.""" + text = str(task_id or "").strip() + if not text: + return False + if str(entry.get("task_id") or "") == text: + return True + if TASK_ID_SEPARATOR not in text and str(entry.get("todo_id") or "") == text: + return True + return False + + +def fail_task( + path: Path, + *, + task_id: str, + worker_id: str | None = None, + error: str | None = None, + transient: bool = True, + max_attempts: int | None = None, + retry_backoff_seconds: int | float | None = None, + recorded_at: str | None = None, + now: float | None = None, +) -> dict[str, Any] | None: + """Transition a claimed/running task on failure. + + * transient failure with attempts remaining -> ``retry_wait`` (with a + ``retry_at`` backoff window so it is not immediately re-claimed); + * permanent failure or retries exhausted -> ``failed``. + """ + entries = _read_entries(path) + stamp = recorded_at or now_utc_iso() + target: dict[str, Any] | None = None + for entry in entries: + if _entry_matches(entry, task_id) and entry.get("status") in { + QUEUE_STATUS_CLAIMED, + QUEUE_STATUS_RUNNING, + }: + if worker_id and str(entry.get("claimed_by") or "") not in { + "", + str(worker_id), + }: + continue + target = entry + break + if target is None: + return None + try: + attempt = int(target.get("attempt") or 1) + except (TypeError, ValueError): + attempt = 1 + max_attempts_value = DEFAULT_MAX_ATTEMPTS if max_attempts is None else int(max_attempts) + retry_allowed = transient and attempt < max(1, max_attempts_value) + target["failed_at"] = stamp + target["last_error"] = str(error or "").strip() or None + target["lease_until"] = None + if retry_allowed: + backoff = ( + DEFAULT_RETRY_BACKOFF_SECONDS + if retry_backoff_seconds is None + else max(0, float(retry_backoff_seconds)) + ) + retry_at = (now if now is not None else time.time()) + backoff + target["status"] = QUEUE_STATUS_RETRY_WAIT + target["retry_at"] = retry_at + target["retry_count"] = attempt + else: + target["status"] = QUEUE_STATUS_FAILED + target.pop("retry_at", None) + _write_entries(path, entries) + return target + + +def promote_retry_ready( + path: Path, + *, + recorded_at: str | None = None, + now: float | None = None, +) -> list[dict[str, Any]]: + """Move ``retry_wait`` tasks whose backoff has elapsed back to ``pending``.""" + entries = _read_entries(path) + current = now if now is not None else time.time() + stamp = recorded_at or now_utc_iso() + promoted: list[dict[str, Any]] = [] + changed = False + for entry in entries: + if entry.get("status") != QUEUE_STATUS_RETRY_WAIT: + continue + retry_at = entry.get("retry_at") + if retry_at is None: + entry["status"] = QUEUE_STATUS_PENDING + entry.pop("retry_at", None) + entry["retry_promoted_at"] = stamp + promoted.append(dict(entry)) + changed = True + continue + try: + when = float(retry_at) + except (TypeError, ValueError): + when = 0.0 + if current >= when: + entry["status"] = QUEUE_STATUS_PENDING + entry.pop("retry_at", None) + entry["retry_promoted_at"] = stamp + promoted.append(dict(entry)) + changed = True + if changed: + _write_entries(path, entries) + return promoted + + +def cancel_task( + path: Path, + *, + task_id: str, + reason: str | None = None, + recorded_at: str | None = None, +) -> dict[str, Any] | None: + """Cancel a pending/claimed/running/retry_wait task -> ``cancelled``.""" + entries = _read_entries(path) + stamp = recorded_at or now_utc_iso() + target: dict[str, Any] | None = None + for entry in entries: + if _entry_matches(entry, task_id) and entry.get("status") in { + QUEUE_STATUS_PENDING, + QUEUE_STATUS_CLAIMED, + QUEUE_STATUS_RUNNING, + QUEUE_STATUS_RETRY_WAIT, + }: + target = entry + break + if target is None: + return None + target["status"] = QUEUE_STATUS_CANCELLED + target["cancelled_at"] = stamp + target["cancel_reason"] = str(reason or "").strip() or None + target["lease_until"] = None + _write_entries(path, entries) + return target + + +def requeue_failed( + path: Path, + *, + task_id: str, + worker_id: str | None = None, + recorded_at: str | None = None, +) -> dict[str, Any] | None: + """Manually move a ``failed``/``dead_letter``/``cancelled`` task to ``pending``.""" + entries = _read_entries(path) + stamp = recorded_at or now_utc_iso() + target: dict[str, Any] | None = None + for entry in entries: + if _entry_matches(entry, task_id) and entry.get("status") in { + QUEUE_STATUS_FAILED, + QUEUE_STATUS_DEAD_LETTER, + QUEUE_STATUS_CANCELLED, + }: + target = entry + break + if target is None: + return None + target["status"] = QUEUE_STATUS_PENDING + target["requeued_at"] = stamp + target["requeued_by"] = worker_id + target.pop("retry_at", None) + target["lease_until"] = None + _write_entries(path, entries) + return target + + +def dead_letter_exhausted( + path: Path, + *, + task_id: str, + worker_id: str | None = None, + recorded_at: str | None = None, +) -> dict[str, Any] | None: + """Explicitly move a failed task to ``dead_letter`` (operator attention).""" + entries = _read_entries(path) + stamp = recorded_at or now_utc_iso() + target: dict[str, Any] | None = None + for entry in entries: + if _entry_matches(entry, task_id) and entry.get("status") in { + QUEUE_STATUS_FAILED, + QUEUE_STATUS_RETRY_WAIT, + }: + target = entry + break + if target is None: + return None + target["status"] = QUEUE_STATUS_DEAD_LETTER + target["dead_lettered_at"] = stamp + target["dead_lettered_by"] = worker_id + target["lease_until"] = None + target.pop("retry_at", None) + _write_entries(path, entries) + return target + + +# --------------------------------------------------------------------------- +# Views / reconciliation +# --------------------------------------------------------------------------- + + +def reconcile_queue( + path: Path, + *, + worker_id: str | None = None, + recorded_at: str | None = None, + now: float | None = None, +) -> dict[str, Any]: + """Run the idle-maintenance passes in order (zombie + retry promotion). + + Returns a summary of expired leases and retry promotions. Intended to be + called at the top of each resident scheduler tick so the queue stays clean + even when workers crash or tasks need retry. + """ + expired = expire_stale_leases(path, worker_id=worker_id, recorded_at=recorded_at, now=now) + promoted = promote_retry_ready(path, recorded_at=recorded_at, now=now) + return { + "ok": True, + "expired_leases": [e.get("todo_id") for e in expired], + "expired_count": len(expired), + "retry_promoted": [e.get("todo_id") for e in promoted], + "retry_promoted_count": len(promoted), + } + + +def extended_queue_view(path: Path) -> dict[str, Any]: + """A queue view that counts the extended lifecycle statuses.""" + view = load_task_queue(path) + entries = _read_entries(path) + counts: dict[str, int] = {} + for status in EXTENDED_QUEUE_STATUSES: + counts[status] = sum(1 for e in entries if e.get("status") == status) + view["extended"] = { + "running_count": counts[QUEUE_STATUS_RUNNING], + "expired_count": counts[QUEUE_STATUS_EXPIRED], + "failed_count": counts[QUEUE_STATUS_FAILED], + "retry_wait_count": counts[QUEUE_STATUS_RETRY_WAIT], + "cancelled_count": counts[QUEUE_STATUS_CANCELLED], + "dead_letter_count": counts[QUEUE_STATUS_DEAD_LETTER], + } + return view + + +__all__ = [ + "QUEUE_STATUS_RUNNING", + "QUEUE_STATUS_EXPIRED", + "QUEUE_STATUS_FAILED", + "QUEUE_STATUS_RETRY_WAIT", + "QUEUE_STATUS_CANCELLED", + "QUEUE_STATUS_DEAD_LETTER", + "EXTENDED_QUEUE_STATUSES", + "DEFAULT_LEASE_SECONDS", + "DEFAULT_MAX_ATTEMPTS", + "DEFAULT_RETRY_BACKOFF_SECONDS", + "TASK_ID_SEPARATOR", + "build_task_id", + "parse_task_id", + "task_generation", + "normalize_worker_capabilities", + "normalize_required_capabilities", + "task_required_capabilities", + "eligible", + "worker_satisfies_capabilities", + "lease_expiry", + "is_expired", + "expire_stale_leases", + "claim_next_eligible_task", + "complete_task", + "fail_task", + "promote_retry_ready", + "cancel_task", + "requeue_failed", + "dead_letter_exhausted", + "reconcile_queue", + "extended_queue_view", +] diff --git a/loopx/control_plane/status/control_plane_observability.py b/loopx/control_plane/status/control_plane_observability.py new file mode 100644 index 000000000..d27838c0b --- /dev/null +++ b/loopx/control_plane/status/control_plane_observability.py @@ -0,0 +1,295 @@ +"""Control-plane observability snapshot (plan/new_plan.md §7, P2). + +Builds a single operator/agent-visible status view of the event-driven control +plane: scheduler, worker pool, task queue (including the extended lifecycle +states), task history, decision history, and rollout event history. + +The snapshot is *read-only*: it never mutates state. It composes existing read +models (``load_task_queue``, ``extended_queue_view``, the rollout event log, and +the policy decision ledger) into one digest for debugging and monitoring. + +Nothing here is required for scheduling correctness; it is purely observational. +""" + +from __future__ import annotations + +import json +from collections import Counter +from pathlib import Path +from typing import Any, Mapping, Sequence + +from ...rollout_event_log import ( + load_rollout_events, + rollout_event_log_path, + summarize_rollout_events, +) +from ..scheduler.event_driven_dispatch import ( + QUEUE_STATUS_CLAIMED, + QUEUE_STATUS_DONE, + TASK_QUEUE_ENTRY_SCHEMA_VERSION, + task_queue_path, +) +from ..scheduler.task_lifecycle import extended_queue_view + + +def _read_queue_entries(path: Path) -> list[dict[str, Any]]: + """Read raw queue entries (empty when the file is absent or malformed).""" + try: + lines = path.read_text(encoding="utf-8").splitlines() + except OSError: + return [] + entries: list[dict[str, Any]] = [] + for line in lines: + if not line.strip(): + continue + try: + parsed = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(parsed, dict) and parsed.get("schema_version") == TASK_QUEUE_ENTRY_SCHEMA_VERSION: + entries.append(parsed) + return entries + +# Default limits for history digests (kept small so snapshots stay cheap). +DEFAULT_EVENT_HISTORY_LIMIT = 200 +DEFAULT_DECISION_HISTORY_LIMIT = 100 +DEFAULT_TASK_HISTORY_LIMIT = 200 + +# Queue statuses considered "in flight" (claimed or retrying). +_IN_FLIGHT_STATUSES = { + QUEUE_STATUS_CLAIMED, + "running", + "retry_wait", +} + +# Terminal / exception statuses of interest for an operator. +_EXCEPTION_STATUSES = {"failed", "dead_letter", "cancelled"} + + +def build_task_history( + entries: Sequence[Mapping[str, Any]], + *, + limit: int = DEFAULT_TASK_HISTORY_LIMIT, +) -> list[dict[str, Any]]: + """A compact chronological task history from queue entries (newest first). + + Each entry carries the todo, generation-aware task id, lifecycle status, + claimer, attempt count, and key timestamps. Oldest entries are truncated to + ``limit``. + """ + ordered = list(entries) + ordered.sort(key=lambda e: str(e.get("created_at") or "")) + selected = ordered[-max(0, limit):] + selected.reverse() + history: list[dict[str, Any]] = [] + for entry in selected: + history.append( + { + "todo_id": entry.get("todo_id"), + "task_id": entry.get("task_id"), + "status": entry.get("status"), + "claimed_by": entry.get("claimed_by"), + "attempt": entry.get("attempt"), + "required_capabilities": entry.get("required_capabilities"), + "created_at": entry.get("created_at"), + "claimed_at": entry.get("claimed_at"), + "completed_at": entry.get("completed_at"), + "failed_at": entry.get("failed_at"), + "last_error": entry.get("last_error"), + "retry_at": entry.get("retry_at"), + } + ) + return history + + +def build_worker_status(entries: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Worker status derived from queue entries: who holds which tasks. + + Reports per-worker in-flight claims and any capability declarations carried + on the claimed entry. Purely derived from the queue (no external registry + dependency). + """ + by_worker: dict[str, dict[str, Any]] = {} + for entry in entries: + status = str(entry.get("status") or "") + claimed_by = str(entry.get("claimed_by") or "") + if status not in _IN_FLIGHT_STATUSES or not claimed_by: + continue + worker = by_worker.setdefault( + claimed_by, + {"worker_id": claimed_by, "in_flight": [], "task_ids": []}, + ) + worker["in_flight"].append( + { + "todo_id": entry.get("todo_id"), + "status": status, + "attempt": entry.get("attempt"), + "required_capabilities": entry.get("required_capabilities"), + } + ) + worker["task_ids"].append(str(entry.get("todo_id") or "")) + workers = list(by_worker.values()) + for worker in workers: + worker["in_flight_count"] = len(worker["in_flight"]) + return {"worker_count": len(workers), "workers": workers} + + +def build_queue_digest(path: Path) -> dict[str, Any]: + """Queue status: standard view + extended lifecycle counts + in-flight tally.""" + view = extended_queue_view(path) + extended = view.get("extended") or {} + entries = _read_queue_entries(path) + done_todo_ids = [ + str(e.get("todo_id")) for e in entries if e.get("status") == QUEUE_STATUS_DONE + ] + return { + "ok": True, + "pending_count": view.get("pending_count", 0), + "claimed_count": view.get("claimed_count", 0), + "done_count": view.get("done_count", 0), + "pending_todo_ids": view.get("pending_todo_ids", []), + "claimed_todo_ids": view.get("claimed_todo_ids", []), + "done_todo_ids": done_todo_ids, + "extended": extended, + "in_flight_count": int(extended.get("running_count", 0)) + + view.get("claimed_count", 0) + + int(extended.get("retry_wait_count", 0)), + "exception_count": sum( + int(extended.get(k, 0)) + for k in ("failed_count", "dead_letter_count", "cancelled_count") + ), + } + + +def build_event_history( + log_path: Path, + *, + limit: int = DEFAULT_EVENT_HISTORY_LIMIT, +) -> dict[str, Any]: + """Rollout event history digest: recent events + per-kind counts.""" + events = load_rollout_events(log_path, limit=limit) + summary = summarize_rollout_events(events) + history = [] + for event in events[-limit:]: + history.append( + { + "event_id": event.get("event_id"), + "event_kind": event.get("event_kind"), + "goal_id": event.get("goal_id"), + "todo_id": event.get("todo_id"), + "recorded_at": event.get("recorded_at"), + } + ) + history.reverse() + return { + "event_count": len(events), + "counts_by_kind": summary.get("counts_by_kind", {}), + "recent_events": history, + } + + +def build_decision_history( + decision_event_log_path: Path, + *, + limit: int = DEFAULT_DECISION_HISTORY_LIMIT, +) -> dict[str, Any]: + """Policy decision history digest from the decision ledger. + + Falls back to an empty digest when the ledger path does not exist (decisions + are opt-in and may not be recorded). + """ + try: + lines = decision_event_log_path.read_text(encoding="utf-8").splitlines() + except OSError: + return {"ok": True, "decision_count": 0, "counts_by_outcome": {}, "recent_decisions": []} + import json + + decisions: list[dict[str, Any]] = [] + for line in lines: + if not line.strip(): + continue + try: + parsed = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(parsed, dict): + decisions.append(parsed) + counts = Counter(str(d.get("outcome") or d.get("decision") or "unknown") for d in decisions) + recent = decisions[-max(0, limit):] + recent.reverse() + compact = [] + for d in recent: + compact.append( + { + "event_id": d.get("event_id"), + "goal_id": d.get("goal_id"), + "todo_id": d.get("todo_id"), + "outcome": d.get("outcome") or d.get("decision"), + "action": d.get("action"), + "source": d.get("source"), + "recorded_at": d.get("recorded_at"), + } + ) + return { + "ok": True, + "decision_count": len(decisions), + "counts_by_outcome": dict(counts), + "recent_decisions": compact, + } + + +def build_control_plane_status( + *, + runtime_root: Path, + goal_id: str, + event_log_path: Path | None = None, + decision_event_log_path: Path | None = None, + worker_ids: Sequence[str] = (), + scheduler_tick_count: int | None = None, + event_history_limit: int = DEFAULT_EVENT_HISTORY_LIMIT, + decision_history_limit: int = DEFAULT_DECISION_HISTORY_LIMIT, + task_history_limit: int = DEFAULT_TASK_HISTORY_LIMIT, +) -> dict[str, Any]: + """Build a unified control-plane status snapshot for ``goal_id``. + + Aggregates scheduler, worker, queue, task, decision, and event history into a + single read-only digest. Safe to call any time; missing logs yield empty + sections rather than raising. + """ + queue_path = task_queue_path(runtime_root, goal_id=goal_id) + entries = _read_queue_entries(queue_path) + + log_path = event_log_path or rollout_event_log_path(runtime_root, goal_id) + + return { + "ok": True, + "goal_id": str(goal_id or "").strip(), + "schema_version": "loopx_control_plane_status_v0", + "scheduler": { + "goal_id": str(goal_id or "").strip(), + "tick_count": scheduler_tick_count, + "worker_ids": [str(w) for w in worker_ids], + }, + "queue": build_queue_digest(queue_path), + "workers": build_worker_status(entries), + "task_history": build_task_history(entries, limit=task_history_limit), + "decision_history": build_decision_history( + decision_event_log_path, limit=decision_history_limit + ) + if decision_event_log_path is not None + else {"ok": True, "decision_count": 0, "counts_by_outcome": {}, "recent_decisions": []}, + "event_history": build_event_history(log_path, limit=event_history_limit), + } + + +__all__ = [ + "DEFAULT_EVENT_HISTORY_LIMIT", + "DEFAULT_DECISION_HISTORY_LIMIT", + "DEFAULT_TASK_HISTORY_LIMIT", + "build_control_plane_status", + "build_queue_digest", + "build_worker_status", + "build_task_history", + "build_event_history", + "build_decision_history", +] diff --git a/loopx/control_plane/testing/cli_output_budget.py b/loopx/control_plane/testing/cli_output_budget.py index c17c1c076..868445018 100644 --- a/loopx/control_plane/testing/cli_output_budget.py +++ b/loopx/control_plane/testing/cli_output_budget.py @@ -320,7 +320,7 @@ class CliOutputCommandClassification: semantic_json_keys=("ledger", "truncated", "other_agent_frontier"), markdown_anchor="# LoopX Evidence Log", max_chars={ - "small": {"json": 2_900, "markdown": 800}, + "small": {"json": 2_900, "markdown": 900}, "crowded": {"json": 3_500, "markdown": 1_100}, "multi_agent": {"json": 4_300, "markdown": 1_200}, }, @@ -591,6 +591,12 @@ class CliOutputCommandClassification: surface_id=None, rationale="interactive local workspace launched explicitly by the operator", ), + CliOutputCommandClassification( + command_id="dashboard", + qualification="explicit_cold_path_exception", + surface_id=None, + rationale="interactive local dashboard service launched explicitly by the operator", + ), CliOutputCommandClassification( command_id="diagnose", qualification="qualified_default", diff --git a/loopx/heartbeat_prequota.py b/loopx/heartbeat_prequota.py index f10e6a3ff..59898cdc2 100644 --- a/loopx/heartbeat_prequota.py +++ b/loopx/heartbeat_prequota.py @@ -6,12 +6,22 @@ from .capabilities.issue_fix.pr_gate_reconcile import ( reconcile_acknowledged_issue_fix_pr_reviews, ) +from .control_plane.capabilities_bridge import CapabilityHookRegistry HEARTBEAT_PRE_QUOTA_SCHEMA_VERSION = "heartbeat_pre_quota_v0" +# Hook point for best-effort, no-quota reconciliation hooks that run before the +# heartbeat quota decision. Capability packs self-register under this point so +# this module no longer needs a static import per pack. +PRE_QUOTA_HOOK_POINT = "pre_quota" -def run_heartbeat_pre_quota( +# A process-wide hook registry. Capability packs (and tests) register their +# pre-quota hooks here; the host flow runs whatever is registered. +_capability_hook_registry = CapabilityHookRegistry() + + +def issue_fix_pr_review_reconcile_hook( *, registry_path: Path, runtime_root_arg: str | None, @@ -19,6 +29,7 @@ def run_heartbeat_pre_quota( agent_id: str, fetch_timeout_seconds: int = 10, ) -> dict[str, Any]: + """The built-in issue-fix PR-review reconcile hook (legacy default).""" try: review_reconciliation = reconcile_acknowledged_issue_fix_pr_reviews( registry_path=registry_path, @@ -30,20 +41,99 @@ def run_heartbeat_pre_quota( fetch_timeout_seconds=fetch_timeout_seconds, execute=True, ) - degraded = bool(review_reconciliation.get("degraded")) - failure_count = int(review_reconciliation.get("failure_count") or 0) - except Exception as exc: - degraded = True - failure_count = 1 - review_reconciliation = { + return { "ok": True, + "hook": "issue_fix_pr_review_reconcile", + "degraded": bool(review_reconciliation.get("degraded")), + "failure_count": int(review_reconciliation.get("failure_count") or 0), + "review_reconciliation": review_reconciliation, + } + except Exception as exc: # noqa: BLE001 - best-effort hook + return { + "ok": False, + "hook": "issue_fix_pr_review_reconcile", "degraded": True, "failure_count": 1, "failure_categories": [type(exc).__name__], - "external_read_count": 0, - "write_count": 0, } + +def register_pre_quota_hook(hook: Any, *, source: str = "") -> None: + """Register a capability-pack pre-quota hook (public extension point). + + Hook contract: ``hook(*, registry_path, runtime_root_arg, goal_id, agent_id, + fetch_timeout_seconds=10) -> dict``. The hook is invoked with exactly these + keyword arguments; a hook whose signature does not accept them is treated as + failed (``degraded``). The returned dict is merged into the pre-quota + ``checks.hooks`` result keyed by the hook's ``__name__`` (falling back to + ``"hook"`` for anonymous callables). Hooks that raise or return a non-dict + value are reported as failures without breaking the other hooks. + """ + _capability_hook_registry.register(PRE_QUOTA_HOOK_POINT, hook, source=source) + + +def get_pre_quota_hook_registry() -> CapabilityHookRegistry: + """Expose the process-wide registry (for inspection/testing).""" + return _capability_hook_registry + + +def run_heartbeat_pre_quota( + *, + registry_path: Path, + runtime_root_arg: str | None, + goal_id: str, + agent_id: str, + fetch_timeout_seconds: int = 10, +) -> dict[str, Any]: + """Run pre-quota checks, fanning out to registered capability hooks. + + Returns ``{"ok", "schema_version", "goal_id", "agent_id", "degraded", + "failure_count", "checks": {"acknowledged_pr_reviews", "hooks"}, ...}``. + Each hook result is keyed in ``checks.hooks`` by the hook ``__name__``; + the legacy ``checks.acknowledged_pr_reviews`` key is preserved as the + built-in issue-fix reconcile projection so downstream renderers keep + working unchanged. See ``register_pre_quota_hook`` for the hook contract. + """ + # Collect hooks from the registry, always including the built-in issue-fix + # reconcile hook so behavior is unchanged when no pack has registered. + hooks = _capability_hook_registry.hooks_for(PRE_QUOTA_HOOK_POINT) + if not any(getattr(h, "__name__", "") == "issue_fix_pr_review_reconcile_hook" for h in hooks): + hooks = [*hooks, issue_fix_pr_review_reconcile_hook] + + degraded = False + failure_count = 0 + checks: dict[str, Any] = {} + + for hook in hooks: + try: + result = hook( + registry_path=registry_path, + runtime_root_arg=runtime_root_arg, + goal_id=goal_id, + agent_id=agent_id, + fetch_timeout_seconds=fetch_timeout_seconds, + ) + except Exception as exc: # noqa: BLE001 - isolate hook failures + result = { + "ok": False, + "degraded": True, + "failure_count": 1, + "failure_categories": [type(exc).__name__], + } + if not isinstance(result, dict): + result = {} + if result.get("degraded"): + degraded = True + failure_count += int(result.get("failure_count") or 0) + # Key by the hook callable name so the built-in issue-fix hook can be + # located deterministically for the compatibility projection below. + hook_name = getattr(hook, "__name__", None) or str(result.get("hook") or "hook") + checks[hook_name] = result + + # Preserve the legacy ``acknowledged_pr_reviews`` key for compatibility. + review_reconciliation = checks.get("issue_fix_pr_review_reconcile_hook", {}) + acknowledged = review_reconciliation.get("review_reconciliation", review_reconciliation) + return { "ok": True, "schema_version": HEARTBEAT_PRE_QUOTA_SCHEMA_VERSION, @@ -52,7 +142,8 @@ def run_heartbeat_pre_quota( "degraded": degraded, "failure_count": failure_count, "checks": { - "acknowledged_pr_reviews": review_reconciliation, + "acknowledged_pr_reviews": acknowledged, + "hooks": checks, }, "quota_spend_required": False, "continue_to_quota": True, diff --git a/loopx/heartbeat_prompt.py b/loopx/heartbeat_prompt.py index c7a5299eb..bffb22abc 100644 --- a/loopx/heartbeat_prompt.py +++ b/loopx/heartbeat_prompt.py @@ -28,6 +28,7 @@ CODEX_NATIVE_GOAL_UNCHANGED_WAIT_RULE, DEFAULT_MATERIAL_QUEUE_RULE, DEFAULT_PERMISSION_RULE, + EVENT_DRIVEN_EXECUTION_RULE, HEARTBEAT_NOTIFICATION_RULE_SHORT, HEARTBEAT_VISION_WRITEBACK_RULE_SHORT, RUNTIME_CAPABILITY_PROJECTION_THIN_RULE, @@ -50,6 +51,7 @@ "CODEX_NATIVE_GOAL_UNCHANGED_WAIT_RULE", "DEFAULT_MATERIAL_QUEUE_RULE", "DEFAULT_PERMISSION_RULE", + "EVENT_DRIVEN_EXECUTION_RULE", "HEARTBEAT_NOTIFICATION_RULE_SHORT", "HEARTBEAT_VISION_WRITEBACK_RULE_SHORT", "INTERFACE_BUDGET_CHARS", diff --git a/loopx/opencode_goal_mode/goal-bridge-runtime.mjs b/loopx/opencode_goal_mode/goal-bridge-runtime.mjs index e149d183e..1cdbeb23f 100644 --- a/loopx/opencode_goal_mode/goal-bridge-runtime.mjs +++ b/loopx/opencode_goal_mode/goal-bridge-runtime.mjs @@ -10,6 +10,7 @@ const execFile = promisify(execFileCallback) const BRIDGE_SCHEMA_VERSION = "loopx_opencode_goal_bridge_v0" const TERMINAL_STATE_SCHEMA_VERSION = "goal_terminal_state_v0" const SOURCE_COMPLETENESS_SCHEMA_VERSION = "goal_terminal_source_completeness_v0" +const POLICY_DECISION_SCHEMA_VERSION = "loopx_opencode_goal_policy_decision_v0" const DEFAULT_RETRY_MINUTES = 3 const LOOPX_GOAL_LIMITS = { maxTurns: 10000, @@ -177,10 +178,77 @@ export async function probeLoopxQuota(binding, { directory, execFileImpl = execF } +// Phase 5/6 new-architecture compatibility: advance the event-driven Task +// Queue for the bound goal (record task_ready / task_enqueued / task_dispatched +// audit facts and claim the next pending task for the bound worker). This is the +// event-driven dispatch half of the loop: ``quota should-run`` remains the +// policy decision source (its ``policy_decision`` carries the authoritative +// run | wait | deny outcome), while this probe keeps the Task Queue lifecycle +// (claim -> complete -> fail) actually moving so the resident scheduler and the +// goal acceptance / closure layer see real queue advancement. +// +// The dispatch CLI is enabled by default (its ``--event-driven`` flag falls +// back to the new-architecture master switch, which is on). When the master +// switch is explicitly off (``LOOPX_NEW_ARCHITECTURE=0``) the dispatch returns +// a ``disabled`` marker and writes nothing, preserving the legacy heartbeat +// path. Callers treat any failure as a silent no-op so the quota-gated loop +// never breaks on the optional dispatch advancement. +export async function probeEventDrivenDispatch(binding, { directory, execFileImpl = execFile } = {}) { + const args = [] + if (binding.registryPath) args.push("--registry", binding.registryPath) + args.push( + "--format", + "json", + "codex-cli-local-scheduler-dispatch", + "--goal-id", + binding.goalId, + ) + const workerId = binding.agentId || binding.workerId || "" + if (workerId) { + args.push("--worker-id", workerId) + } + const { stdout } = await execFileImpl(process.env.LOOPX_BIN || "loopx", args, { + cwd: binding.directory || directory, + timeout: 20_000, + maxBuffer: 4 * 1024 * 1024, + }) + const payload = JSON.parse(stdout || "{}") + if (!payload || typeof payload !== "object" || Array.isArray(payload)) { + throw new Error("loopx codex-cli-local-scheduler-dispatch returned a non-object payload") + } + return payload +} + + +// Phase 5 new-architecture compatibility: the control_plane now attaches a +// normalized `policy_decision` (outcome = run | wait | deny) to the quota +// should-run payload by default (master switch). When present, the loop prefers +// the unified contract and falls back to the legacy quota fields otherwise, +// keeping the opt-out path behaviour identical. +// +// policy_decision.outcome: +// "run" -> continue now (like legacy should_run === true / run_now) +// "wait" -> back off using retry_after_seconds / retry_at +// "deny" -> not authorized to continue; non-terminal denies hold the loop +// +// `retry_at` is an ISO 8601 datetime; `retry_after_seconds` is seconds. +export function policyDecisionOf(decision) { + const unified = decision?.policy_decision + if (!unified || typeof unified !== "object") return null + const outcome = String(unified.outcome || "") + if (outcome !== "run" && outcome !== "wait" && outcome !== "deny") return null + return unified +} + + export function isTerminalNoFollowup(decision) { const frontier = decision?.goal_frontier_projection const terminal = frontier?.terminal_state const completeness = frontier?.source_completeness + // The unified policy decision does not carry the validated goal-closure + // projection (that lives on the quota layer), so terminal detection still + // requires the frontier proof. A policy outcome of "deny" alone is not a + // validated terminal state and must never self-close the goal. return Boolean( decision?.should_run === false && decision?.effective_action === "terminal_no_followup" && @@ -196,11 +264,65 @@ export function isTerminalNoFollowup(decision) { function shouldRunNow(decision) { + const unified = policyDecisionOf(decision) + if (unified) { + // Prefer the unified policy contract; the legacy fields are still present + // but are subsumed by the composed outcome. + if (unified.outcome === "run") return true + if (unified.outcome === "deny") return false + if (unified.outcome === "wait") return false + } return decision?.scheduler_hint?.action === "run_now" || decision?.should_run === true } +// Minutes to wait for the next poll from a unified policy decision. +function unifiedRetryMinutes(unified) { + const seconds = Number(unified?.retry_after_seconds) + if (Number.isFinite(seconds) && seconds >= 0) { + const minutes = Math.ceil(seconds / 60) + return Math.max(1, minutes) + } + const at = unified?.retry_at + if (typeof at === "string" && at) { + const when = Date.parse(at) + if (Number.isFinite(when)) { + const minutes = Math.ceil((when - Date.now()) / 60_000) + if (minutes > 0) return minutes + } + } + return null +} + + function waitPlan(decision, binding) { + const unified = policyDecisionOf(decision) + if (unified) { + // Unified path: "run" is intercepted by shouldRunNow; "deny" without a + // validated terminal is a non-terminal hold, so we back off rather than + // poll-spin and must not self-close the goal. + const fromUnified = unifiedRetryMinutes(unified) + if (fromUnified !== null) { + return { + stop: false, + minutes: fromUnified, + schedulerToken: binding.schedulerToken, + unchangedPolls: binding.unchangedPolls, + } + } + if (unified.outcome === "deny") { + // No retry hint and not terminal: hold the loop. Stop polling now; a + // later explicit activation / resume re-arms it. + return { + stop: true, + minutes: DEFAULT_RETRY_MINUTES, + schedulerToken: binding.schedulerToken, + unchangedPolls: binding.unchangedPolls, + } + } + // "wait" without a retry hint: fall through to the legacy scheduler hints so + // behaviour matches the pre-PolicyEngine path when no hint is provided. + } const local = decision?.scheduler_hint?.unchanged_poll?.local_scheduler if (!local || local === "stop") return { stop: true } const token = decision?.scheduler_hint?.reset_policy?.reset_token || "" @@ -259,6 +381,7 @@ export function createLoopxGoalPlugin({ tool, bindingStore = createFileBindingStore(), quotaProbe = probeLoopxQuota, + dispatchProbe = probeEventDrivenDispatch, setTimer = setTimeout, clearTimer = clearTimeout, } = {}) { @@ -435,6 +558,27 @@ export function createLoopxGoalPlugin({ } binding = currentBinding + // Phase 5/6: advance the event-driven Task Queue for this goal. This is a + // fire-and-forget best-effort probe — it records task_ready / enqueued / + // dispatched audit facts and claims the next pending task, but never gates + // the policy decision (which stays authoritative via ``quota should-run``). + // Any failure is a silent no-op so the quota-gated loop cannot break, and + // a disabled master switch (``LOOPX_NEW_ARCHITECTURE=0``) returns a + // ``disabled`` marker that writes nothing (legacy heartbeat path). + if (typeof dispatchProbe === "function") { + void (async () => { + try { + await dispatchProbe(binding, { directory: context.directory }) + } catch (error) { + await log("debug", "LoopX event-driven dispatch advancement skipped", { + goalId: binding.goalId, + sessionID, + error: error?.message || String(error), + }) + } + })() + } + if (isTerminalNoFollowup(decision)) { await completeTerminalGoal(sessionID, binding, decision) return diff --git a/loopx/pi_goal_mode/loopx-goal.ts b/loopx/pi_goal_mode/loopx-goal.ts index e445ca133..7d1a41189 100644 --- a/loopx/pi_goal_mode/loopx-goal.ts +++ b/loopx/pi_goal_mode/loopx-goal.ts @@ -27,6 +27,7 @@ import { execFile as execFileCallback } from "node:child_process"; import { promisify } from "node:util"; import { Type } from "typebox"; import { + buildDispatchArgs, buildQuotaArgs, createBindingStore, createEphemeralSessionIdentity, @@ -56,6 +57,23 @@ async function probeLoopxQuota(binding: Record): Promise; } +// Phase 5/6: advance the event-driven Task Queue (record task_ready / enqueued +// / dispatched audit facts and claim the next pending task). Best-effort — the +// loop treats any failure as a silent no-op, and a disabled master switch +// (``LOOPX_NEW_ARCHITECTURE=0``) returns a ``disabled`` marker that writes +// nothing, preserving the legacy heartbeat path. +async function probeEventDrivenDispatch( + binding: Record, +): Promise> { + const args = buildDispatchArgs(binding); + const stdout = await runLoopxCli(args, String(binding.directory || "")); + const payload: unknown = JSON.parse(stdout || "{}"); + if (!payload || typeof payload !== "object" || Array.isArray(payload)) { + throw new Error("loopx codex-cli-local-scheduler-dispatch returned a non-object payload"); + } + return payload as Record; +} + export default function (pi: ExtensionAPI) { // Durable, TUI-only digest of the last inspected LoopX packet. Custom entries // never enter LLM context; the agent reads full state through the CLI itself. @@ -84,6 +102,7 @@ export default function (pi: ExtensionAPI) { // the whole instance so no old session can keep continuing. const loop = createGoalLoop({ quotaProbe: probeLoopxQuota, + dispatchProbe: probeEventDrivenDispatch, sendMessage: (prompt: string) => { pi.sendUserMessage(prompt, { deliverAs: "followUp", triggerTurn: true }); }, diff --git a/loopx/pi_goal_mode/pi-goal-loop-runtime.mjs b/loopx/pi_goal_mode/pi-goal-loop-runtime.mjs index f87e00f90..f0e9aa429 100644 --- a/loopx/pi_goal_mode/pi-goal-loop-runtime.mjs +++ b/loopx/pi_goal_mode/pi-goal-loop-runtime.mjs @@ -32,6 +32,7 @@ import path from "node:path" export const BRIDGE_SCHEMA_VERSION = "loopx_pi_goal_bridge_v0" export const TERMINAL_STATE_SCHEMA_VERSION = "goal_terminal_state_v0" export const SOURCE_COMPLETENESS_SCHEMA_VERSION = "goal_terminal_source_completeness_v0" +export const POLICY_DECISION_SCHEMA_VERSION = "loopx_pi_goal_policy_decision_v0" export const DEFAULT_RETRY_MINUTES = 3 // The label prefix of a session key reserves room for the digest suffix so @@ -229,10 +230,62 @@ export function buildQuotaArgs(binding) { return args } +// Phase 5/6 new-architecture compatibility: advance the event-driven Task Queue +// for the bound goal. ``quota should-run`` stays the policy decision source (its +// ``policy_decision`` carries the authoritative run | wait | deny outcome); this +// dispatch args array drives the Task Queue lifecycle (task_ready / enqueued / +// dispatched audit facts + claim the next pending task) so the resident +// scheduler and goal acceptance / closure layer see real queue advancement. +// +// The dispatch CLI is enabled by default (its ``--event-driven`` flag falls +// back to the new-architecture master switch, which is on). When the master +// switch is explicitly off (``LOOPX_NEW_ARCHITECTURE=0``) the dispatch returns a +// ``disabled`` marker and writes nothing, preserving the legacy heartbeat path. +export function buildDispatchArgs(binding) { + const args = [] + if (binding.registryPath) args.push("--registry", binding.registryPath) + args.push( + "--format", + "json", + "codex-cli-local-scheduler-dispatch", + "--goal-id", + binding.goalId, + ) + const workerId = binding.agentId || binding.workerId || "" + if (workerId) { + args.push("--worker-id", workerId) + } + return args +} + +// Phase 5 compatibility: the control_plane PolicyEngine attaches a normalized +// `policy_decision` to the quota should-run payload. When present, the loop +// prefers the unified contract (outcome = run | wait | deny) and falls back to +// the legacy quota fields otherwise, keeping the default path behaviour identical. +// +// policy_decision.outcome: +// "run" -> continue now (like legacy should_run === true / run_now) +// "wait" -> back off using retry_after_seconds / retry_at, else legacy scheduler +// "deny" -> not authorized to continue; non-terminal denies hold the loop +// +// `retry_at` is an ISO 8601 datetime; `retry_after_seconds` is a plain seconds +// count. Both are honoured when the unified decision is in effect. +export function policyDecisionOf(decision) { + const unified = decision?.policy_decision + if (!unified || typeof unified !== "object") return null + const outcome = String(unified.outcome || "") + if (outcome !== "run" && outcome !== "wait" && outcome !== "deny") return null + return unified +} + export function isTerminalNoFollowup(decision) { const frontier = decision?.goal_frontier_projection const terminal = frontier?.terminal_state const completeness = frontier?.source_completeness + // The unified policy decision does not carry the validated goal-closure + // projection (that lives on the quota layer), so terminal detection still + // requires the frontier proof. A policy outcome of "deny" alone is not a + // validated terminal state and must never self-close the loop. return Boolean( decision?.should_run === false && decision?.effective_action === "terminal_no_followup" && @@ -247,11 +300,64 @@ export function isTerminalNoFollowup(decision) { } export function shouldRunNow(decision) { + const unified = policyDecisionOf(decision) + if (unified) { + // Prefer the unified policy contract; the legacy fields are still present + // but are subsumed by the composed outcome. + if (unified.outcome === "run") return true + if (unified.outcome === "deny") return false + if (unified.outcome === "wait") return false + } const hint = decision?.scheduler_hint return hint?.action === "run_now" || decision?.should_run === true } +// Minutes to wait for the next poll from a unified policy decision. +function unifiedRetryMinutes(unified) { + const seconds = Number(unified?.retry_after_seconds) + if (Number.isFinite(seconds) && seconds >= 0) { + const minutes = Math.ceil(seconds / 60) + return Math.max(1, minutes) + } + const at = unified?.retry_at + if (typeof at === "string" && at) { + const when = Date.parse(at) + if (Number.isFinite(when)) { + const minutes = Math.ceil((when - Date.now()) / 60_000) + if (minutes > 0) return minutes + } + } + return null +} + export function waitPlan(decision, binding) { + const unified = policyDecisionOf(decision) + if (unified) { + // Unified path: the policy engine already resolved the action. "run" should + // never reach here (shouldRunNow intercepts it); "deny" without a validated + // terminal is a non-terminal hold, so we back off rather than poll-spin. + const fromUnified = unifiedRetryMinutes(unified) + if (fromUnified !== null) { + return { + stop: false, + minutes: fromUnified, + schedulerToken: binding.schedulerToken, + unchangedPolls: binding.unchangedPolls, + } + } + if (unified.outcome === "deny") { + // No retry hint and not terminal: hold the loop. Stop polling now; a later + // explicit activation / resume re-arms it. + return { + stop: true, + minutes: DEFAULT_RETRY_MINUTES, + schedulerToken: binding.schedulerToken, + unchangedPolls: binding.unchangedPolls, + } + } + // "wait" without a retry hint: fall through to the legacy scheduler hints so + // behaviour matches the pre-PolicyEngine path when no hint is provided. + } const hint = decision?.scheduler_hint || {} const unchanged = hint?.unchanged_poll || {} const local = unchanged?.local_scheduler @@ -309,7 +415,7 @@ export function waitPlan(decision, binding) { // goalId, so a stale evaluation cannot commit past a re-activation even when // its write was already in-flight. export function createGoalLoop(options) { - const { quotaProbe, sendMessage, setTimer, clearTimer } = options + const { quotaProbe, dispatchProbe, sendMessage, setTimer, clearTimer } = options const timers = new Map() const evaluations = new Map() const contexts = new Map() @@ -407,6 +513,23 @@ export function createGoalLoop(options) { return } + // Phase 5/6: advance the event-driven Task Queue for this goal. This is a + // best-effort fire-and-forget probe — it records task_ready / enqueued / + // dispatched audit facts and claims the next pending task, but never gates + // the policy decision (which stays authoritative via ``quota should-run``). + // Any failure is a silent no-op so the quota-gated loop cannot break, and a + // disabled master switch (``LOOPX_NEW_ARCHITECTURE=0``) returns a + // ``disabled`` marker that writes nothing (legacy heartbeat path). + if (typeof dispatchProbe === "function") { + void (async () => { + try { + await dispatchProbe(binding) + } catch { + // Best-effort: the Task Queue advancement is optional to the loop. + } + })() + } + if (isTerminalNoFollowup(decision)) { // Commit through the store's compare-and-swap: if the same session // activated a new goal while this write was in-flight, the commit is diff --git a/loopx/presentation/renderers/status_markdown.py b/loopx/presentation/renderers/status_markdown.py index df284162e..944a0f056 100644 --- a/loopx/presentation/renderers/status_markdown.py +++ b/loopx/presentation/renderers/status_markdown.py @@ -482,6 +482,115 @@ def append_decision_freshness_summary_markdown( ) +def append_policy_decision_markdown( + lines: list[str], + policy_decision: dict[str, Any], +) -> None: + """Render the Phase 5 unified policy decision (outcome = run | wait | deny). + + The new-architecture PolicyEngine attaches this normalized contract to the + quota should-run payload. It is read-only display; absent it emits nothing. + """ + if not isinstance(policy_decision, dict): + return + outcome = markdown_scalar(policy_decision.get("outcome") or "") + if outcome not in {"run", "wait", "deny"}: + return + retry = "" + if policy_decision.get("retry_after_seconds") is not None: + retry = f" retry_after_seconds={policy_decision.get('retry_after_seconds')}" + elif policy_decision.get("retry_at"): + retry = f" retry_at={markdown_scalar(policy_decision.get('retry_at') or '')}" + reason = markdown_scalar(policy_decision.get("reason") or policy_decision.get("reason_code") or "") + reason_text = f" reason={reason}" if reason else "" + lines.append( + " - policy_decision: " + f"outcome={outcome}" + f"{retry}" + f"{reason_text}" + f" action={markdown_scalar(policy_decision.get('action') or policy_decision.get('effective_action') or '')}" + ) + + +def append_control_plane_status_markdown( + lines: list[str], + control_plane_status: dict[str, Any], +) -> None: + """Render the Phase 5/6 control-plane observability snapshot (P2). + + Read-only digest of the event-driven control plane: scheduler, worker pool, + task queue (incl. lifecycle states), and decision history. Absent it emits + nothing. + """ + if not isinstance(control_plane_status, dict): + return + # Emit nothing unless at least one observability section carries data, so a + # missing ``control_plane_status`` key (rendered as ``{}`` upstream) never + # produces an empty heading. + if not any(control_plane_status.get(k) for k in ("scheduler", "queue", "workers", "decision_history")): + return + lines.extend(["", "## Control-Plane Status (event-driven)", "- scheduler:"]) + scheduler = ( + control_plane_status.get("scheduler") + if isinstance(control_plane_status.get("scheduler"), dict) + else {} + ) + lines.append( + f" - tick_count={scheduler.get('tick_count')} " + f"workers={','.join(markdown_scalar(w) for w in (scheduler.get('worker_ids') or [])) or '-'}" + ) + queue = ( + control_plane_status.get("queue") + if isinstance(control_plane_status.get("queue"), dict) + else {} + ) + if queue: + extended = queue.get("extended") if isinstance(queue.get("extended"), dict) else {} + lines.append( + " - queue: " + f"pending={queue.get('pending_count', 0)} " + f"claimed={queue.get('claimed_count', 0)} " + f"done={queue.get('done_count', 0)} " + f"in_flight={queue.get('in_flight_count', 0)} " + f"exceptions={queue.get('exception_count', 0)} " + f"retry_wait={extended.get('retry_wait_count', 0)} " + f"failed={extended.get('failed_count', 0)} " + f"dead_letter={extended.get('dead_letter_count', 0)} " + f"cancelled={extended.get('cancelled_count', 0)}" + ) + workers = ( + control_plane_status.get("workers") + if isinstance(control_plane_status.get("workers"), dict) + else {} + ) + if workers: + lines.append( + " - workers: " + f"count={workers.get('worker_count', 0)} " + f"active={workers.get('active_count', 0)} " + f"idle={workers.get('idle_count', 0)}" + ) + decision_history = ( + control_plane_status.get("decision_history") + if isinstance(control_plane_status.get("decision_history"), dict) + else {} + ) + if decision_history: + counts = ( + decision_history.get("counts_by_outcome") + if isinstance(decision_history.get("counts_by_outcome"), dict) + else {} + ) + counts_text = " ".join( + f"{markdown_scalar(k)}={v}" for k, v in counts.items() + ) + lines.append( + " - decisions: " + f"count={decision_history.get('decision_count', 0)} " + f"{counts_text}" + ) + + def append_usage_summary_markdown(lines: list[str], usage: dict[str, Any]) -> None: usage_totals = usage.get("totals") if isinstance(usage.get("totals"), dict) else {} if not usage.get("available") or not usage_totals: @@ -983,6 +1092,13 @@ def render_status_markdown( ) append_decision_freshness_summary_markdown(lines, decision_freshness) + control_plane_status = ( + payload.get("control_plane_status") + if isinstance(payload.get("control_plane_status"), dict) + else {} + ) + append_control_plane_status_markdown(lines, control_plane_status) + usage = payload.get("usage_summary") if isinstance(payload.get("usage_summary"), dict) else {} append_usage_summary_markdown(lines, usage) @@ -1542,6 +1658,10 @@ def append_attention_queue_item_operational_markdown( if control_plane: lines.append(f" - control_plane: {control_plane_policy_summary(control_plane)}") + policy_decision = item.get("policy_decision") if isinstance(item.get("policy_decision"), dict) else None + if policy_decision is not None: + append_policy_decision_markdown(lines, policy_decision) + operator_question = item.get("operator_question") agent_command = item.get("agent_command") if operator_question: diff --git a/loopx/registry.py b/loopx/registry.py index 2090dce0a..9a70344ee 100644 --- a/loopx/registry.py +++ b/loopx/registry.py @@ -5,6 +5,7 @@ import re import subprocess import tempfile +from datetime import datetime, timezone from pathlib import Path from typing import Any @@ -40,6 +41,42 @@ def find_registry_goal(registry: dict[str, Any], goal_id: str) -> dict[str, Any] return None +def sync_registry_goal_closed( + registry_path: Path, + goal_id: str, + *, + recorded_at: str | None = None, +) -> bool: + """Synchronize the registry goal's ``status`` to ``closed``. + + The Goal Closure Evaluator records a ``goal_closed`` rollout event when it + derives closure, but the registry goal entry's ``status`` field was never + updated alongside it. That split made ``start-goal``'s guided packet (which + reads the rollout log via ``_goal_already_closed``) report the goal as closed + while ``status``/registry still showed ``active`` — the agent then looped + between "packet says closed" and "registry says active". + + This writes ``status=closed`` on the goal entry so the two state sources + agree. Idempotent: an already-``closed`` goal is a no-op. + """ + try: + registry = read_json(registry_path) + except Exception: + return False + goal = find_registry_goal(registry, goal_id) + if goal is None: + return False + if str(goal.get("status") or "") == "closed": + return False + goal["status"] = "closed" + goal["updated_at"] = recorded_at or datetime.now(timezone.utc).isoformat() + try: + atomic_write_json(registry_path, registry) + except Exception: + return False + return True + + def atomic_write_json( path: Path, payload: dict[str, Any], diff --git a/loopx/rollout_event_log.py b/loopx/rollout_event_log.py index e6cc2efb1..9a49c48f4 100644 --- a/loopx/rollout_event_log.py +++ b/loopx/rollout_event_log.py @@ -22,6 +22,12 @@ "compact_blocker", "compact_case_result", "failure_attribution", + "goal_closed", + "goal_closure_ready", + "goal_acceptance_pending", + "goal_acceptance_satisfied", + "heartbeat_observed", + "policy_decision", "pr_merge", "pr_review_ack", "quota_monitor_poll", @@ -31,6 +37,12 @@ "refresh_state", "research_evidence", "research_hypothesis", + "task_completed", + "task_dispatched", + "task_enqueued", + "task_executed", + "task_failed", + "task_ready", "todo_add", "todo_archive_completed", "todo_claim", diff --git a/loopx/slash_command_install.py b/loopx/slash_command_install.py index 6dce82008..ff8643d2b 100644 --- a/loopx/slash_command_install.py +++ b/loopx/slash_command_install.py @@ -152,11 +152,13 @@ def _command_prompt_specs(*, cli_bin: str, include_legacy_aliases: bool) -> list f"Treat the returned `ordered_steps` as a required transaction. On first connection, run its bootstrap command and resolve the returned agent-identity gate before planning; a stable unbound host thread is a new session and defaults to fresh registration. Then plan and execute at least one business `{cli_bin} todo add` derived from `$ARGUMENTS` before substantive task work. Encode priority in the todo text such as `[P0]`; `{cli_bin} todo add` has no `--priority` flag. Do not continue until LoopX status shows that business Agent Todo.", f"If `selected_capability_route` is present, run its entry and admission commands before substantive implementation, and treat `{cli_bin} capability show --format json` as the authoritative later-transition command surface. Use capability-owned commands for listed external transitions instead of substituting provider CLIs. Keep capability facts in capability-owned state; generic Todos remain scheduling records.", f"Before dependent work, persist material scope, acceptance, or non-goal changes in current Todo evidence and the next executable Todo; then run `{cli_bin} refresh-state` and verify quota readback. Chat/model summaries are not durable state.", - f"If that packet exposes a goal-selection gate, rerun one exact choice before any mutation. Resolve agent identity in this order: reuse the packet's verified thread binding; otherwise reuse the exact agent id already named by this host task's active LoopX interaction contract or heartbeat; otherwise treat a stable unbound host thread as a new session and follow the packet's fresh-registration default. Select an existing registered lane only for explicit takeover, then complete the returned bind/readback step. Codex App `start-goal` automatically reads its stable ambient thread id when available. Never treat a new Todo, worktree, or argument-bearing invocation as a new peer by itself, and never infer identity from registry order. When no stable thread id is available, fresh registration requires explicit new-peer/session intent and `--new-peer`. Preview `{cli_bin} register-agent --goal-id --agent-id --require-new`, then apply with `--execute` and continue only when the result reports `ok=true`, `changed=true`, `written=true`, successful global sync, and verified registration readback. Rerun start-goal with the verified `--agent-id` before todo writeback.", + f"If that packet exposes a goal-selection gate, rerun one exact choice before any mutation. Resolve agent identity in this order: reuse the packet's verified thread binding; otherwise reuse the exact agent id already named by this host task's active LoopX interaction contract or heartbeat; otherwise treat a stable unbound host thread as a new session and follow the packet's fresh-registration default. Select an existing registered lane only for explicit takeover, then complete the returned bind/readback step. Codex App `start-goal` automatically reads its stable ambient thread id when available. Never treat a new Todo, worktree, or argument-bearing invocation as a new peer by itself, and never infer identity from registry order. When no stable thread id is available, fresh registration requires explicit new-peer/session intent and `--require-new`. Preview `{cli_bin} register-agent --goal-id --agent-id --require-new`, then apply with `--execute` and continue only when the result reports `ok=true`, `changed=true`, `written=true`, successful global sync, and verified registration readback. Rerun start-goal with the verified `--agent-id` before todo writeback.", f"If arguments are empty and the host task already identifies an active LoopX goal, run its exact CLI `interaction_contract` or quota command first; do not call `start-goal` or bootstrap another goal. Only when no active goal contract is present, inspect `{cli_bin} bootstrap-command-pack --project .`, `{cli_bin} status`, and `{cli_bin} slash-commands` before changing files.", f"Use `{cli_bin} agent-onboard --list-agent-types` when the host runtime is unclear; pass an exact type such as `codex-app`, `codex-app-ssh`, `codex-ide-plugin`, `codex-cli`, `claude-code`, `opencode`, `traex-cli`, `pi`, or `ark-managed-agent`, never ambiguous `codex`.", f"Do not configure optional features during first-run. Only when the task needs bounded child agents or Explore, inspect `{cli_bin} configure-goal --goal-id ` and its `configuration_catalog`; preview before explicit apply and never auto-enable a feature merely because it exists.", "When project work is started, plan ordered P0/P1/P2 todos, write them through LoopX todo state, refresh state, activate the host loop if missing/stale, run quota, and complete one bounded delivery segment through validation plus LoopX writeback or an exact blocker; do not return merely after setup, planning, or claim.", + f"Prefer the event-driven scheduler (RFC Phase 5/6 new architecture) for task advancement when it is available. Each time you write or complete a business Todo, after `{cli_bin} refresh-state`, advance the queue by running `{cli_bin} codex-cli-local-scheduler-dispatch --goal-id --project . --completed-todo-id --agent-id --event-driven` (add `--worker-id ` to also claim the next ready task). This recomputes READY successors from handoff gates, records `task_ready`/`task_enqueued`/`task_dispatched` audit facts, and enqueues the next runnable task. When you need bounded batch advancement or a long-lived scheduling loop, prefer `{cli_bin} codex-cli-local-scheduler-resident --goal-id --project . --agent-id --iterations --event-driven` (add `--execute-worker-command '' --worker-command-prefix '' --guard-checked` only to actually run the claimed task behind explicit gates). When a heartbeat poll is needed, prefer the merged path `{cli_bin} codex-cli-local-scheduler-merge --goal-id --project . --agent-id --merge` (records `heartbeat_observed`, runs `PolicyEngine` decision, and event-driven dispatch in one tick). These paths are opt-in: if any of these commands is unavailable or errors in this environment, fall back to the legacy `quota` / heartbeat flow and continue the business task without failing.", + "To keep the new architecture active, ensure the relevant opt-in env flags are set when invoking the scheduler commands (they default on when `LOOPX_NEW_ARCHITECTURE` is unset): `LOOPX_EVENT_DRIVEN_DISPATCH=1` for dispatch/resident, `LOOPX_HEARTBEAT_EVENT_SOURCE=1` for the heartbeat event source, `LOOPX_MERGE_EVENT_DRIVEN_AND_HEARTBEAT=1` for the merged path, `LOOPX_USE_POLICY_ENGINE=1` so `quota should-run` attaches the unified `policy_decision`, and optionally `LOOPX_POLICY_DECISION_RECORD=1` to persist policy decisions as audit events.", "Host loop activation means Codex App heartbeat automation; Codex App over SSH, the Codex IDE plugin, or CLI visible `/goal `; Claude Code native `/loop`; OpenCode `loopx_goal_activate`; TraeX visible `/goal `; Ark Managed Agent one-shot Goal submission; or a custom host-loop gate from `loopx agent-onboard`.", "If this session cannot mutate the host loop surface, surface the exact pasteable gate instead of saying LoopX is autonomously connected.", ], diff --git a/loopx/todos.py b/loopx/todos.py index 0700234f6..2fb244e92 100644 --- a/loopx/todos.py +++ b/loopx/todos.py @@ -8,7 +8,12 @@ from .file_lock import exclusive_file_lock from .history import load_registry from .paths import resolve_runtime_root -from .rollout_event_log import load_rollout_events, rollout_event_log_path +from .rollout_event_log import ( + append_rollout_event_once, + build_rollout_event, + load_rollout_events, + rollout_event_log_path, +) from .control_plane.runtime.local_state_write_correctness import build_todo_write_correctness_dry_run_packet from .state_refresh import now_local, resolve_goal_state from .status import ( @@ -1064,6 +1069,50 @@ def add_goal_todo( if changed and not dry_run: resolved_state_file.write_text(new_text, encoding="utf-8") + # Bridge todo add -> event-driven dispatch. ``add_goal_todo`` writes the + # markdown active-state file only; the event-driven scheduler rebuilds + # todo items from the rollout event log (``todo_add``/``todo_complete``) + # and never reads the markdown file. Without this, a todo created via + # ``loopx todo add`` is invisible to dispatch and the agent falls back to + # hand-editing + ``todo complete`` (the bypass seen in the website1 color + # session). Appending a public-safe ``todo_add`` event keeps the todo + # discoverable by ``load_todo_items_from_rollout_log``. + added_todo_id = add_result.get("todo_id") + if added and not dry_run and added_todo_id: + try: + _registry = load_registry(registry_path) + _runtime_root = resolve_runtime_root( + _registry, None, registry_path=registry_path + ) + except Exception: + _runtime_root = None + if _runtime_root is not None: + _log_path = rollout_event_log_path(_runtime_root, goal_id) + _details: dict[str, Any] = {"role": role} + if task_class: + _details["task_class"] = str(task_class) + if action_kind: + _details["action_kind"] = str(action_kind) + if normalized_unblocks_todo_id: + _details["unblocks_todo_id"] = str(normalized_unblocks_todo_id) + if effective_excluded_agents: + _details["excluded_agents"] = [ + str(a) for a in effective_excluded_agents + ] + _event = build_rollout_event( + goal_id=goal_id, + event_kind="todo_add", + todo_id=str(added_todo_id), + status=str(normalized_status), + agent_id=effective_agent_id or effective_claimed_by, + details=_details, + ) + append_rollout_event_once( + _log_path, + _event, + identity_fields=("goal_id", "event_kind", "todo_id"), + ) + payload = { "ok": True, "dry_run": dry_run, @@ -1830,6 +1879,37 @@ def complete_goal_todo( new_text = replace_updated_at(new_text, updated_at) if changed and not dry_run: resolved_state_file.write_text(new_text, encoding="utf-8") + + # Symmetric bridge: ``todo add`` writes a rollout ``todo_add`` event (see + # ``add_goal_todo``); ``todo complete`` must write a matching rollout + # ``todo_complete`` event so ``load_todo_items_from_rollout_log`` can + # derive the terminal ``done`` state from the rollout log alone. Without + # this, a completed todo stays ``open`` in the rollout-log projection and + # the Closure Evaluator forever sees "open work remaining". + completed_todo_id_value = update_result.get("todo_id") or todo_id + if changed and not dry_run and completed_todo_id_value: + try: + _registry = load_registry(registry_path) + _runtime_root = resolve_runtime_root( + _registry, None, registry_path=registry_path + ) + except Exception: + _runtime_root = None + if _runtime_root is not None: + _log_path = rollout_event_log_path(_runtime_root, goal_id) + _event = build_rollout_event( + goal_id=goal_id, + event_kind="todo_complete", + todo_id=str(completed_todo_id_value), + status=TODO_STATUS_DONE, + agent_id=agent_id or effective_claimed_by, + ) + append_rollout_event_once( + _log_path, + _event, + identity_fields=("goal_id", "event_kind", "todo_id"), + ) + result = { "ok": True, "dry_run": dry_run, diff --git a/tests/control_plane/test_capabilities_bridge.py b/tests/control_plane/test_capabilities_bridge.py new file mode 100644 index 000000000..204adb4ef --- /dev/null +++ b/tests/control_plane/test_capabilities_bridge.py @@ -0,0 +1,271 @@ +"""Tests for the capability-pack bridge (P1/P2/P3).""" + +from __future__ import annotations + +import pytest + +from loopx.capabilities.catalog import build_capability_registry +from loopx.capabilities.registry import CapabilityRegistry +from loopx.control_plane.capabilities_bridge import ( + CapabilityEventHub, + CapabilityHookRegistry, + capability_pack_ready, + capability_token, + capability_token_set, + discover_cli_registrars, + eligible_bridged, + register_all_capability_commands, + resolve_required_tokens, + split_binding_ref, +) + + +# --------------------------------------------------------------------------- +# Token normalization +# --------------------------------------------------------------------------- + + +def test_capability_token_normalizes_hyphens(): + assert capability_token("issue-fix") == "issue_fix" + assert capability_token("Issue-Fix") == "issue_fix" + assert capability_token("issue fix") == "issue_fix" + assert capability_token(None) is None + assert capability_token("") is None + + +def test_capability_token_set_accepts_various_shapes(): + assert capability_token_set("issue-fix, pull-request-review") == { + "issue_fix", + "pull_request_review", + } + assert capability_token_set(["issue-fix", "shell"]) == {"issue_fix", "shell"} + assert capability_token_set(None) == set() + + +def test_split_binding_ref(): + assert split_binding_ref("issue-fix:feasibility_v0") == ("issue-fix", "feasibility_v0") + assert split_binding_ref(None) is None + assert split_binding_ref("not-a-valid-binding") is None + + +# --------------------------------------------------------------------------- +# P1: registry-driven eligibility +# --------------------------------------------------------------------------- + + +def _registry() -> CapabilityRegistry: + return build_capability_registry() + + +def test_capability_pack_ready_for_builtin(): + registry = _registry() + assert capability_pack_ready(registry, "issue-fix") is True + assert capability_pack_ready(registry, "issue_fix") is True # token form too + + +def test_capability_pack_ready_unknown_is_false(): + registry = _registry() + assert capability_pack_ready(registry, "does-not-exist") is False + + +def test_eligible_bridged_plain_tokens_unchanged(): + # No binding, no registry -> original token matching. + worker = {"capabilities": ["shell", "filesystem_read"]} + assert eligible_bridged(worker, {"required_capabilities": ["shell"]}) is True + assert eligible_bridged(worker, {"required_capabilities": ["network"]}) is False + + +def test_eligible_bridged_binding_requires_pack_token(): + worker = {"capabilities": ["issue_fix"]} + task = {"capability_binding_ref": "issue-fix:feasibility_v0"} + assert eligible_bridged(worker, task) is True + assert eligible_bridged({"capabilities": ["shell"]}, task) is False + + +def test_eligible_bridged_binding_gated_by_registry_ready(): + registry = _registry() + worker = {"capabilities": ["issue_fix"]} + task = {"capability_binding_ref": "issue-fix:feasibility_v0"} + assert eligible_bridged(worker, task, registry=registry) is True + # Unknown pack binding fails closed even if worker declares a token. + unknown_task = {"capability_binding_ref": "nope:xyz"} + assert eligible_bridged(worker, unknown_task, registry=registry) is False + + +def test_resolve_required_tokens_merges_binding(): + task = { + "required_capabilities": ["shell"], + "capability_binding_ref": "issue-fix:feasibility_v0", + } + tokens = resolve_required_tokens(task) + assert tokens == ["shell", "issue_fix"] + + +# --------------------------------------------------------------------------- +# P2: registry-driven CLI registration (real catalog) +# --------------------------------------------------------------------------- + + +def test_discover_cli_registrars_finds_real_packs(): + registry = _registry() + records = registry.records(include_internal=False) + registrars = discover_cli_registrars(records) + # issue-fix and change-quality-qualification have CLIs whose module names + # differ from their ids; the bridge must still find them. + assert "issue-fix" in registrars + assert "change-quality-qualification" in registrars + assert "integration-branch-reconcile" in registrars + + +def test_discover_cli_registrars_is_tolerant(): + # Records that lack implemented_protocols or a CLI are simply skipped. + registrars = discover_cli_registrars( + [ + {"id": "no-such-pack", "implemented_protocols": []}, + {"id": "bare-pack"}, + ] + ) + assert registrars == {} + + +class _FakeSubparsers: + def __init__(self): + self.registered: list[tuple] = [] + + def add_parser(self, name, **kwargs): + self.registered.append((name, kwargs)) + return _FakeParser() + + +class _FakeParser: + def add_argument(self, *args, **kwargs): + return None + + +def test_register_all_capability_commands_runs_registrars(): + # A synthetic pack with a single-arg registrar and one with the legacy + # two-arg shape both register without error. + calls: list[str] = [] + + def single_arg_registrar(subparsers): + calls.append("single") + + def two_arg_registrar(subparsers, add_format): + calls.append("two") + + import types + + fake_module_a = types.SimpleNamespace(register_commands=single_arg_registrar) + fake_module_b = types.SimpleNamespace(register_commands=two_arg_registrar) + + monkeypatch = pytest.MonkeyPatch() + import importlib + + monkeypatch.setattr( + importlib, + "import_module", + lambda p: fake_module_a if p.endswith(".a.cli") else fake_module_b, + ) + try: + registrars = register_all_capability_commands( + _FakeSubparsers(), + None, + capability_records=[ + {"id": "pack-a", "implemented_protocols": [{"module": "loopx.capabilities.a.core"}]}, + {"id": "pack-b", "implemented_protocols": [{"module": "loopx.capabilities.b.core"}]}, + ], + ) + finally: + monkeypatch.undo() + + assert len(registrars) == 2 + assert set(calls) == {"single", "two"} + + +def test_register_all_capability_commands_internal_type_error_not_retried(): + # A two-arg registrar that raises TypeError *inside* its body must not be + # re-invoked with a single argument (the old exception-probe fallback would + # mask this kind of bug by calling the registrar a second time). + calls: list[str] = [] + + def flaky_two_arg(subparsers, add_format): + calls.append("two") + raise TypeError("internal boom") + + import types + + fake_module = types.SimpleNamespace(register_commands=flaky_two_arg) + monkeypatch = pytest.MonkeyPatch() + import importlib + + monkeypatch.setattr(importlib, "import_module", lambda p: fake_module) + try: + with pytest.raises(TypeError, match="internal boom"): + register_all_capability_commands( + _FakeSubparsers(), + None, + capability_records=[ + {"id": "pack-flaky", "implemented_protocols": [{"module": "loopx.capabilities.flaky.core"}]}, + ], + ) + finally: + monkeypatch.undo() + + assert calls == ["two"] # invoked exactly once, no fallback re-try + + +# --------------------------------------------------------------------------- +# P3: event hub + hook registry +# --------------------------------------------------------------------------- + + +def test_capability_event_hub_publish_and_subscribe(): + hub = CapabilityEventHub() + received: list[dict] = [] + + def on_pr_merge(event): + received.append(dict(event)) + + hub.subscribe("pr_merge", on_pr_merge, source="issue-fix") + hub.publish("pr_merge", {"pr_ref": "owner/repo#1"}) + assert received == [{"pr_ref": "owner/repo#1"}] + assert hub.kinds() == ["pr_merge"] + + +def test_capability_event_hub_isolates_failures(): + hub = CapabilityEventHub() + + def broken(event): + raise RuntimeError("boom") + + def good(event): + return {"ok": True} + + hub.subscribe("task_completed", broken, source="bad") + hub.subscribe("task_completed", good, source="good") + results, errors = hub.publish("task_completed", {}) + # Results and errors are returned separately so callers can tell a broken + # subscriber apart from a legitimate result. + assert results == [{"ok": True}] + assert errors == [{"source": "bad", "error": "boom", "event_kind": "task_completed"}] + + +def test_capability_hook_registry_run_and_isolate(): + registry = CapabilityHookRegistry() + calls: list[str] = [] + + def hook_a(**kwargs): + calls.append("a") + return {"ok": True, "hook": "a"} + + def hook_b(**kwargs): + raise RuntimeError("boom") + + registry.register("pre_quota", hook_a, source="pack-a") + registry.register("pre_quota", hook_b, source="pack-b") + results = registry.run("pre_quota", goal_id="g1") + + assert calls == ["a"] + assert {"ok": True, "hook": "a"} in results + assert any(r.get("ok") is False and "boom" in r.get("error", "") for r in results) + assert registry.hook_points() == ["pre_quota"] diff --git a/tests/control_plane/test_checkpoint_replay.py b/tests/control_plane/test_checkpoint_replay.py new file mode 100644 index 000000000..5adf0d0dc --- /dev/null +++ b/tests/control_plane/test_checkpoint_replay.py @@ -0,0 +1,349 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from loopx.control_plane.runtime.checkpoint import ( + Checkpoint, + build_checkpoint, + compute_state_hash, + load_checkpoints, + load_latest_checkpoint, + remove_checkpoints, + verify_checkpoint_integrity, + write_checkpoint, +) +from loopx.control_plane.runtime.replay import ( + REPLAY_SCHEMA_VERSION, + ReplayViolationError, + load_task_events, + partition_events_after_checkpoint, + recover_task_state, + replay_audit_record, + replay_from_checkpoint, + replay_task, + state_digest, + verify_replay_equivalence, +) + +GOAL_ID = "checkpoint-fixture" +TODO_ID = "todo-1" + + +def _events() -> list[dict]: + return [ + {"event_id": "E1", "todo_id": TODO_ID, "delta": 1, "kind": "inc"}, + {"event_id": "E2", "todo_id": TODO_ID, "delta": 2, "kind": "inc"}, + {"event_id": "E3", "todo_id": TODO_ID, "delta": 3, "kind": "inc"}, + {"event_id": "E4", "todo_id": TODO_ID, "delta": 4, "kind": "inc"}, + ] + + +def _apply(state, event) -> dict: + return {"count": int(state.get("count") or 0) + int(event.get("delta") or 0)} + + +def _snapshot_after_events(events) -> dict: + return replay_task(events, apply=_apply) + + +# --------------------------------------------------------------------------- +# Checkpoint construction + integrity +# --------------------------------------------------------------------------- + + +def test_build_checkpoint_derives_hash_and_id() -> None: + checkpoint = build_checkpoint( + goal_id=GOAL_ID, + todo_id=TODO_ID, + run_id="run-1", + last_event_id="E2", + state_snapshot={"count": 3}, + ) + assert checkpoint.schema_version == REPLAY_SCHEMA_VERSION + assert checkpoint.state_hash == compute_state_hash({"count": 3}) + assert checkpoint.checkpoint_id.startswith(f"{GOAL_ID}:{TODO_ID}:E2:") + assert verify_checkpoint_integrity(checkpoint) + + +def test_checkpoint_round_trip() -> None: + checkpoint = build_checkpoint( + goal_id=GOAL_ID, + todo_id=TODO_ID, + run_id="run-1", + last_event_id="E2", + state_snapshot={"count": 3}, + ) + restored = Checkpoint.from_dict(checkpoint.to_dict()) + assert restored == checkpoint + + +def test_checkpoint_integrity_detects_tampering() -> None: + checkpoint = build_checkpoint( + goal_id=GOAL_ID, + todo_id=TODO_ID, + run_id="run-1", + last_event_id="E2", + state_snapshot={"count": 3}, + ) + tampered = Checkpoint( + checkpoint_id=checkpoint.checkpoint_id, + goal_id=checkpoint.goal_id, + todo_id=checkpoint.todo_id, + run_id=checkpoint.run_id, + last_event_id=checkpoint.last_event_id, + state_snapshot={"count": 999}, + state_hash=checkpoint.state_hash, + created_at=checkpoint.created_at, + ) + assert verify_checkpoint_integrity(tampered) is False + + +# --------------------------------------------------------------------------- +# Idempotent checkpoint persistence +# --------------------------------------------------------------------------- + + +def test_write_checkpoint_appends_once(tmp_path: Path) -> None: + checkpoint = build_checkpoint( + goal_id=GOAL_ID, + todo_id=TODO_ID, + run_id="run-1", + last_event_id="E2", + state_snapshot={"count": 3}, + ) + _, first_new = write_checkpoint(tmp_path, checkpoint) + _, second_new = write_checkpoint(tmp_path, checkpoint) + assert first_new is True + assert second_new is False + assert len(load_checkpoints(tmp_path, GOAL_ID, TODO_ID)) == 1 + + +def test_write_distinct_checkpoints_appends(tmp_path: Path) -> None: + first = build_checkpoint( + goal_id=GOAL_ID, todo_id=TODO_ID, run_id="run-1", last_event_id="E1", state_snapshot={"count": 1} + ) + second = build_checkpoint( + goal_id=GOAL_ID, todo_id=TODO_ID, run_id="run-1", last_event_id="E2", state_snapshot={"count": 3} + ) + write_checkpoint(tmp_path, first) + write_checkpoint(tmp_path, second) + assert len(load_checkpoints(tmp_path, GOAL_ID, TODO_ID)) == 2 + + +def test_load_latest_checkpoint_returns_newest(tmp_path: Path) -> None: + write_checkpoint( + tmp_path, + build_checkpoint( + goal_id=GOAL_ID, todo_id=TODO_ID, run_id="r1", last_event_id="E1", state_snapshot={"count": 1} + ), + ) + write_checkpoint( + tmp_path, + build_checkpoint( + goal_id=GOAL_ID, todo_id=TODO_ID, run_id="r1", last_event_id="E2", state_snapshot={"count": 3} + ), + ) + latest = load_latest_checkpoint(tmp_path, GOAL_ID, TODO_ID) + assert latest is not None + assert latest.last_event_id == "E2" + assert latest.state_snapshot == {"count": 3} + + +def test_remove_checkpoints(tmp_path: Path) -> None: + write_checkpoint( + tmp_path, + build_checkpoint( + goal_id=GOAL_ID, todo_id=TODO_ID, run_id="r1", last_event_id="E1", state_snapshot={"count": 1} + ), + ) + assert load_latest_checkpoint(tmp_path, GOAL_ID, TODO_ID) is not None + assert remove_checkpoints(tmp_path, GOAL_ID, TODO_ID) is True + assert load_latest_checkpoint(tmp_path, GOAL_ID, TODO_ID) is None + + +# --------------------------------------------------------------------------- +# Replay determinism + idempotency +# --------------------------------------------------------------------------- + + +def test_replay_is_deterministic() -> None: + events = _events() + assert replay_task(events, apply=_apply) == replay_task(events, apply=_apply) + + +def test_replay_side_effect_free_does_not_mutate_inputs() -> None: + events = _events() + snapshot = list(events) + replay_task(events, apply=_apply) + assert events == snapshot + + +def test_replay_is_idempotent_when_applied_twice_from_same_base() -> None: + events = _events() + first = replay_task(events, apply=_apply) + second = replay_task(events, apply=_apply, initial_state=first) + assert second["count"] == 20 # first pass: 1+2+3+4=10; second pass adds 10 more + + +def test_replay_from_checkpoint_equals_full_replay() -> None: + events = _events() + checkpoint = build_checkpoint( + goal_id=GOAL_ID, + todo_id=TODO_ID, + run_id="run-1", + last_event_id="E2", + state_snapshot=_snapshot_after_events(events[:2]), + ) + after = partition_events_after_checkpoint( + events, checkpoint, event_id_of=lambda e: str(e.get("event_id") or "") + ) + assert [e["event_id"] for e in after] == ["E3", "E4"] + recovered = replay_from_checkpoint(checkpoint, after, apply=_apply) + assert recovered == _snapshot_after_events(events) == {"count": 10} + + +def test_verify_replay_equivalence() -> None: + events = _events() + checkpoint = build_checkpoint( + goal_id=GOAL_ID, + todo_id=TODO_ID, + run_id="run-1", + last_event_id="E2", + state_snapshot=_snapshot_after_events(events[:2]), + ) + after = partition_events_after_checkpoint( + events, checkpoint, event_id_of=lambda e: str(e.get("event_id") or "") + ) + equivalent, message = verify_replay_equivalence( + events, + checkpoint=checkpoint, + events_after_checkpoint=after, + apply=_apply, + ) + assert equivalent is True + assert "equals" in message + + +def test_replay_without_checkpoint_starts_empty() -> None: + events = _events() + state = replay_from_checkpoint(None, events, apply=_apply) + assert state == {"count": 10} + + +def test_replay_schema_mismatch_raises() -> None: + checkpoint = Checkpoint( + checkpoint_id="x", + goal_id=GOAL_ID, + todo_id=TODO_ID, + run_id="r", + last_event_id="E1", + schema_version=REPLAY_SCHEMA_VERSION + 1, + state_snapshot={"count": 1}, + state_hash=compute_state_hash({"count": 1}), + ) + with pytest.raises(ReplayViolationError): + replay_from_checkpoint(checkpoint, [], apply=_apply) + + +def test_replay_tampered_checkpoint_raises() -> None: + checkpoint = Checkpoint( + checkpoint_id="x", + goal_id=GOAL_ID, + todo_id=TODO_ID, + run_id="r", + last_event_id="E1", + schema_version=REPLAY_SCHEMA_VERSION, + state_snapshot={"count": 999}, + state_hash=compute_state_hash({"count": 1}), + ) + with pytest.raises(ReplayViolationError): + replay_from_checkpoint(checkpoint, [], apply=_apply) + + +def test_partition_events_after_checkpoint_none_returns_all() -> None: + events = _events() + assert partition_events_after_checkpoint(events, None, event_id_of=lambda e: str(e.get("event_id") or "")) == events + + +# --------------------------------------------------------------------------- +# Full recovery workflow over a runtime root +# --------------------------------------------------------------------------- + + +def _write_index(tmp_path: Path, events: list[dict]) -> Path: + index_path = tmp_path / "goals" / GOAL_ID / "runs" / "index.jsonl" + index_path.parent.mkdir(parents=True, exist_ok=True) + index_path.write_text( + "\n".join(json.dumps(e) for e in events) + "\n", + encoding="utf-8", + ) + return index_path + + +def test_recover_task_state_full_flow(tmp_path: Path) -> None: + events = _events() + _write_index(tmp_path, events) + checkpoint = build_checkpoint( + goal_id=GOAL_ID, + todo_id=TODO_ID, + run_id="run-1", + last_event_id="E2", + state_snapshot=_snapshot_after_events(events[:2]), + ) + write_checkpoint(tmp_path, checkpoint) + + state, used, replayed = recover_task_state( + tmp_path, + goal_id=GOAL_ID, + todo_id=TODO_ID, + apply=_apply, + ) + assert used is not None + assert used.last_event_id == "E2" + assert [e["event_id"] for e in replayed] == ["E3", "E4"] + assert state == {"count": 10} + + +def test_recover_task_state_without_checkpoint(tmp_path: Path) -> None: + _write_index(tmp_path, _events()) + state, used, replayed = recover_task_state( + tmp_path, + goal_id=GOAL_ID, + todo_id=TODO_ID, + apply=_apply, + ) + assert used is None + assert len(replayed) == 4 + assert state == {"count": 10} + + +def test_load_task_events_filters_other_todos(tmp_path: Path) -> None: + _write_index( + tmp_path, + [ + {"event_id": "E1", "todo_id": TODO_ID, "delta": 1}, + {"event_id": "X1", "todo_id": "other", "delta": 99}, + ], + ) + events = load_task_events(tmp_path, GOAL_ID, TODO_ID) + assert [e["event_id"] for e in events] == ["E1"] + + +def test_replay_audit_record_shape() -> None: + record = replay_audit_record( + goal_id=GOAL_ID, + todo_id=TODO_ID, + state={"count": 10}, + checkpoint=None, + events_replayed=_events(), + events_total=4, + equivalent=True, + ) + assert record["goal_id"] == GOAL_ID + assert record["events_replayed"] == 4 + assert record["equivalent"] is True + assert record["state_hash"] == state_digest({"count": 10}) + assert "recorded_at" in record diff --git a/tests/control_plane/test_control_plane_observability.py b/tests/control_plane/test_control_plane_observability.py new file mode 100644 index 000000000..1a4330848 --- /dev/null +++ b/tests/control_plane/test_control_plane_observability.py @@ -0,0 +1,157 @@ +"""Tests for the control-plane observability snapshot (plan/new_plan.md §7, P2). + +Verifies the unified status view aggregates queue (incl. extended lifecycle), +worker, task, decision, and event history into one read-only digest. +""" + +from __future__ import annotations + +from pathlib import Path + +from loopx.control_plane.scheduler.event_driven_dispatch import ( + enqueue_tasks, + task_queue_path, +) +from loopx.control_plane.scheduler.task_lifecycle import ( + claim_next_eligible_task, + complete_task, + fail_task, +) +from loopx.control_plane.status.control_plane_observability import ( + build_control_plane_status, + build_decision_history, + build_event_history, + build_queue_digest, + build_task_history, + build_worker_status, +) +from loopx.rollout_event_log import rollout_event_log_path + + +def _seed_queue(root: Path, goal_id: str = "g1") -> Path: + q = task_queue_path(root, goal_id=goal_id) + enqueue_tasks( + q, + goal_id=goal_id, + todo_ids=["todo_a", "todo_b", "todo_c"], + recorded_at="2026-08-14T00:00:00Z", + ) + # todo_a -> done; todo_b -> claimed; todo_c -> retry_wait (zombie expires later). + claim_next_eligible_task(q, worker_id="w1", lease_seconds=100, now=1000.0) + complete_task(q, task_id="todo_a", worker_id="w1") + claim_next_eligible_task(q, worker_id="w1", lease_seconds=100, now=1000.0) + fail_task( + q, + task_id="todo_b", + worker_id="w1", + transient=True, + max_attempts=3, + retry_backoff_seconds=60, + now=1001.0, + ) + claim_next_eligible_task(q, worker_id="w1", lease_seconds=100, now=1000.0) + return q + + +def test_build_control_plane_status_aggregates_all_sections( + tmp_path: Path, +) -> None: + q = _seed_queue(tmp_path, goal_id="g1") + status = build_control_plane_status( + runtime_root=tmp_path, + goal_id="g1", + worker_ids=["w1"], + scheduler_tick_count=7, + ) + assert status["ok"] is True + assert status["goal_id"] == "g1" + # Scheduler section. + assert status["scheduler"]["tick_count"] == 7 + assert status["scheduler"]["worker_ids"] == ["w1"] + # Queue section reflects the extended lifecycle. + queue = status["queue"] + assert queue["pending_count"] == 0 + assert queue["done_count"] == 1 + assert queue["extended"]["retry_wait_count"] == 1 + assert queue["in_flight_count"] == 2 # claimed + retry_wait + # Worker section. + assert status["workers"]["worker_count"] == 1 + # Task history is present. + assert len(status["task_history"]) == 3 + # Event history section exists. + assert "counts_by_kind" in status["event_history"] + # Queue file was not mutated by the read-only snapshot. + assert q.exists() + + +def test_build_queue_digest_counts_done_and_exception(tmp_path: Path) -> None: + q = _seed_queue(tmp_path, goal_id="g1") + digest = build_queue_digest(q) + assert digest["done_count"] == 1 + assert "todo_a" in digest["done_todo_ids"] + assert digest["in_flight_count"] == 2 + # No failures/dead-letters in this scenario. + assert digest["exception_count"] == 0 + + +def test_build_worker_status_derives_in_flight(tmp_path: Path) -> None: + _seed_queue(tmp_path, goal_id="g1") + q = task_queue_path(tmp_path, goal_id="g1") + from loopx.control_plane.status.control_plane_observability import ( + _read_queue_entries, + ) + + worker_status = build_worker_status(_read_queue_entries(q)) + assert worker_status["worker_count"] == 1 + assert worker_status["workers"][0]["worker_id"] == "w1" + assert worker_status["workers"][0]["in_flight_count"] == 2 + + +def test_build_task_history_orders_newest_first(tmp_path: Path) -> None: + q = _seed_queue(tmp_path, goal_id="g1") + from loopx.control_plane.status.control_plane_observability import ( + _read_queue_entries, + ) + + history = build_task_history(_read_queue_entries(q)) + # 3 tasks: done, retry_wait, claimed. + assert len(history) == 3 + statuses = {h["status"] for h in history} + assert "done" in statuses + assert "retry_wait" in statuses + assert "claimed" in statuses + + +def test_build_event_history_empty_when_no_log(tmp_path: Path) -> None: + log_path = rollout_event_log_path(tmp_path, goal_id="g1") + digest = build_event_history(log_path) + assert digest["event_count"] == 0 + assert digest["recent_events"] == [] + + +def test_build_decision_history_empty_when_no_ledger(tmp_path: Path) -> None: + digest = build_decision_history(tmp_path / "no-such-decision.jsonl") + assert digest["ok"] is True + assert digest["decision_count"] == 0 + + +def test_build_decision_history_parses_ledger(tmp_path: Path) -> None: + ledger = tmp_path / "decisions.jsonl" + lines = [ + '{"event_id":"e1","goal_id":"g1","todo_id":"t1","outcome":"run","source":"quota","recorded_at":"2026-08-14T00:00:00Z"}', + '{"event_id":"e2","goal_id":"g1","todo_id":"t2","outcome":"deny","source":"capability","recorded_at":"2026-08-14T00:01:00Z"}', + ] + ledger.write_text("\n".join(lines) + "\n", encoding="utf-8") + digest = build_decision_history(ledger) + assert digest["decision_count"] == 2 + assert digest["counts_by_outcome"] == {"run": 1, "deny": 1} + assert digest["recent_decisions"][0]["outcome"] == "deny" + + +def test_build_control_plane_status_missing_logs_is_safe(tmp_path: Path) -> None: + # Empty runtime root -> no queue, no event log; snapshot still succeeds. + status = build_control_plane_status(runtime_root=tmp_path, goal_id="ghost") + assert status["ok"] is True + assert status["queue"]["pending_count"] == 0 + assert status["workers"]["worker_count"] == 0 + assert status["event_history"]["event_count"] == 0 diff --git a/tests/control_plane/test_cost_projection.py b/tests/control_plane/test_cost_projection.py new file mode 100644 index 000000000..07b113a3d --- /dev/null +++ b/tests/control_plane/test_cost_projection.py @@ -0,0 +1,195 @@ +from __future__ import annotations + +from pathlib import Path + +from loopx.control_plane.quota.cost_projection import ( + QUOTA_SLOT_SPENT_CLASSIFICATION, + goal_cost_summary, + load_all_spend_facts, + load_goal_spend_facts, + project_usage_summary, + spend_fact, + spend_facts, + task_cost, +) + +GOAL_ID = "cost-projection-fixture" + + +def _spend_run( + *, + todo_id: str, + agent_id: str, + slots: int, + generated_at: str, + source: str = "eligible", +) -> dict: + return { + "generated_at": generated_at, + "goal_id": GOAL_ID, + "classification": QUOTA_SLOT_SPENT_CLASSIFICATION, + "agent_id": agent_id, + "quota_event": { + "event_type": QUOTA_SLOT_SPENT_CLASSIFICATION, + "source": source, + "todo_id": todo_id, + "slots": slots, + "before": {"spent_slots": 0}, + "after": {"spent_slots": slots}, + }, + } + + +def _facts() -> list[dict]: + return [ + _spend_run(todo_id="t1", agent_id="a1", slots=3, generated_at="2026-08-13T10:00:00Z"), + _spend_run(todo_id="t1", agent_id="a1", slots=2, generated_at="2026-08-13T11:00:00Z"), + _spend_run(todo_id="t2", agent_id="a2", slots=5, generated_at="2026-08-14T09:00:00Z"), + ] + + +# --------------------------------------------------------------------------- +# spend_fact normalization +# --------------------------------------------------------------------------- + + +def test_spend_fact_normalizes_quota_event_shape() -> None: + fact = spend_fact(_spend_run(todo_id="t1", agent_id="a1", slots=4, generated_at="2026-08-13T12:00:00Z")) + assert fact is not None + assert fact["goal_id"] == GOAL_ID + assert fact["todo_id"] == "t1" + assert fact["agent_id"] == "a1" + assert fact["usage_units"] == 4 + assert fact["day"] == "2026-08-13" + assert fact["source"] == "eligible" + + +def test_spend_fact_none_for_non_spend_event() -> None: + assert spend_fact({"goal_id": GOAL_ID, "classification": "state_refreshed"}) is None + + +def test_spend_fact_supports_rollout_event_shape() -> None: + event = { + "goal_id": GOAL_ID, + "event_kind": "quota_spend", + "agent_id": "a1", + "todo_id": "t1", + "recorded_at": "2026-08-15T08:00:00Z", + "details": {"slots": 7, "source": "delivery"}, + } + fact = spend_fact(event) + assert fact is not None + assert fact["usage_units"] == 7 + assert fact["day"] == "2026-08-15" + assert fact["source"] == "delivery" + + +def test_spend_facts_filters_stream() -> None: + events = [ + _spend_run(todo_id="t1", agent_id="a1", slots=1, generated_at="2026-08-13T10:00:00Z"), + {"goal_id": GOAL_ID, "classification": "state_refreshed"}, + {"goal_id": GOAL_ID, "event_kind": "quota_should_run"}, + ] + facts = spend_facts(events) + assert len(facts) == 1 + assert facts[0]["usage_units"] == 1 + + +# --------------------------------------------------------------------------- +# goal_cost_summary +# --------------------------------------------------------------------------- + + +def test_goal_cost_summary_total_and_dimensions() -> None: + summary = goal_cost_summary(GOAL_ID, _facts()) + assert summary["goal_id"] == GOAL_ID + assert summary["total_usage"] == 10 + assert summary["by_agent"] == {"a2": 5, "a1": 5} + assert summary["by_task"] == {"t2": 5, "t1": 5} + assert summary["by_day"] == {"2026-08-14": 5, "2026-08-13": 5} + assert summary["by_source"] == {"eligible": 10} + + +def test_goal_cost_summary_ignores_other_goal() -> None: + summary = goal_cost_summary("other-goal", _facts()) + assert summary["total_usage"] == 0 + assert summary["by_agent"] == {} + assert summary["by_day"] == {} + + +def test_goal_cost_summary_no_monetary_cost_by_default() -> None: + summary = goal_cost_summary(GOAL_ID, _facts()) + assert "monetary_cost" not in summary + assert summary["usage_units"] == 10 + + +def test_goal_cost_summary_accepts_precomputed_facts() -> None: + facts = spend_facts(_facts()) + summary = goal_cost_summary(GOAL_ID, facts=facts) + assert summary["total_usage"] == 10 + + +# --------------------------------------------------------------------------- +# task_cost +# --------------------------------------------------------------------------- + + +def test_task_cost_aggregates_single_todo() -> None: + summary = task_cost(GOAL_ID, "t1", _facts()) + assert summary["total_usage"] == 5 + assert summary["by_agent"] == {"a1": 5} + + +def test_task_cost_unknown_todo_is_zero() -> None: + summary = task_cost(GOAL_ID, "missing", _facts()) + assert summary["total_usage"] == 0 + assert summary["by_agent"] == {} + + +# --------------------------------------------------------------------------- +# File loading helpers +# --------------------------------------------------------------------------- + + +def _write_index(tmp_path: Path, goal_id: str, records: list[dict]) -> Path: + index_path = tmp_path / "goals" / goal_id / "runs" / "index.jsonl" + index_path.parent.mkdir(parents=True, exist_ok=True) + import json + + index_path.write_text( + "\n".join(json.dumps(r) for r in records) + "\n", + encoding="utf-8", + ) + return index_path + + +def test_load_goal_spend_facts_from_runtime(tmp_path: Path) -> None: + _write_index(tmp_path, GOAL_ID, _facts()) + facts = load_goal_spend_facts(tmp_path, GOAL_ID) + assert len(facts) == 3 + assert sum(f["usage_units"] for f in facts) == 10 + + +def test_load_all_spend_facts_across_goals(tmp_path: Path) -> None: + _write_index(tmp_path, GOAL_ID, _facts()) + _write_index(tmp_path, "g2", [_spend_run(todo_id="t9", agent_id="a9", slots=2, generated_at="2026-08-13T10:00:00Z")]) + facts = load_all_spend_facts(tmp_path) + assert len(facts) == 4 + assert sum(f["usage_units"] for f in facts) == 12 + + +def test_project_usage_summary(tmp_path: Path) -> None: + _write_index(tmp_path, GOAL_ID, _facts()) + summary = project_usage_summary(tmp_path) + assert summary["total_usage"] == 10 + assert summary["by_goal"] == {GOAL_ID: 10} + + +def test_project_usage_summary_as_of_limit(tmp_path: Path) -> None: + _write_index(tmp_path, GOAL_ID, _facts()) + summary = project_usage_summary(tmp_path, as_of="2026-08-13") + assert summary["total_usage"] == 5 + + +def test_load_goal_spend_facts_missing_goal_is_empty(tmp_path: Path) -> None: + assert load_goal_spend_facts(tmp_path, "nonexistent") == [] diff --git a/tests/control_plane/test_event_driven_dispatch.py b/tests/control_plane/test_event_driven_dispatch.py new file mode 100644 index 000000000..94bb9e275 --- /dev/null +++ b/tests/control_plane/test_event_driven_dispatch.py @@ -0,0 +1,789 @@ +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any + +import pytest + +from loopx.control_plane.scheduler.event_driven_dispatch import ( + EVENT_DRIVEN_DISPATCH_ENV, + QUEUE_STATUS_CLAIMED, + QUEUE_STATUS_PENDING, + TASK_DISPATCHED_EVENT_KIND, + TASK_ENQUEUED_EVENT_KIND, + TASK_QUEUE_ENTRY_SCHEMA_VERSION, + TASK_READY_EVENT_KIND, + advance_ready_todo_ids, + build_event_driven_dispatch, + claim_next_task, + enqueue_tasks, + event_driven_dispatch_enabled, + load_task_queue, + record_task_event, + task_queue_path, +) +from loopx.rollout_event_log import load_rollout_events, rollout_event_log_path + + +def _gate_items() -> list[dict[str, Any]]: + """todo_first is a completed handoff gate that unlocks advancement todo_second.""" + return [ + { + "todo_id": "todo_first", + "text": "setup done", + "status": "done", + "excluded_agents": ["agent_worker"], + "unblocks_todo_id": "todo_second", + }, + { + "todo_id": "todo_second", + "text": "followup advancement", + "task_class": "advancement_task", + "unblocks_todo_id": "todo_first", + "status": "open", + }, + ] + + +def test_advance_ready_todo_ids_pure() -> None: + ready = advance_ready_todo_ids(_gate_items()) + assert ready == ["todo_second"] + + +def test_advance_ready_todo_ids_blocked_when_gate_open() -> None: + items = _gate_items() + items[0]["status"] = "open" + assert advance_ready_todo_ids(items) == [] + + +def test_advance_ready_unconstrained_open_advancement_todo() -> None: + # An independent open advancement todo with no handoff gate dependency must + # still be READY so a resident Worker can claim it (RFC "initial READY todos"). + items = [ + { + "todo_id": "todo_solo", + "text": "independent advancement", + "task_class": "advancement_task", + "status": "open", + }, + ] + assert advance_ready_todo_ids(items) == ["todo_solo"] + + +def test_advance_ready_skips_open_handoff_gate_itself() -> None: + # An open gate (has excluded_agents) must not be advanced as a free task. + items = [ + { + "todo_id": "todo_gate", + "text": "await user", + "status": "open", + "excluded_agents": ["agent_worker"], + "unblocks_todo_id": "todo_next", + }, + { + "todo_id": "todo_next", + "text": "followup", + "task_class": "advancement_task", + "status": "open", + }, + ] + # todo_next is gated by todo_gate (open) so neither is READY yet. + assert advance_ready_todo_ids(items) == [] + + +def test_advance_ready_skips_done_and_gated_successor() -> None: + items = [ + { + "todo_id": "todo_done_solo", + "text": "already done", + "task_class": "advancement_task", + "status": "done", + }, + { + "todo_id": "todo_gate", + "text": "await user", + "status": "done", + "excluded_agents": ["agent_worker"], + "unblocks_todo_id": "todo_gated", + }, + { + "todo_id": "todo_gated", + "text": "gated followup", + "task_class": "advancement_task", + "status": "open", + "unblocks_todo_id": "todo_gate", + }, + { + "todo_id": "todo_free", + "text": "free advancement", + "task_class": "advancement_task", + "status": "open", + }, + ] + # todo_gated is a cleared gate's successor (READY); todo_free is unconstrained. + assert advance_ready_todo_ids(items) == ["todo_free", "todo_gated"] + + +def test_advance_ready_excludes_terminal_gate_successor( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Regression: a handoff gate successor already 'done' must NOT be READY. + + This reproduces the website1 color session bug: stale successors left over + from a previous (font) task were enqueued and claimed by the event-driven + scheduler even though the authoritative markdown state considered them done. + """ + items = [ + { + "todo_id": "todo_font_done", # stale successor, already done + "text": "old font task that is finished", + "task_class": "advancement_task", + "status": "done", + "unblocks_todo_id": "todo_gate", + }, + { + "todo_id": "todo_gate", + "text": "setup gate", + "status": "done", + "excluded_agents": ["agent_worker"], + "unblocks_todo_id": "todo_font_done", + }, + { + "todo_id": "todo_color", + "text": "change color to green", + "task_class": "advancement_task", + "status": "open", + }, + ] + ready = advance_ready_todo_ids(items) + # Only the current open color task is READY; the done font successor is not. + assert "todo_font_done" not in ready + assert ready == ["todo_color"] + + +def test_build_event_driven_dispatch_skips_terminal_successor( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """End-to-end: a done successor is never enqueued or claimed.""" + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + goal_id = "dispatch-terminal" + event_log = rollout_event_log_path(tmp_path, goal_id) + items = [ + { + "todo_id": "todo_gate", + "text": "setup gate", + "status": "done", + "excluded_agents": ["agent_worker"], + "unblocks_todo_id": "todo_stale", + }, + { + "todo_id": "todo_stale", + "text": "stale done successor", + "task_class": "advancement_task", + "status": "done", + "unblocks_todo_id": "todo_gate", + }, + ] + payload = build_event_driven_dispatch( + runtime_root=tmp_path, + goal_id=goal_id, + items=items, + completed_todo_id="todo_gate", + event_log_path=event_log, + worker_id="worker_one", + recorded_at="2026-08-14T00:00:00Z", + ) + dispatch = payload["event_driven_dispatch"] + # The stale done successor must not be enqueued nor dispatched. + assert "todo_stale" not in dispatch.get("newly_enqueued", []) + assert dispatch.get("dispatched") is None + + +def test_event_driven_dispatch_emits_closure_when_no_ready_successors( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """When no ready successors remain, the Closure Evaluator emits goal_closure_ready.""" + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + goal_id = "dispatch-closure" + event_log = rollout_event_log_path(tmp_path, goal_id) + items = [ + { + "todo_id": "todo_gate", + "text": "setup gate", + "status": "done", + "excluded_agents": ["agent_worker"], + "unblocks_todo_id": "todo_only", + }, + { + "todo_id": "todo_only", + "text": "the only advancement", + "task_class": "advancement_task", + "status": "done", # already done -> not READY + "unblocks_todo_id": "todo_gate", + }, + ] + payload = build_event_driven_dispatch( + runtime_root=tmp_path, + goal_id=goal_id, + items=items, + completed_todo_id="todo_gate", + event_log_path=event_log, + recorded_at="2026-08-14T00:00:00Z", + ) + closure = (payload["event_driven_dispatch"] or {}).get("closure") + assert closure is not None + assert closure.get("ready") is True + assert closure.get("reason") == "no_followup_work" + # The goal_closure_ready event is actually recorded. + kinds = [e["event_kind"] for e in load_rollout_events(event_log, limit=10)] + assert "goal_closure_ready" in kinds + + +def test_event_driven_dispatch_no_closure_when_ready_work_remains( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + goal_id = "dispatch-no-close" + event_log = rollout_event_log_path(tmp_path, goal_id) + payload = build_event_driven_dispatch( + runtime_root=tmp_path, + goal_id=goal_id, + items=_gate_items(), # todo_second is READY + completed_todo_id="todo_first", + event_log_path=event_log, + recorded_at="2026-08-14T00:00:00Z", + ) + # With ready work remaining, closure is NOT evaluated (None) — no premature close. + closure = (payload["event_driven_dispatch"] or {}).get("closure") + assert closure is None + kinds = [e["event_kind"] for e in load_rollout_events(event_log, limit=10)] + assert "goal_closure_ready" not in kinds + + +def test_event_driven_dispatch_no_close_when_blocked_todo_remains( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A blocked (unfinished) todo must block goal closure, even with no READY successors.""" + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + goal_id = "dispatch-blocked" + event_log = rollout_event_log_path(tmp_path, goal_id) + items = [ + {"todo_id": "todo_done", "status": "done", "goal_id": goal_id}, + {"todo_id": "todo_blocked", "status": "blocked", "goal_id": goal_id}, + ] + payload = build_event_driven_dispatch( + runtime_root=tmp_path, + goal_id=goal_id, + items=items, + event_log_path=event_log, + recorded_at="2026-08-14T00:00:00Z", + ) + closure = (payload["event_driven_dispatch"] or {}).get("closure") + assert closure is not None + assert closure.get("ready") is False + assert closure.get("reason") == "blocked_work_pending" + kinds = [e["event_kind"] for e in load_rollout_events(event_log, limit=10)] + assert "goal_closure_ready" not in kinds + assert "goal_closed" not in kinds + + +def test_event_driven_dispatch_no_close_when_deferred_todo_remains( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A deferred (unscheduled) todo must block goal closure.""" + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + goal_id = "dispatch-deferred" + event_log = rollout_event_log_path(tmp_path, goal_id) + items = [ + {"todo_id": "todo_done", "status": "done", "goal_id": goal_id}, + {"todo_id": "todo_deferred", "status": "deferred", "goal_id": goal_id}, + ] + payload = build_event_driven_dispatch( + runtime_root=tmp_path, + goal_id=goal_id, + items=items, + event_log_path=event_log, + recorded_at="2026-08-14T00:00:00Z", + ) + closure = (payload["event_driven_dispatch"] or {}).get("closure") + assert closure is not None + assert closure.get("ready") is False + assert closure.get("reason") == "deferred_work_pending" + kinds = [e["event_kind"] for e in load_rollout_events(event_log, limit=10)] + assert "goal_closure_ready" not in kinds + assert "goal_closed" not in kinds + + +def test_event_driven_dispatch_acceptance_closes_goal_in_one_tick( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """With acceptance criteria + evidence satisfied and no work left, one dispatch + atomically emits goal_closure_ready AND goal_closed (no manual goal-closure).""" + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + goal_id = "dispatch-one-tick-close" + event_log = rollout_event_log_path(tmp_path, goal_id) + items = [{"todo_id": "todo_done", "status": "done", "goal_id": goal_id}] + payload = build_event_driven_dispatch( + runtime_root=tmp_path, + goal_id=goal_id, + items=items, + event_log_path=event_log, + recorded_at="2026-08-14T00:00:00Z", + acceptance_criteria=[{"criterion_id": "c1", "description": "done"}], + evidence=[{"criterion_ids": ["c1"], "kind": "grep", "ref": "f", "ok": True}], + ) + closure = (payload["event_driven_dispatch"] or {}).get("closure") + assert closure is not None + assert closure.get("ready") is True + assert closure.get("tri_state") == "CLOSE" + kinds = [e["event_kind"] for e in load_rollout_events(event_log, limit=20)] + assert "goal_closure_ready" in kinds + assert "goal_closed" in kinds + + +def test_event_driven_dispatch_close_ignores_non_advancement_open_todos( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """An open continuous_monitor / user_gate / user_action todo must NOT block + goal closure: it is not executable advancement work and lives on its own + lifecycle. (Regression: these were previously miscounted as `ready_todo_ids`, + wedging the goal in RUN forever.)""" + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + goal_id = "dispatch-non-advancement" + event_log = rollout_event_log_path(tmp_path, goal_id) + items = [ + {"todo_id": "todo_done", "status": "done", "goal_id": goal_id, + "task_class": "advancement_task", "action_kind": "edit"}, + {"todo_id": "todo_mon", "status": "open", "goal_id": goal_id, + "task_class": "continuous_monitor"}, + {"todo_id": "todo_gate", "status": "open", "goal_id": goal_id, + "task_class": "user_gate"}, + ] + payload = build_event_driven_dispatch( + runtime_root=tmp_path, + goal_id=goal_id, + items=items, + event_log_path=event_log, + recorded_at="2026-08-14T00:00:00Z", + ) + dispatch = payload["event_driven_dispatch"] + assert dispatch["ready_successors"] == [] + closure = dispatch.get("closure") + assert closure is not None + assert closure.get("ready") is True + assert closure.get("tri_state") == "CLOSE" + + +def test_event_driven_dispatch_open_advancement_still_blocks_close( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """An open advancement todo still blocks closure (executable work remains).""" + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + goal_id = "dispatch-open-advancement" + event_log = rollout_event_log_path(tmp_path, goal_id) + items = [ + {"todo_id": "todo_done", "status": "done", "goal_id": goal_id, + "task_class": "advancement_task", "action_kind": "edit"}, + {"todo_id": "todo_open", "status": "open", "goal_id": goal_id, + "task_class": "advancement_task", "action_kind": "edit"}, + ] + payload = build_event_driven_dispatch( + runtime_root=tmp_path, + goal_id=goal_id, + items=items, + event_log_path=event_log, + recorded_at="2026-08-14T00:00:00Z", + ) + dispatch = payload["event_driven_dispatch"] + assert dispatch["ready_successors"] == ["todo_open"] + assert dispatch.get("closure") is None + + +def test_event_driven_dispatch_disabled_writes_nothing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv(EVENT_DRIVEN_DISPATCH_ENV, raising=False) + goal_id = "dispatch-disabled" + event_log = rollout_event_log_path(tmp_path, goal_id) + payload = build_event_driven_dispatch( + runtime_root=tmp_path, + goal_id=goal_id, + items=_gate_items(), + completed_todo_id="todo_first", + event_log_path=event_log, + worker_id="worker_one", + use_event_driven=False, + ) + assert payload.get("disabled") is True + assert payload.get("ok") is True + # No queue file, no rollout events written. + assert not task_queue_path(tmp_path, goal_id=goal_id).exists() + assert not event_log.exists() + + +def test_event_driven_dispatch_enabled_full_path( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + goal_id = "dispatch-on" + event_log = rollout_event_log_path(tmp_path, goal_id) + payload = build_event_driven_dispatch( + runtime_root=tmp_path, + goal_id=goal_id, + items=_gate_items(), + completed_todo_id="todo_first", + event_log_path=event_log, + worker_id="worker_one", + recorded_at="2026-08-13T00:00:00Z", + ) + assert payload.get("disabled") is not True + dispatch = payload["event_driven_dispatch"] + assert dispatch["ready_successors"] == ["todo_second"] + assert dispatch["newly_enqueued"] == ["todo_second"] + assert dispatch["dispatched"] == { + "todo_id": "todo_second", + "claimed_by": "worker_one", + "status": QUEUE_STATUS_CLAIMED, + } + queue = dispatch["queue"] + assert queue["pending_count"] == 0 + assert queue["claimed_count"] == 1 + assert queue["claimed_todo_ids"] == ["todo_second"] + + # Public audit events: task_ready, task_enqueued, task_dispatched. + events = load_rollout_events(event_log) + kinds = [event.get("event_kind") for event in events] + assert TASK_READY_EVENT_KIND in kinds + assert TASK_ENQUEUED_EVENT_KIND in kinds + assert TASK_DISPATCHED_EVENT_KIND in kinds + for event in events: + assert event.get("goal_id") == goal_id + + +def test_event_driven_dispatch_records_registered_agent_id_on_dispatched( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + goal_id = "dispatch-agent" + event_log = rollout_event_log_path(tmp_path, goal_id) + build_event_driven_dispatch( + runtime_root=tmp_path, + goal_id=goal_id, + items=_gate_items(), + completed_todo_id="todo_first", + event_log_path=event_log, + worker_id="worker_one", + agent_id="agent_registered", + recorded_at="2026-08-13T00:00:00Z", + ) + events = load_rollout_events(event_log) + dispatched = [e for e in events if e.get("event_kind") == TASK_DISPATCHED_EVENT_KIND] + assert len(dispatched) == 1 + # The registered LoopX agent identity is recorded, not the raw claimer. + assert dispatched[0].get("agent_id") == "agent_registered" + + +def test_event_driven_dispatch_agent_id_falls_back_to_worker_id( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + goal_id = "dispatch-agent-fallback" + event_log = rollout_event_log_path(tmp_path, goal_id) + build_event_driven_dispatch( + runtime_root=tmp_path, + goal_id=goal_id, + items=_gate_items(), + completed_todo_id="todo_first", + event_log_path=event_log, + worker_id="worker_one", + recorded_at="2026-08-13T00:00:00Z", + ) + events = load_rollout_events(event_log) + dispatched = [e for e in events if e.get("event_kind") == TASK_DISPATCHED_EVENT_KIND] + assert len(dispatched) == 1 + # Without an explicit agent_id, the claimer (worker_id) is recorded, + # preserving legacy behavior. + assert dispatched[0].get("agent_id") == "worker_one" + + +def test_event_driven_dispatch_idempotent( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + goal_id = "dispatch-idem" + event_log = rollout_event_log_path(tmp_path, goal_id) + kwargs: dict[str, Any] = dict( + runtime_root=tmp_path, + goal_id=goal_id, + items=_gate_items(), + completed_todo_id="todo_first", + event_log_path=event_log, + recorded_at="2026-08-13T00:00:00Z", + ) + first = build_event_driven_dispatch(**kwargs) + second = build_event_driven_dispatch(**kwargs) + assert first["event_driven_dispatch"]["newly_enqueued"] == ["todo_second"] + # Second tick: the READY successor is already queued, so nothing new is + # enqueued and the queue is not duplicated. + assert second["event_driven_dispatch"]["newly_enqueued"] == [] + assert second["event_driven_dispatch"]["queue"]["pending_count"] == 1 + # Events remain deduplicated by (goal_id, event_kind, todo_id). + events = load_rollout_events(event_log) + ready_count = sum(1 for e in events if e.get("event_kind") == TASK_READY_EVENT_KIND) + enqueued_count = sum(1 for e in events if e.get("event_kind") == TASK_ENQUEUED_EVENT_KIND) + assert ready_count == 1 + assert enqueued_count == 1 + + +def test_enqueue_tasks_idempotent_and_claim( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + queue = task_queue_path(tmp_path, goal_id="queue") + first = enqueue_tasks( + queue, + goal_id="queue", + todo_ids=["todo_a", "todo_b"], + recorded_at="2026-08-13T00:00:00Z", + ) + assert first["newly_enqueued"] == ["todo_a", "todo_b"] + second = enqueue_tasks( + queue, + goal_id="queue", + todo_ids=["todo_b", "todo_c"], + recorded_at="2026-08-13T00:00:00Z", + ) + assert second["newly_enqueued"] == ["todo_c"] + assert second["skipped_duplicates"] == ["todo_b"] + + view = load_task_queue(queue) + assert view["pending_count"] == 3 + assert view["pending_todo_ids"] == ["todo_a", "todo_b", "todo_c"] + + claimed = claim_next_task(queue, worker_id="worker_one") + assert claimed is not None + assert claimed["todo_id"] == "todo_a" + assert claimed["status"] == QUEUE_STATUS_CLAIMED + assert claimed["claimed_by"] == "worker_one" + view = load_task_queue(queue) + assert view["claimed_todo_ids"] == ["todo_a"] + assert view["pending_todo_ids"] == ["todo_b", "todo_c"] + + +def test_record_task_event_idempotent(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + event_log = rollout_event_log_path(tmp_path, "events") + first = record_task_event( + event_log, + goal_id="events", + event_kind=TASK_READY_EVENT_KIND, + todo_id="todo_x", + recorded_at="2026-08-13T00:00:00Z", + ) + second = record_task_event( + event_log, + goal_id="events", + event_kind=TASK_READY_EVENT_KIND, + todo_id="todo_x", + recorded_at="2026-08-13T00:00:01Z", + ) + assert first["new"] is True + assert second["new"] is False + assert len(load_rollout_events(event_log)) == 1 + + +def test_queue_entries_schema_version(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + queue = task_queue_path(tmp_path, goal_id="schema") + enqueue_tasks(queue, goal_id="schema", todo_ids=["todo_x"], recorded_at="2026-08-13T00:00:00Z") + raw = queue.read_text(encoding="utf-8").strip().splitlines() + entry = json.loads(raw[0]) + assert entry["schema_version"] == TASK_QUEUE_ENTRY_SCHEMA_VERSION + assert entry["status"] == QUEUE_STATUS_PENDING + + +def test_claim_next_task_empty_queue(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + queue = task_queue_path(tmp_path, goal_id="empty") + assert claim_next_task(queue, worker_id="worker_one") is None + + +def test_claim_next_task_with_capability_and_lease( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + from loopx.control_plane.scheduler.task_lifecycle import claim_next_eligible_task + + queue = task_queue_path(tmp_path, goal_id="cap") + enqueue_tasks( + queue, + goal_id="cap", + todo_ids=["todo_gpu", "todo_py"], + recorded_at="2026-08-14T00:00:00Z", + ) + entries = [ + json.loads(line) for line in queue.read_text(encoding="utf-8").splitlines() if line.strip() + ] + entries[0]["required_capabilities"] = ["gpu"] + entries[1]["required_capabilities"] = ["python"] + queue.write_text( + "".join(json.dumps(e, sort_keys=True) + "\n" for e in entries), + encoding="utf-8", + ) + # A python-only worker cannot claim the gpu task; claims the python task with a lease. + claimed = claim_next_task( + queue, + worker_id="worker_py", + capabilities=["python"], + lease_seconds=120, + ) + assert claimed is not None + assert claimed["todo_id"] == "todo_py" + assert claimed["status"] == QUEUE_STATUS_CLAIMED + assert claimed["lease_until"] is not None + assert claimed["attempt"] == 1 + + +def test_claim_next_task_binding_requires_pack_token( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A legacy capability_binding_ref requires the bound pack token in claim.""" + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + queue = task_queue_path(tmp_path, goal_id="binding-token") + enqueue_tasks( + queue, + goal_id="binding-token", + todo_ids=["todo_bound"], + recorded_at="2026-08-14T00:00:00Z", + ) + entries = [ + json.loads(line) + for line in queue.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + entries[0]["capability_binding_ref"] = "issue-fix:feasibility_v0" + queue.write_text( + "".join(json.dumps(e, sort_keys=True) + "\n" for e in entries), + encoding="utf-8", + ) + # A worker without the pack token cannot claim the bound task. + assert claim_next_task(queue, worker_id="worker_shell", capabilities=["shell"]) is None + # A worker declaring the pack token can. + claimed = claim_next_task(queue, worker_id="worker_issue", capabilities=["issue_fix"]) + assert claimed is not None + assert claimed["todo_id"] == "todo_bound" + assert claimed["claimed_by"] == "worker_issue" + + +def test_claim_next_task_binding_fails_closed_for_unknown_pack( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A binding to a pack unknown to the registry is not claimable at all.""" + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + queue = task_queue_path(tmp_path, goal_id="binding-unknown") + enqueue_tasks( + queue, + goal_id="binding-unknown", + todo_ids=["todo_mystery"], + recorded_at="2026-08-14T00:00:00Z", + ) + entries = [ + json.loads(line) + for line in queue.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + entries[0]["capability_binding_ref"] = "unknown-pack:xyz" + queue.write_text( + "".join(json.dumps(e, sort_keys=True) + "\n" for e in entries), + encoding="utf-8", + ) + # Even a worker declaring a plausible token cannot claim a pack that the + # registry does not know (fail closed). + assert claim_next_task(queue, worker_id="worker_unk", capabilities=["unknown_pack"]) is None + + +def test_build_event_driven_dispatch_reconciles_zombie_lease( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from loopx.control_plane.scheduler.task_lifecycle import claim_next_eligible_task + + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + goal_id = "dispatch-reconcile" + queue = task_queue_path(tmp_path, goal_id=goal_id) + # Seed a zombie: enqueue + claim with a short lease, then advance time past it. + enqueue_tasks(queue, goal_id=goal_id, todo_ids=["todo_first"], recorded_at="2026-08-14T00:00:00Z") + claim_next_eligible_task(queue, worker_id="w1", lease_seconds=10, now=1000.0) + payload = build_event_driven_dispatch( + runtime_root=tmp_path, + goal_id=goal_id, + items=_gate_items(), + completed_todo_id="todo_first", + recorded_at="2026-08-14T00:11:00Z", + reconcile=True, + ) + # The zombie lease is expired by the reconcile pass and re-enqueued. + reconcile = (payload["event_driven_dispatch"] or {}).get("reconcile") or {} + assert reconcile.get("expired_count") == 1 + assert reconcile.get("expired_leases") == ["todo_first"] + + +def test_build_event_driven_dispatch_capability_forwarded( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + goal_id = "dispatch-cap" + queue = task_queue_path(tmp_path, goal_id=goal_id) + enqueue_tasks( + queue, + goal_id=goal_id, + todo_ids=["todo_gated"], + recorded_at="2026-08-14T00:00:00Z", + ) + entries = [ + json.loads(line) for line in queue.read_text(encoding="utf-8").splitlines() if line.strip() + ] + entries[0]["required_capabilities"] = ["gpu"] + queue.write_text(json.dumps(entries[0], sort_keys=True) + "\n", encoding="utf-8") + # A python-only worker is not eligible for the gpu task, so it stays pending + # and is never claimed by the python worker. + payload = build_event_driven_dispatch( + runtime_root=tmp_path, + goal_id=goal_id, + items=_gate_items(), + completed_todo_id="todo_first", + worker_id="worker_py", + worker_capabilities=["python"], + recorded_at="2026-08-14T00:00:00Z", + ) + dispatched = (payload["event_driven_dispatch"] or {}).get("dispatched") + # The python worker claims the (non-gated) READY successor, never the gpu task. + assert dispatched is not None + assert dispatched["todo_id"] != "todo_gated" + # The gpu task remains pending in the queue (never claimed by a python worker). + view = load_task_queue(queue) + assert "todo_gated" in view["pending_todo_ids"] + assert "todo_gated" not in view["claimed_todo_ids"] + + +def test_event_driven_dispatch_enabled_flag(monkeypatch: pytest.MonkeyPatch) -> None: + # The new architecture is ON by default (master switch), so an unset feature + # env inherits the master switch. + monkeypatch.delenv(EVENT_DRIVEN_DISPATCH_ENV, raising=False) + monkeypatch.delenv("LOOPX_NEW_ARCHITECTURE", raising=False) + assert event_driven_dispatch_enabled() is True + # An explicit flag always wins over the master switch. + assert event_driven_dispatch_enabled(use_event_driven=False) is False + assert event_driven_dispatch_enabled(use_event_driven=True) is True + # The feature env var still wins over the master switch default. + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "0") + assert event_driven_dispatch_enabled() is False + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + assert event_driven_dispatch_enabled() is True + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "true") + assert event_driven_dispatch_enabled() is True + # The master switch can turn the whole new architecture off. + monkeypatch.delenv(EVENT_DRIVEN_DISPATCH_ENV, raising=False) + monkeypatch.setenv("LOOPX_NEW_ARCHITECTURE", "0") + assert event_driven_dispatch_enabled() is False diff --git a/tests/control_plane/test_goal_acceptance.py b/tests/control_plane/test_goal_acceptance.py new file mode 100644 index 000000000..6005f3ae1 --- /dev/null +++ b/tests/control_plane/test_goal_acceptance.py @@ -0,0 +1,240 @@ +"""Tests for the Goal Acceptance / Evidence Verification layer (plan §5.10). + +Closure is NOT just "no work left": a goal must be *actually realized* with +sufficient evidence. This verifies that: +* every acceptance criterion requires satisfying evidence, +* unsatisfied criteria block goal closure (WAIT / pending), +* satisfying all criteria permits closure, +* `goal_acceptance_pending` is emitted when evidence is insufficient. +""" + +from __future__ import annotations + +from pathlib import Path + +from loopx.control_plane.goals.goal_acceptance import ( + acceptance_blocker, + build_grep_evidence, + build_manual_evidence, + evaluate_goal_acceptance, + emit_goal_acceptance_pending, + emit_goal_acceptance_satisfied, + normalize_acceptance_criteria, + normalize_evidence, + verify_criterion, + verify_grep_evidence, +) +from loopx.control_plane.goals.goal_closure import ( + build_goal_closure_state, + evaluate_goal_closure, + is_goal_closable, + maybe_close_goal, +) +from loopx.rollout_event_log import load_rollout_events, rollout_event_log_path + + +def _criteria() -> list[dict]: + return [ + {"criterion_id": "color_green", "description": "theme color is #22c55e"}, + {"criterion_id": "font_poppins", "description": "font is Poppins"}, + ] + + +def test_acceptance_satisfied_when_all_criteria_have_evidence() -> None: + evidence = [ + build_grep_evidence(ref="index.html", pattern="#22c55e", match=True, criterion_ids=["color_green"]), + build_manual_evidence(ref="index.html", content="font-family:Poppins", ok=True, criterion_ids=["font_poppins"]), + ] + result = evaluate_goal_acceptance(acceptance_criteria=_criteria(), evidence=evidence) + assert result["satisfied"] is True + assert result["acceptance_gaps"] == [] + assert result["criteria_count"] == 2 + assert result["evidence_count"] == 2 + + +def test_acceptance_gap_when_criterion_missing_evidence() -> None: + evidence = [ + build_grep_evidence(ref="index.html", pattern="#22c55e", match=True, criterion_ids=["color_green"]), + ] + result = evaluate_goal_acceptance(acceptance_criteria=_criteria(), evidence=evidence) + assert result["satisfied"] is False + assert [g["criterion_id"] for g in result["acceptance_gaps"]] == ["font_poppins"] + + +def test_acceptance_gap_when_evidence_ok_false() -> None: + # A failed grep (match=False -> ok=False) must NOT satisfy the criterion. + evidence = [ + build_grep_evidence(ref="index.html", pattern="#22c55e", match=False, criterion_ids=["color_green"]), + ] + result = evaluate_goal_acceptance(acceptance_criteria=_criteria(), evidence=evidence) + assert result["satisfied"] is False + assert len(result["acceptance_gaps"]) == 2 + + +def test_acceptance_no_criteria_is_satisfied() -> None: + result = evaluate_goal_acceptance(acceptance_criteria=None, evidence=[]) + assert result["satisfied"] is True + assert result["acceptance_gaps"] == [] + + +def test_verify_criterion_returns_evidence_refs() -> None: + evidence = [ + build_grep_evidence(ref="index.html", pattern="#22c55e", match=True, criterion_ids=["color_green"]), + ] + result = verify_criterion(_criteria()[0], evidence) + assert result["satisfied"] is True + assert result["evidence_refs"] == ["grep:#22c55e"] + + +def test_acceptance_blocker() -> None: + satisfied = evaluate_goal_acceptance(acceptance_criteria=_criteria(), evidence=[]) + assert satisfied["satisfied"] is False + assert acceptance_blocker(satisfied) == "acceptance_gaps_remaining" + full = evaluate_goal_acceptance( + acceptance_criteria=_criteria(), + evidence=[ + build_grep_evidence(ref="i", pattern="a", match=True, criterion_ids=["color_green"]), + build_grep_evidence(ref="i", pattern="b", match=True, criterion_ids=["font_poppins"]), + ], + ) + assert full["satisfied"] is True + assert acceptance_blocker(full) is None + + +def test_unsatisfied_acceptance_blocks_goal_closure() -> None: + acceptance = evaluate_goal_acceptance( + acceptance_criteria=_criteria(), + evidence=[build_grep_evidence(ref="index.html", pattern="#22c55e", match=True, criterion_ids=["color_green"])], + ) + state = build_goal_closure_state(acceptance=acceptance) + assert is_goal_closable(state) is False + evaluation = evaluate_goal_closure(state) + assert evaluation["reason"] == "acceptance_gaps_remaining" + assert evaluation["tri_state"] == "WAIT" + assert evaluation["evidence"]["acceptance_satisfied"] is False + assert evaluation["evidence"]["acceptance_gap_count"] == 1 + + +def test_satisfied_acceptance_allows_goal_closure() -> None: + acceptance = evaluate_goal_acceptance( + acceptance_criteria=_criteria(), + evidence=[ + build_grep_evidence(ref="index.html", pattern="#22c55e", match=True, criterion_ids=["color_green"]), + build_manual_evidence(ref="index.html", content="Poppins", ok=True, criterion_ids=["font_poppins"]), + ], + ) + state = build_goal_closure_state(acceptance=acceptance) + assert is_goal_closable(state) is True + + +def test_maybe_close_goal_with_gap_emits_pending_not_closed(tmp_path: Path) -> None: + log_path = rollout_event_log_path(tmp_path, goal_id="g1") + acceptance = evaluate_goal_acceptance( + acceptance_criteria=_criteria(), + evidence=[build_grep_evidence(ref="i", pattern="a", match=True, criterion_ids=["color_green"])], + ) + state = build_goal_closure_state(acceptance=acceptance) + result = maybe_close_goal(log_path=log_path, goal_id="g1", state=state) + assert result["ready"] is False + assert result.get("closed") is not True + kinds = [e["event_kind"] for e in load_rollout_events(log_path, limit=10)] + assert kinds == ["goal_acceptance_pending"] + assert "goal_closed" not in kinds + + +def test_maybe_close_goal_with_satisfied_acceptance_closes(tmp_path: Path) -> None: + log_path = rollout_event_log_path(tmp_path, goal_id="g1") + acceptance = evaluate_goal_acceptance( + acceptance_criteria=_criteria(), + evidence=[ + build_grep_evidence(ref="i", pattern="a", match=True, criterion_ids=["color_green"]), + build_manual_evidence(ref="i", content="Poppins", ok=True, criterion_ids=["font_poppins"]), + ], + ) + state = build_goal_closure_state(acceptance=acceptance) + result = maybe_close_goal(log_path=log_path, goal_id="g1", state=state) + assert result["ready"] is True + assert result.get("closed") is True + kinds = [e["event_kind"] for e in load_rollout_events(log_path, limit=10)] + assert kinds == ["goal_closure_ready", "goal_closed"] + + +def test_emit_acceptance_events_idempotent(tmp_path: Path) -> None: + log_path = rollout_event_log_path(tmp_path, goal_id="g1") + emit_goal_acceptance_pending(log_path=log_path, goal_id="g1", acceptance_gaps=[{"criterion_id": "x"}]) + emit_goal_acceptance_pending(log_path=log_path, goal_id="g1", acceptance_gaps=[{"criterion_id": "x"}]) + emit_goal_acceptance_satisfied(log_path=log_path, goal_id="g1", criteria_results=[]) + emit_goal_acceptance_satisfied(log_path=log_path, goal_id="g1", criteria_results=[]) + events = load_rollout_events(log_path, limit=10) + kinds = [e["event_kind"] for e in events] + assert kinds.count("goal_acceptance_pending") == 1 + assert kinds.count("goal_acceptance_satisfied") == 1 + + +def test_normalize_evidence_caps_unknown_kind() -> None: + evidence = [{"evidence_id": "e1", "kind": "weird", "ref": "r", "ok": True}] + normalized = normalize_evidence(evidence) + assert normalized[0]["kind"] == "manual" + assert normalize_acceptance_criteria([{"description": "desc only"}])[0]["criterion_id"] + + +def test_grep_evidence_independently_verified(tmp_path: Path) -> None: + """kind=grep evidence is verified against the real file when base_dir is set: + the framework recomputes `ok` from an actual match, overriding the caller flag.""" + target = tmp_path / "index.html" + target.write_text('', encoding="utf-8") + criteria = [{"criterion_id": "c1", "description": "背景为黄色"}] + + # Honest match -> satisfied, independently verified. + ev_match = build_grep_evidence( + ref="index.html", pattern="f9d616", match=True, criterion_ids=["c1"] + ) + r1 = evaluate_goal_acceptance( + acceptance_criteria=criteria, evidence=[ev_match], base_dir=tmp_path + ) + assert r1["satisfied"] is True + assert r1["verified_count"] == 1 + + # Self-reported ok=True but the pattern does NOT exist in the file -> + # the framework must override `ok` to False (no false "goal_closed"). + ev_lie = build_grep_evidence( + ref="index.html", pattern="00ff00", match=True, criterion_ids=["c1"] + ) + r2 = evaluate_goal_acceptance( + acceptance_criteria=criteria, evidence=[ev_lie], base_dir=tmp_path + ) + assert r2["satisfied"] is False + assert r2["acceptance_gaps"][0]["criterion_id"] == "c1" + + # Missing target file is a genuine negative finding. + ev_missing = build_grep_evidence( + ref="nope.html", pattern="x", match=True, criterion_ids=["c1"] + ) + r3 = evaluate_goal_acceptance( + acceptance_criteria=criteria, evidence=[ev_missing], base_dir=tmp_path + ) + assert r3["satisfied"] is False + + # Without base_dir, grep evidence degrades to the caller's ok (back-compat). + r4 = evaluate_goal_acceptance(acceptance_criteria=criteria, evidence=[ev_match]) + assert r4["satisfied"] is True + assert r4["verified_count"] == 0 + + +def test_verify_grep_evidence_overrides_caller_ok(tmp_path: Path) -> None: + target = tmp_path / "app.py" + target.write_text("COLOR = '#f9d616'", encoding="utf-8") + # Caller claims ok, pattern absent -> framework returns not ok + verified. + verified = verify_grep_evidence( + {"kind": "grep", "ref": "app.py", "pattern": "nope", "ok": True}, + base_dir=tmp_path, + ) + assert verified["ok"] is False + assert verified["verified"] is True + assert verified["matched_lines"] == 0 + # Non-grep evidence untouched. + manual = verify_grep_evidence( + {"kind": "manual", "ref": "app.py", "ok": True}, base_dir=tmp_path + ) + assert manual["ok"] is True + assert manual.get("verified") is not True diff --git a/tests/control_plane/test_goal_channel_projection_quota.py b/tests/control_plane/test_goal_channel_projection_quota.py new file mode 100644 index 000000000..b666e9e57 --- /dev/null +++ b/tests/control_plane/test_goal_channel_projection_quota.py @@ -0,0 +1,121 @@ +from __future__ import annotations + +from loopx.control_plane.goals.goal_channel_projection import ( + _compact_quota, + _compact_policy_decision, + _compact_scheduler_hint, +) + + +def _scheduler_hint_payload() -> dict: + return { + "schema_version": "scheduler_hint_v0", + "source": "quota.should-run", + "action": "backoff", + "cadence_class": "agent_monitor_only", + "reason_code": "agent_monitor_only_quiet_poll", + "reason": "Agent is in monitor-only quiet poll; hold cadence.", + "spend_policy": "no quota spend for monitor-only wait", + "codex_app": { + "applicability": "applies", + "apply": "reschedule", + "host_action": "reschedule_heartbeat", + }, + "heartbeat_recommendation": { + "recommended_mode": "resume", + "cadence_class": "agent_monitor_only", + "recommended_interval_seconds": 900, + }, + } + + +def _policy_decision_payload() -> dict: + return { + "outcome": "wait", + "source": "quota", + "reason": "Quota slot exhausted; back off before next poll.", + "retry_after_seconds": 300, + "manual_approval_required": False, + } + + +def test_compact_quota_passes_through_scheduler_hint_and_policy_decision() -> None: + quota = { + "state": "paused", + "reason": "agent monitor-only", + "spend_policy": "pause", + "scheduler_hint": _scheduler_hint_payload(), + "policy_decision": _policy_decision_payload(), + "scheduler_rrule": "FREQ=HOURLY;INTERVAL=1", + "scheduler_reset_token": "tok-123", + "cadence_class": "agent_monitor_only", + } + compact = _compact_quota({"quota": quota}, {}) + + # Legacy scalar fields preserved. + assert compact["state"] == "paused" + assert compact["reason"] == "agent monitor-only" + assert compact["spend_policy"] == "pause" + + # New-architecture scheduler hint surfaced (whitelisted public fields only). + hint = compact["scheduler_hint"] + assert hint["action"] == "backoff" + assert hint["cadence_class"] == "agent_monitor_only" + assert hint["heartbeat_recommendation"]["recommended_interval_seconds"] == "900" + # Raw/private nested structures (codex_app) are not copied. + assert "codex_app" not in hint + + # Unified policy decision surfaced. + policy = compact["policy_decision"] + assert policy["outcome"] == "wait" + assert policy["retry_after_seconds"] == "300" + + # Top-level cadence scalars surfaced for frontstage direct reads. + assert compact["scheduler_rrule"] == "FREQ=HOURLY;INTERVAL=1" + assert compact["scheduler_reset_token"] == "tok-123" + assert compact["cadence_class"] == "agent_monitor_only" + + +def test_compact_quota_unchanged_when_no_new_architecture_fields() -> None: + quota = { + "state": "ok", + "reason": "quota available", + "spend_policy": "proceed", + "spent_slots": 1, + "allowed_slots": 5, + } + compact = _compact_quota({"quota": quota}, {}) + + # Byte-identical to the pre-Phase-5 shape: only the five legacy scalars. + assert compact == { + "state": "ok", + "reason": "quota available", + "spend_policy": "proceed", + "spent_slots": "1", + "allowed_slots": "5", + } + + +def test_compact_scheduler_hint_returns_none_for_empty_or_non_mapping() -> None: + assert _compact_scheduler_hint(None) is None + assert _compact_scheduler_hint({}) is None + assert _compact_scheduler_hint("not-a-mapping") is None + + +def test_compact_policy_decision_returns_none_for_empty_or_non_mapping() -> None: + assert _compact_policy_decision(None) is None + assert _compact_policy_decision({}) is None + assert _compact_policy_decision("not-a-mapping") is None + + +def test_compact_quota_falls_back_to_project_asset_source() -> None: + project_asset = { + "quota": { + "state": "ok", + "spend_policy": "proceed", + "policy_decision": {"outcome": "run", "source": "quota", "reason": "go"}, + } + } + compact = _compact_quota({}, project_asset) + assert compact["state"] == "ok" + assert compact["policy_decision"]["outcome"] == "run" diff --git a/tests/control_plane/test_goal_closure.py b/tests/control_plane/test_goal_closure.py new file mode 100644 index 000000000..bcad2e874 --- /dev/null +++ b/tests/control_plane/test_goal_closure.py @@ -0,0 +1,125 @@ +"""Tests for the Goal Closure Evaluator + Controller (elegant, event-driven). + +Covers the design from `plan/new_plan.md` + user feedback: Goal closure is a +*distinct lifecycle* from Todo completion. A goal is closable purely from state +(no ready work, no pending deps, no replan, no external follow-up) — WITHOUT +requiring every todo to carry an explicit ``no_followup`` intent. +""" + +from __future__ import annotations + +from pathlib import Path + +from loopx.control_plane.goals.goal_closure import ( + GOAL_CLOSE, + GOAL_RUN, + GOAL_WAIT, + build_goal_closure_state, + classify_goal_continuation, + emit_goal_closed, + emit_goal_closure_ready, + evaluate_goal_closure, + goal_closure_reason, + is_goal_closable, + maybe_close_goal, +) +from loopx.rollout_event_log import load_rollout_events, rollout_event_log_path + + +def test_goal_closable_when_no_work_no_deps_no_replan() -> None: + state = build_goal_closure_state() + assert is_goal_closable(state) is True + assert goal_closure_reason(state) is None + assert classify_goal_continuation(state) == GOAL_CLOSE + + +def test_goal_not_closable_when_ready_work() -> None: + state = build_goal_closure_state(ready_todo_ids=["todo_a"]) + assert is_goal_closable(state) is False + assert goal_closure_reason(state) == "ready_work_remaining" + assert classify_goal_continuation(state) == GOAL_RUN + + +def test_goal_not_closable_when_pending_dependency() -> None: + state = build_goal_closure_state(pending_dependency_ids=["todo_b"]) + assert is_goal_closable(state) is False + assert goal_closure_reason(state) == "pending_dependencies" + assert classify_goal_continuation(state) == GOAL_WAIT + + +def test_goal_not_closable_when_replan_required() -> None: + state = build_goal_closure_state(replan_required=True) + assert is_goal_closable(state) is False + assert goal_closure_reason(state) == "replan_required" + + +def test_goal_not_closable_when_external_followup_required() -> None: + state = build_goal_closure_state(external_followup_required=True) + assert is_goal_closable(state) is False + assert goal_closure_reason(state) == "external_followup_required" + + +def test_goal_not_closable_when_open_or_claimed_count() -> None: + open_state = build_goal_closure_state(open_todo_count=1) + assert is_goal_closable(open_state) is False + claimed_state = build_goal_closure_state(claimed_advancement_count=1) + assert is_goal_closable(claimed_state) is False + + +def test_goal_closable_with_blocked_but_no_open_work() -> None: + # Blocked work is future work -> WAIT, not CLOSE. Deferred + blocked keeps it open. + state = build_goal_closure_state(blocked_todo_ids=["todo_c"]) + assert is_goal_closable(state) is False + assert classify_goal_continuation(state) == GOAL_WAIT + + +def test_evaluate_goal_closure_returns_evidence() -> None: + state = build_goal_closure_state( + ready_todo_ids=[], + blocked_todo_ids=["todo_x"], + deferred_todo_ids=["todo_y"], + replan_required=False, + ) + evaluation = evaluate_goal_closure(state) + assert evaluation["ready"] is False + assert evaluation["tri_state"] == GOAL_WAIT + assert evaluation["evidence"]["blocked_todo_ids"] == ["todo_x"] + assert evaluation["evidence"]["deferred_todo_ids"] == ["todo_y"] + assert evaluation["evidence"]["replan_required"] is False + + +def test_maybe_close_goal_emits_events_when_ready(tmp_path: Path) -> None: + log_path = rollout_event_log_path(tmp_path, goal_id="g1") + result = maybe_close_goal( + log_path=log_path, + goal_id="g1", + state=build_goal_closure_state(), + ) + assert result["ready"] is True + assert result.get("closed") is True + kinds = [e["event_kind"] for e in load_rollout_events(log_path, limit=10)] + assert kinds == ["goal_closure_ready", "goal_closed"] + + +def test_maybe_close_goal_does_nothing_when_not_ready(tmp_path: Path) -> None: + log_path = rollout_event_log_path(tmp_path, goal_id="g1") + result = maybe_close_goal( + log_path=log_path, + goal_id="g1", + state=build_goal_closure_state(ready_todo_ids=["todo_a"]), + ) + assert result["ready"] is False + assert result.get("closed") is not True + assert load_rollout_events(log_path, limit=10) == [] + + +def test_emit_events_idempotent(tmp_path: Path) -> None: + log_path = rollout_event_log_path(tmp_path, goal_id="g1") + emit_goal_closure_ready(log_path=log_path, goal_id="g1", reason="no_followup_work") + emit_goal_closure_ready(log_path=log_path, goal_id="g1", reason="no_followup_work") + emit_goal_closed(log_path=log_path, goal_id="g1", kind="derived") + emit_goal_closed(log_path=log_path, goal_id="g1", kind="derived") + events = load_rollout_events(log_path, limit=10) + kinds = [e["event_kind"] for e in events] + assert kinds.count("goal_closure_ready") == 1 + assert kinds.count("goal_closed") == 1 diff --git a/tests/control_plane/test_heartbeat_event_source.py b/tests/control_plane/test_heartbeat_event_source.py new file mode 100644 index 000000000..183cbe13c --- /dev/null +++ b/tests/control_plane/test_heartbeat_event_source.py @@ -0,0 +1,164 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from loopx.control_plane.heartbeat.event_source import ( + HEARTBEAT_EVENT_SOURCE_ENV, + HEARTBEAT_OBSERVED_EVENT_KIND, + HeartbeatEventSource, + build_heartbeat_observation_event, + compute_observation_fingerprint, + heartbeat_event_source_enabled, + record_heartbeat_observation, +) +from loopx.rollout_event_log import load_rollout_events, rollout_event_log_path + + +def _gate_items() -> list[dict[str, object]]: + return [ + { + "todo_id": "todo_first", + "text": "setup done", + "status": "done", + "excluded_agents": ["agent_worker"], + "unblocks_todo_id": "todo_second", + }, + { + "todo_id": "todo_second", + "text": "followup advancement", + "task_class": "advancement_task", + "unblocks_todo_id": "todo_first", + "status": "open", + }, + ] + + +def test_heartbeat_event_source_enabled_flag(monkeypatch: pytest.MonkeyPatch) -> None: + # The new architecture is ON by default (master switch), so an unset feature + # env inherits the master switch. + monkeypatch.delenv(HEARTBEAT_EVENT_SOURCE_ENV, raising=False) + monkeypatch.delenv("LOOPX_NEW_ARCHITECTURE", raising=False) + assert heartbeat_event_source_enabled() is True + # An explicit flag always wins over the master switch. + assert heartbeat_event_source_enabled(use_event_source=False) is False + assert heartbeat_event_source_enabled(use_event_source=True) is True + # The feature env var still wins over the master switch default. + monkeypatch.setenv(HEARTBEAT_EVENT_SOURCE_ENV, "0") + assert heartbeat_event_source_enabled() is False + monkeypatch.setenv(HEARTBEAT_EVENT_SOURCE_ENV, "1") + assert heartbeat_event_source_enabled() is True + monkeypatch.setenv(HEARTBEAT_EVENT_SOURCE_ENV, "true") + assert heartbeat_event_source_enabled() is True + # The master switch can turn the whole new architecture off. + monkeypatch.delenv(HEARTBEAT_EVENT_SOURCE_ENV, raising=False) + monkeypatch.setenv("LOOPX_NEW_ARCHITECTURE", "0") + assert heartbeat_event_source_enabled() is False + + +def test_compute_observation_fingerprint_is_deterministic() -> None: + a = compute_observation_fingerprint(goal_id="g1", agent_id="a1", source="heartbeat_poll") + b = compute_observation_fingerprint(goal_id="g1", agent_id="a1", source="heartbeat_poll") + assert a == b + c = compute_observation_fingerprint(goal_id="g1", agent_id="a2", source="heartbeat_poll") + assert a != c + + +def test_heartbeat_observation_disabled_writes_nothing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv(HEARTBEAT_EVENT_SOURCE_ENV, raising=False) + goal_id = "heartbeat-disabled" + result = record_heartbeat_observation( + runtime_root=tmp_path, + goal_id=goal_id, + agent_id="agent_one", + use_event_source=False, + ) + assert result.get("disabled") is True + assert result.get("ok") is True + assert not rollout_event_log_path(tmp_path, goal_id).exists() + + +def test_heartbeat_observation_records_event_fact( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(HEARTBEAT_EVENT_SOURCE_ENV, "1") + goal_id = "heartbeat-on" + result = record_heartbeat_observation( + runtime_root=tmp_path, + goal_id=goal_id, + agent_id="agent_one", + source="heartbeat_poll", + tick_id="tick-001", + status="run", + details={"cause": "policy_test"}, + recorded_at="2026-08-14T00:00:00Z", + ) + assert result.get("disabled") is not True + assert result["new"] is True + assert result["event"]["event_kind"] == HEARTBEAT_OBSERVED_EVENT_KIND + assert result["event"]["goal_id"] == goal_id + assert result["event"]["agent_id"] == "agent_one" + + events = load_rollout_events(rollout_event_log_path(tmp_path, goal_id)) + assert len(events) == 1 + assert events[0]["event_kind"] == HEARTBEAT_OBSERVED_EVENT_KIND + assert events[0]["boundary"]["raw_task_text_recorded"] is False + + +def test_heartbeat_observation_idempotent_same_tick( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(HEARTBEAT_EVENT_SOURCE_ENV, "1") + goal_id = "heartbeat-idem" + kwargs = dict( + runtime_root=tmp_path, + goal_id=goal_id, + agent_id="agent_one", + tick_id="tick-001", + source="heartbeat_poll", + recorded_at="2026-08-14T00:00:00Z", + ) + first = record_heartbeat_observation(**kwargs) + second = record_heartbeat_observation(**kwargs) + assert first["new"] is True + assert second["new"] is False + events = load_rollout_events(rollout_event_log_path(tmp_path, goal_id)) + assert len(events) == 1 + + +def test_heartbeat_event_source_class_observe() -> None: + # Class-level observe writes the fact without touching decisions. + class FakeRecorder: + def __init__(self) -> None: + self.calls: list[tuple] = [] + + def observe(self, *args: object, **kwargs: object) -> dict[str, object]: + self.calls.append((args, kwargs)) + return {"ok": True, "new": True} + + source = FakeRecorder() + out = source.observe(tick_id="t1") + assert out["new"] is True + assert len(source.calls) == 1 + + +def test_heartbeat_observation_builds_fact_only_payload( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # The event fact must not carry business decision fields or raw task text. + monkeypatch.setenv(HEARTBEAT_EVENT_SOURCE_ENV, "1") + event = build_heartbeat_observation_event( + goal_id="g1", + agent_id="a1", + source="heartbeat_poll", + tick_id="t1", + details={"schema_version": "v0"}, + recorded_at="2026-08-14T00:00:00Z", + ) + assert event["event_kind"] == HEARTBEAT_OBSERVED_EVENT_KIND + details = event.get("details") or {} + assert "decision" not in details + assert "task_text" not in details diff --git a/tests/control_plane/test_heartbeat_prequota.py b/tests/control_plane/test_heartbeat_prequota.py new file mode 100644 index 000000000..4a8021dc3 --- /dev/null +++ b/tests/control_plane/test_heartbeat_prequota.py @@ -0,0 +1,107 @@ +"""Tests for the heartbeat pre-quota hook registry integration (P3).""" + +from __future__ import annotations + +from pathlib import Path + +from loopx.heartbeat_prequota import ( + PRE_QUOTA_HOOK_POINT, + get_pre_quota_hook_registry, + register_pre_quota_hook, + render_heartbeat_pre_quota_markdown, + run_heartbeat_pre_quota, +) + + +def _run(**overrides): + return run_heartbeat_pre_quota( + registry_path=Path("/tmp/loopx-heartbeat-test-registry"), + runtime_root_arg=None, + goal_id="goal-1", + agent_id="agent-1", + fetch_timeout_seconds=10, + **overrides, + ) + + +def _drop_test_hook(hook): + registry = get_pre_quota_hook_registry() + registry._hooks[PRE_QUOTA_HOOK_POINT] = [ + (source, fn) + for (source, fn) in registry._hooks.get(PRE_QUOTA_HOOK_POINT, []) + if fn is not hook + ] + + +def test_run_heartbeat_pre_quota_includes_builtin_hook(): + payload = _run() + assert payload["ok"] is True + checks = payload["checks"] + # Built-in issue-fix reconcile hook is always present by name. + assert "issue_fix_pr_review_reconcile_hook" in checks["hooks"] + # Legacy compatibility key is preserved. + assert isinstance(checks["acknowledged_pr_reviews"], dict) + assert payload["continue_to_quota"] is True + + +def test_run_heartbeat_pre_quota_fans_out_to_registered_hooks(): + def custom_hook(*, goal_id, **kwargs): + return {"ok": True, "hook": "custom", "goal": goal_id} + + register_pre_quota_hook(custom_hook, source="test-pack") + try: + payload = _run() + finally: + _drop_test_hook(custom_hook) + + hooks = payload["checks"]["hooks"] + assert hooks["custom_hook"]["goal"] == "goal-1" + assert payload["degraded"] is False + assert payload["failure_count"] == 0 + + +def test_run_heartbeat_pre_quota_isolates_failing_hook(): + def failing_hook(**kwargs): + raise RuntimeError("hook boom") + + register_pre_quota_hook(failing_hook, source="bad-pack") + try: + payload = _run() + finally: + _drop_test_hook(failing_hook) + + assert payload["degraded"] is True + assert payload["failure_count"] >= 1 + hooks = payload["checks"]["hooks"] + assert hooks["failing_hook"]["failure_count"] == 1 + + +def test_run_heartbeat_pre_quota_ignores_non_mapping_results(): + def weird_hook(**kwargs): + return "not-a-dict" + + register_pre_quota_hook(weird_hook, source="weird-pack") + try: + payload = _run() + finally: + _drop_test_hook(weird_hook) + + hooks = payload["checks"]["hooks"] + assert hooks["weird_hook"] == {} + + +def test_render_heartbeat_pre_quota_markdown_compat_paths(): + # Missing acknowledged_pr_reviews degrades to defaults. + rendered = render_heartbeat_pre_quota_markdown({"ok": True, "checks": {}}) + assert "reconciled_count: `0`" in rendered + assert "# LoopX Heartbeat Pre-Quota" in rendered + + # A populated review reconciliation is projected through the legacy key. + payload = { + "ok": True, + "degraded": False, + "failure_count": 0, + "checks": {"acknowledged_pr_reviews": {"reconciled_count": 7}}, + } + rendered = render_heartbeat_pre_quota_markdown(payload) + assert "reconciled_count: `7`" in rendered diff --git a/tests/control_plane/test_policy_decision_events.py b/tests/control_plane/test_policy_decision_events.py new file mode 100644 index 000000000..179275982 --- /dev/null +++ b/tests/control_plane/test_policy_decision_events.py @@ -0,0 +1,177 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from loopx.control_plane.policy import ( + POLICY_DECISION_EVENT_KIND, + Decision, + PolicyDecisionRecorder, + compute_decision_fingerprint, + policy_decision_events, + record_policy_decision, +) + + +def _wait_decision() -> Decision: + return Decision(outcome="wait", reason="quota_backoff", source="quota") + + +def _run_decision() -> Decision: + return Decision(outcome="run", reason="normal_delivery", source="quota") + + +def _read_events(log_path: Path) -> list[dict]: + if not log_path.exists(): + return [] + return [json.loads(line) for line in log_path.read_text(encoding="utf-8").splitlines() if line.strip()] + + +# --------------------------------------------------------------------------- +# Fingerprint determinism +# --------------------------------------------------------------------------- + + +def test_fingerprint_deterministic_and_stable() -> None: + first = compute_decision_fingerprint( + _wait_decision(), goal_id="g1", todo_id="t1", agent_id="a1" + ) + second = compute_decision_fingerprint( + _wait_decision(), goal_id="g1", todo_id="t1", agent_id="a1" + ) + assert first == second + assert len(first) == 16 + + +def test_fingerprint_changes_with_decision_semantics() -> None: + wait = compute_decision_fingerprint(_wait_decision(), goal_id="g1", todo_id="t1") + run = compute_decision_fingerprint(_run_decision(), goal_id="g1", todo_id="t1") + assert wait != run + + +def test_fingerprint_excludes_timestamps() -> None: + decision = _wait_decision() + stable = compute_decision_fingerprint(decision, goal_id="g1", todo_id="t1") + unstable = Decision( + outcome=decision.outcome, + reason=decision.reason, + source=decision.source, + retry_at="2026-08-13T12:30:00Z", + ) + assert compute_decision_fingerprint(unstable, goal_id="g1", todo_id="t1") == stable + + +# --------------------------------------------------------------------------- +# Idempotent recording + deduplication (RFC §8.4) +# --------------------------------------------------------------------------- + + +def test_record_appends_once(tmp_path: Path) -> None: + log_path = tmp_path / "events.jsonl" + event, was_new = record_policy_decision( + _wait_decision(), + goal_id="g1", + todo_id="t1", + agent_id="a1", + log_path=log_path, + state_dir=tmp_path / "state", + ) + assert was_new is True + assert event["event_kind"] == POLICY_DECISION_EVENT_KIND + assert event["goal_id"] == "g1" + assert event["todo_id"] == "t1" + assert event["status"] == "wait" + assert event["classification"] == "quota_backoff" + assert event["details"]["decision_source"] == "quota" + assert event["details"]["decision_outcome"] == "wait" + assert event["decision_fingerprint"] + + +def test_transition_only_suppresses_repeated_identical_decision(tmp_path: Path) -> None: + log_path = tmp_path / "events.jsonl" + recorder = PolicyDecisionRecorder(log_path=log_path, state_dir=tmp_path / "state") + _, first_new = recorder.record(_wait_decision(), goal_id="g1", todo_id="t1") + _, second_new = recorder.record(_wait_decision(), goal_id="g1", todo_id="t1") + assert first_new is True + assert second_new is False + assert len(_read_events(log_path)) == 1 + + +def test_transition_only_records_transition(tmp_path: Path) -> None: + log_path = tmp_path / "events.jsonl" + recorder = PolicyDecisionRecorder(log_path=log_path, state_dir=tmp_path / "state") + recorder.record(_wait_decision(), goal_id="g1", todo_id="t1") + _, transition_new = recorder.record(_run_decision(), goal_id="g1", todo_id="t1") + assert transition_new is True + events = _read_events(log_path) + assert len(events) == 2 + assert [event["status"] for event in events] == ["wait", "run"] + + +def test_transition_only_scopes_by_todo(tmp_path: Path) -> None: + log_path = tmp_path / "events.jsonl" + recorder = PolicyDecisionRecorder(log_path=log_path, state_dir=tmp_path / "state") + _, first_new = recorder.record(_wait_decision(), goal_id="g1", todo_id="t1") + _, other_new = recorder.record(_wait_decision(), goal_id="g1", todo_id="t2") + assert first_new is True + assert other_new is True + assert len(_read_events(log_path)) == 2 + + +def test_opt_out_transition_only_relies_on_identity_dedup(tmp_path: Path) -> None: + # Without transition tracking, idempotency still prevents exact duplicates + # because the fingerprint is part of the identity fields. + log_path = tmp_path / "events.jsonl" + recorder = PolicyDecisionRecorder( + log_path=log_path, state_dir=tmp_path / "state", transition_only=False + ) + _, first_new = recorder.record(_wait_decision(), goal_id="g1", todo_id="t1") + _, second_new = recorder.record(_wait_decision(), goal_id="g1", todo_id="t1") + assert first_new is True + assert second_new is False + assert len(_read_events(log_path)) == 1 + + +def test_record_persists_public_safe_boundary(tmp_path: Path) -> None: + log_path = tmp_path / "events.jsonl" + event, _ = record_policy_decision( + _wait_decision(), + goal_id="g1", + todo_id="t1", + log_path=log_path, + state_dir=tmp_path / "state", + ) + boundary = event["boundary"] + assert boundary["raw_task_text_recorded"] is False + assert boundary["credential_values_recorded"] is False + assert boundary["absolute_paths_recorded"] is False + + +def test_policy_decision_events_projection(tmp_path: Path) -> None: + log_path = tmp_path / "events.jsonl" + record_policy_decision(_wait_decision(), goal_id="g1", todo_id="t1", log_path=log_path, state_dir=tmp_path / "state") + # Inject a non-decision event by appending a raw line. + from loopx.rollout_event_log import build_rollout_event, append_rollout_event_once + + other = build_rollout_event( + goal_id="g1", + event_kind="quota_monitor_poll", + summary="poll", + details={"n": 1}, + ) + append_rollout_event_once(log_path, other, identity_fields=["goal_id", "event_kind"]) + decisions = policy_decision_events(_read_events(log_path)) + assert len(decisions) == 1 + assert decisions[0]["event_kind"] == POLICY_DECISION_EVENT_KIND + + +def test_replay_identity_shape(tmp_path: Path) -> None: + recorder = PolicyDecisionRecorder(log_path=tmp_path / "e.jsonl", state_dir=tmp_path / "s") + event, _ = recorder.record(_wait_decision(), goal_id="g1", todo_id="t1", agent_id="a1") + goal_id, todo_id, agent_id, fingerprint = recorder.replay_identity(event) + assert goal_id == "g1" + assert todo_id == "t1" + assert agent_id == "a1" + assert fingerprint == event["decision_fingerprint"] diff --git a/tests/control_plane/test_policy_engine.py b/tests/control_plane/test_policy_engine.py new file mode 100644 index 000000000..66866bbbd --- /dev/null +++ b/tests/control_plane/test_policy_engine.py @@ -0,0 +1,298 @@ +from __future__ import annotations + +import pytest + +from loopx.control_plane.policy import ( + CAPABILITY_ACTION_MAP, + DECISION_MAP, + Decision, + PolicyEngine, + combine_decisions, + normalize_capability_action, + normalize_quota_decision, + normalize_scheduler_resolution, +) +from loopx.control_plane.policy.decision import _OUTCOME_RANK +from loopx.control_plane.scheduler.execution_context import ( + GENERIC_CLI_OUTER_CONTROLLER_SCHEDULER_CONTEXT, + resolve_scheduler_execution_context, +) +from loopx.control_plane.testing.quota_fixtures import quota_status_payload + +GOAL_ID = "policy-engine-fixture" + +VALID_SCHEDULER_CONTEXT = dict(GENERIC_CLI_OUTER_CONTROLLER_SCHEDULER_CONTEXT) + + +def _run_payload(*, required_capabilities: list[str] | None = None) -> dict: + todo_text = "[P1] Advance the bounded slice." + items = [ + { + "index": 1, + "text": todo_text, + "role": "agent", + "status": "open", + "priority": "P1", + "task_class": "advancement_task", + } + ] + if required_capabilities: + items[0]["required_capabilities"] = required_capabilities + return quota_status_payload( + goal_id=GOAL_ID, + status="active", + agent_todo_items=items, + recommended_action=todo_text, + next_action=todo_text, + ) + + +def _engine() -> PolicyEngine: + return PolicyEngine() + + +# --------------------------------------------------------------------------- +# Decision contract +# --------------------------------------------------------------------------- + + +def test_decision_defaults() -> None: + decision = Decision(outcome="run", reason="normal_delivery", source="quota") + assert decision.to_dict() == { + "outcome": "run", + "reason": "normal_delivery", + "source": "quota", + } + assert bool(decision) is True + + +def test_decision_round_trip_with_retry_metadata() -> None: + decision = Decision( + outcome="wait", + reason="quota_backoff", + source="quota", + detail={"state": "backoff"}, + retry_at="2026-08-13T12:30:00Z", + retry_after_seconds=300, + manual_approval_required=True, + ) + restored = Decision.from_dict(decision.to_dict()) + assert restored == decision + assert bool(restored) is False + + +def test_decision_unknown_outcome_from_dict_is_deny() -> None: + decision = Decision.from_dict({"outcome": "bogus"}) + assert decision.outcome == "deny" + assert bool(decision) is False + + +# --------------------------------------------------------------------------- +# Exhaustive normalization maps +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("decision_value", "expected_outcome"), + [ + ("run", "run"), + ("observe", "run"), + ("safe_bypass_recovery", "run"), + ("recovery", "run"), + ("self_repair", "run"), + ("autonomous_replan_required", "run"), + ("repair_bridge", "wait"), + ("workspace_guard", "wait"), + ("automation_prompt_upgrade", "wait"), + ("agent_scope_exhausted", "wait"), + ("agent_scope_wait", "wait"), + ("reassignment_required", "wait"), + ("successor_replan_required", "wait"), + ("skip", "deny"), + ], +) +def test_quota_decision_map_exhaustive(decision_value: str, expected_outcome: str) -> None: + outcome, reason = DECISION_MAP[decision_value] + assert outcome == expected_outcome + assert reason + decision = normalize_quota_decision(decision_value) + assert decision.outcome == expected_outcome + assert decision.source == "quota" + assert decision.detail["quota_decision"] == decision_value + + +def test_quota_decision_unknown_value_is_deny() -> None: + decision = normalize_quota_decision("totally_new_mode") + assert decision.outcome == "deny" + assert decision.reason.startswith("unknown_quota_decision") + + +@pytest.mark.parametrize( + ("action_value", "expected_outcome"), + [ + ("run", "run"), + ("repair_bridge", "wait"), + ("ask_owner", "wait"), + ("deny", "deny"), + ("denied", "deny"), + ("skip", "deny"), + ], +) +def test_capability_action_map_exhaustive(action_value: str, expected_outcome: str) -> None: + outcome, reason = CAPABILITY_ACTION_MAP[action_value] + assert outcome == expected_outcome + decision = normalize_capability_action(action_value) + assert decision.outcome == expected_outcome + assert decision.source == "capability" + assert decision.detail["capability_action"] == action_value + + +def test_scheduler_resolution_deny_and_ok() -> None: + ok_resolution = resolve_scheduler_execution_context(VALID_SCHEDULER_CONTEXT) + assert ok_resolution.ok + assert normalize_scheduler_resolution(ok_resolution).outcome == "run" + + bad_resolution = resolve_scheduler_execution_context({"host_surface": "bogus"}) + decision = normalize_scheduler_resolution(bad_resolution) + assert decision.outcome == "deny" + assert decision.reason == "invalid_scheduler_execution_context" + assert decision.detail["errors"] + + +# --------------------------------------------------------------------------- +# combine_decisions strictness (deny > wait > run) +# --------------------------------------------------------------------------- + + +def test_combine_keeps_strictest_outcome() -> None: + run = Decision(outcome="run", reason="r", source="a") + wait = Decision(outcome="wait", reason="w", source="b") + deny = Decision(outcome="deny", reason="d", source="c") + + assert combine_decisions(run, wait).outcome == "wait" + assert combine_decisions(wait, deny).outcome == "deny" + assert combine_decisions(deny, run).outcome == "deny" + assert combine_decisions(run, run).outcome == "run" + assert combine_decisions(wait, wait).outcome == "wait" # primary wins on tie + + +# --------------------------------------------------------------------------- +# PolicyEngine.decide integration +# --------------------------------------------------------------------------- + + +def test_decide_run_path() -> None: + decision = _engine().decide( + status_payload=_run_payload(), + goal_id=GOAL_ID, + scheduler_execution_context=VALID_SCHEDULER_CONTEXT, + ) + assert decision.outcome == "run" + assert decision.source == "quota" + assert decision.reason == "normal_delivery" + + +def test_decide_denied_when_scheduler_context_invalid() -> None: + decision = _engine().decide( + status_payload=_run_payload(), + goal_id=GOAL_ID, + scheduler_execution_context={"host_surface": "bogus"}, + ) + assert decision.outcome == "deny" + assert decision.source == "scheduler" + assert decision.reason == "invalid_scheduler_execution_context" + + +def test_decide_missing_scheduler_context_is_deny() -> None: + decision = _engine().decide(status_payload=_run_payload(), goal_id=GOAL_ID) + assert decision.outcome == "deny" + assert decision.source == "scheduler" + + +def test_decide_capability_gate_blocks_run() -> None: + # quota passes (no capability requirement in status payload), but the + # independently-supplied capability summary requires "network". + payload = _run_payload() + from loopx.control_plane.testing.quota_fixtures import quota_todo_summary + + capability_summary = quota_todo_summary( + [ + { + "index": 1, + "text": "[P1] Network-only slice.", + "role": "agent", + "status": "open", + "priority": "P1", + "task_class": "advancement_task", + "required_capabilities": ["network"], + } + ] + ) + decision = _engine().decide( + status_payload=payload, + goal_id=GOAL_ID, + available_capabilities=["shell"], + scheduler_execution_context=VALID_SCHEDULER_CONTEXT, + capability_agent_todo_summary=capability_summary, + ) + assert decision.outcome == "wait" + assert decision.source == "capability" + assert decision.reason == "capability_repair_bridge" + + +def test_decide_waives_capability_when_all_available() -> None: + payload = _run_payload(required_capabilities=["network"]) + decision = _engine().decide( + status_payload=payload, + goal_id=GOAL_ID, + available_capabilities=["shell", "network"], + scheduler_execution_context=VALID_SCHEDULER_CONTEXT, + capability_agent_todo_summary=payload, + ) + assert decision.outcome == "run" + + +def test_decide_without_capability_summary_relies_on_quota_only() -> None: + payload = _run_payload(required_capabilities=["network"]) + decision = _engine().decide( + status_payload=payload, + goal_id=GOAL_ID, + available_capabilities=["shell"], + scheduler_execution_context=VALID_SCHEDULER_CONTEXT, + ) + # quota/should_run itself embeds capability matching, so the decision is + # already a wait via the quota layer even without an explicit summary. + assert decision.outcome == "wait" + assert decision.source == "quota" + + +def test_decide_quota_backoff_wait_surface() -> None: + payload = quota_status_payload( + goal_id=GOAL_ID, + status="active", + recommended_action="recovery-eligible", + next_action="backoff", + quota_state="backoff", + ) + decision = _engine().decide( + status_payload=payload, + goal_id=GOAL_ID, + scheduler_execution_context=VALID_SCHEDULER_CONTEXT, + ) + assert decision.outcome in {"run", "wait", "deny"} + assert decision.source in {"quota", "scheduler"} + + +def test_module_level_decide_convenience() -> None: + from loopx.control_plane.policy import decide + + decision = decide( + status_payload=_run_payload(), + goal_id=GOAL_ID, + scheduler_execution_context=VALID_SCHEDULER_CONTEXT, + ) + assert decision.outcome == "run" + + +def test_outcome_rank_ordering_is_deny_wait_run() -> None: + assert _OUTCOME_RANK["run"] < _OUTCOME_RANK["wait"] < _OUTCOME_RANK["deny"] diff --git a/tests/control_plane/test_policy_integration.py b/tests/control_plane/test_policy_integration.py new file mode 100644 index 000000000..3152c615a --- /dev/null +++ b/tests/control_plane/test_policy_integration.py @@ -0,0 +1,122 @@ +"""Phase 5/6 integration assessment (RFC §12 Phase 5 — Policy Integration). + +These tests verify that the unified ``PolicyEngine`` adapter produces a +normalized ``Decision`` that is consistent with the existing live decision +path, without modifying the existing execution behavior. They are the +contract gate for any future rewiring of heartbeat decision composition. +""" + +from __future__ import annotations + +from pathlib import Path + +from loopx.control_plane.policy import Decision, PolicyEngine +from loopx.control_plane.policy.engine import decide +from loopx.control_plane.quota.live_decision import ( + build_live_quota_should_run_decision, +) +from loopx.control_plane.scheduler.execution_context import ( + GENERIC_CLI_OUTER_CONTROLLER_SCHEDULER_CONTEXT, +) +from loopx.control_plane.testing.quota_fixtures import quota_status_payload + +GOAL_ID = "integration-fixture" + +VALID_SCHEDULER_CONTEXT = dict(GENERIC_CLI_OUTER_CONTROLLER_SCHEDULER_CONTEXT) + + +def _payload(*, quota_state: str = "active") -> dict: + todo_text = "[P1] Advance the integration slice." + return quota_status_payload( + goal_id=GOAL_ID, + status=quota_state, + agent_todo_items=[ + { + "index": 1, + "text": todo_text, + "role": "agent", + "status": "open", + "priority": "P1", + "task_class": "advancement_task", + } + ], + recommended_action=todo_text, + next_action=todo_text, + ) + + +def _common_kwargs() -> dict: + return { + "goal_id": GOAL_ID, + "agent_id": None, + "available_capabilities": ["shell"], + "include_scheduler_detail": True, + "codex_app_current_rrule": None, + "registry_path": Path("unused"), + "runtime_root": Path("unused"), + "scheduler_execution_context": VALID_SCHEDULER_CONTEXT, + } + + +def test_decide_live_normalizes_existing_decision() -> None: + engine = PolicyEngine() + decision = engine.decide_live(status_payload=_payload(), **_common_kwargs()) + assert isinstance(decision, Decision) + assert decision.outcome in {"run", "wait", "deny"} + assert decision.reason + assert decision.source in {"quota", "capability", "scope", "scheduler"} + + +def test_decide_live_matches_quota_effective_action_semantics() -> None: + engine = PolicyEngine() + decision = engine.decide_live(status_payload=_payload(), **_common_kwargs()) + + live_payload = build_live_quota_should_run_decision( + _payload(), + **_common_kwargs(), + ) + effective_action = live_payload.get("effective_action") + should_run = live_payload.get("should_run") + + # Normalization contract: run maps to should_run=True; wait/deny to False. + if decision.outcome == "run": + assert should_run is True + else: + assert should_run is not True + # The reason preserves the domain-specific effective action. + assert decision.reason or effective_action + + +def test_decide_live_invalid_scheduler_is_deny() -> None: + engine = PolicyEngine() + kwargs = _common_kwargs() + kwargs["scheduler_execution_context"] = {"host_surface": "bogus"} + decision = engine.decide_live(status_payload=_payload(), **kwargs) + assert decision.outcome == "deny" + assert decision.source == "scheduler" + + +def test_module_decide_with_live_signature() -> None: + decision = decide( + status_payload=_payload(), + goal_id=GOAL_ID, + agent_id=None, + available_capabilities=["shell"], + include_scheduler_detail=True, + codex_app_current_rrule=None, + scheduler_execution_context=VALID_SCHEDULER_CONTEXT, + ) + assert decision.outcome in {"run", "wait", "deny"} + + +def test_decision_contract_stable_for_audit() -> None: + """The Decision dict shape must remain stable for audit recording.""" + engine = PolicyEngine() + decision = engine.decide_live(status_payload=_payload(), **_common_kwargs()) + payload = decision.to_dict() + assert set(payload) >= {"outcome", "reason", "source"} + assert payload["outcome"] in {"run", "wait", "deny"} + # Round-trip through the recorder-facing representation. + restored = Decision.from_dict(payload) + assert restored.outcome == decision.outcome + assert restored.reason == decision.reason diff --git a/tests/control_plane/test_policy_pilot_wiring.py b/tests/control_plane/test_policy_pilot_wiring.py new file mode 100644 index 000000000..f7cbaca37 --- /dev/null +++ b/tests/control_plane/test_policy_pilot_wiring.py @@ -0,0 +1,259 @@ +"""RFC §12 Phase 5 pilot wiring tests. + +These tests cover the opt-in wiring added to ``build_live_quota_should_run_decision``: + +* default behavior is unchanged (no ``policy_decision`` key, no env flags); +* enabling ``use_policy_engine`` attaches the unified decision and verifies + consistency against the legacy quota payload; +* enabling ``record_policy_decisions`` writes ``policy_decision`` audit events + to the goal rollout event log (transition-only, deduplicated); +* a deliberately diverged unified decision raises ``PolicyIntegrationError``. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from loopx.control_plane.policy import Decision +from loopx.control_plane.policy.decision_events import policy_decision_events +from loopx.control_plane.quota.live_decision import ( + PolicyIntegrationError, + build_live_quota_should_run_decision, +) +from loopx.control_plane.scheduler.execution_context import ( + GENERIC_CLI_OUTER_CONTROLLER_SCHEDULER_CONTEXT, +) +from loopx.control_plane.testing.quota_fixtures import quota_status_payload +from loopx.rollout_event_log import rollout_event_log_path + +GOAL_ID = "pilot-wiring-fixture" + +VALID_SCHEDULER_CONTEXT = dict(GENERIC_CLI_OUTER_CONTROLLER_SCHEDULER_CONTEXT) + + +def _payload(*, quota_state: str = "active") -> dict: + todo_text = "[P1] Advance the pilot wiring slice." + return quota_status_payload( + goal_id=GOAL_ID, + status=quota_state, + agent_todo_items=[ + { + "index": 1, + "text": todo_text, + "role": "agent", + "status": "open", + "priority": "P1", + "task_class": "advancement_task", + } + ], + recommended_action=todo_text, + next_action=todo_text, + ) + + +def _common_kwargs(runtime_root: Path) -> dict: + return { + "goal_id": GOAL_ID, + "agent_id": None, + "available_capabilities": ["shell"], + "include_scheduler_detail": True, + "codex_app_current_rrule": None, + "registry_path": Path("unused"), + "runtime_root": runtime_root, + "scheduler_execution_context": VALID_SCHEDULER_CONTEXT, + } + + +def test_default_behavior_unchanged(tmp_path: Path) -> None: + """With no env flag the new architecture is ON by default, so the unified + policy_decision is attached while the legacy fields remain intact.""" + payload = build_live_quota_should_run_decision( + _payload(), + **_common_kwargs(tmp_path), + ) + assert "policy_decision" in payload + assert payload.get("should_run") is True + assert payload.get("decision") in {"run", "observe"} + + +def test_master_switch_off_restores_legacy_shape( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """With LOOPX_NEW_ARCHITECTURE=0 the new architecture is off and the legacy + payload shape (no policy_decision) is restored.""" + monkeypatch.setenv("LOOPX_NEW_ARCHITECTURE", "0") + payload = build_live_quota_should_run_decision( + _payload(), + **_common_kwargs(tmp_path), + ) + assert "policy_decision" not in payload + assert payload.get("should_run") is True + assert payload.get("decision") in {"run", "observe"} + + +def test_use_policy_engine_attaches_unified_decision(tmp_path: Path) -> None: + payload = build_live_quota_should_run_decision( + _payload(), + use_policy_engine=True, + **_common_kwargs(tmp_path), + ) + policy_decision = payload.get("policy_decision") + assert isinstance(policy_decision, dict) + assert policy_decision["outcome"] in {"run", "wait", "deny"} + assert policy_decision["source"] in {"quota", "capability", "scope", "scheduler"} + # Consistency: unified outcome must agree with the legacy should_run flag. + if policy_decision["outcome"] == "run": + assert payload.get("should_run") is True + else: + assert payload.get("should_run") is not True + + +def test_use_policy_engine_env_flag(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LOOPX_USE_POLICY_ENGINE", "1") + payload = build_live_quota_should_run_decision( + _payload(), + **_common_kwargs(tmp_path), + ) + assert "policy_decision" in payload + + +def test_record_policy_decisions_writes_audit_event(tmp_path: Path) -> None: + payload = build_live_quota_should_run_decision( + _payload(), + use_policy_engine=True, + record_policy_decisions=True, + **_common_kwargs(tmp_path), + ) + assert "policy_decision" in payload + log_path = rollout_event_log_path(tmp_path, GOAL_ID) + assert log_path.exists() + events = policy_decision_events( + [_line_to_event(line) for line in log_path.read_text(encoding="utf-8").splitlines()] + ) + assert len(events) >= 1 + event = events[-1] + assert event["event_kind"] == "policy_decision" + assert event["goal_id"] == GOAL_ID + assert event["status"] in {"run", "wait", "deny"} + assert event["decision_fingerprint"] + + +def test_record_policy_decisions_is_transition_deduplicated(tmp_path: Path) -> None: + kwargs = _common_kwargs(tmp_path) + for _ in range(3): + build_live_quota_should_run_decision( + _payload(), + use_policy_engine=True, + record_policy_decisions=True, + **kwargs, + ) + log_path = rollout_event_log_path(tmp_path, GOAL_ID) + events = policy_decision_events( + [_line_to_event(line) for line in log_path.read_text(encoding="utf-8").splitlines()] + ) + # Identical decisions collapse into a single transition event. + assert len(events) == 1 + + +def test_invalid_scheduler_context_does_not_raise_deny_is_legit(tmp_path: Path) -> None: + """Root-cause regression: an invalid scheduler execution-context makes the + PolicyEngine deny (scheduler layer) even though the legacy quota payload + still reports ``should_run`` (quota only). This is the intended stricter + ``deny > wait > run`` combination, not a divergence — it must not raise. + """ + kwargs = _common_kwargs(tmp_path) + kwargs["scheduler_execution_context"] = { + "presenter": "non_loopx", + "outer_controller": "unsupported", + } + payload = build_live_quota_should_run_decision( + _payload(), + use_policy_engine=True, + **kwargs, + ) + # The composed policy decision denies via the scheduler layer... + policy_decision = payload["policy_decision"] + assert policy_decision["outcome"] == "deny" + assert policy_decision["source"] == "scheduler" + # ...while the legacy quota flag may still be permissive; this coexistence + # is valid and must not have raised PolicyIntegrationError. + + +def test_policy_integration_error_on_permissive_drift() -> None: + """A composed ``run`` against a non-running quota is a real policy bypass + and must raise (the only genuine divergence left after the root-cause fix). + Directly exercises ``_verify_policy_decision_consistency`` to isolate the + source-aware check. + """ + from loopx.control_plane.quota import live_decision as live_decision_module + + verify = live_decision_module._verify_policy_decision_consistency + # Composed layer (capability) returning ``run`` over a non-running quota. + with pytest.raises(live_decision_module.PolicyIntegrationError): + verify( + {"should_run": False, "decision": "skip"}, + Decision(outcome="run", reason="forced_permissive", source="capability"), + ) + + +def test_stricter_composed_deny_over_quota_run_is_accepted() -> None: + """A composed stricter layer (scheduler/capability) may deny/wait over a + quota ``run`` — the intended ``deny > wait > run`` combination. + """ + from loopx.control_plane.quota import live_decision as live_decision_module + + verify = live_decision_module._verify_policy_decision_consistency + verify( + {"should_run": True, "decision": "run"}, + Decision(outcome="deny", reason="invalid_scheduler_execution_context", source="scheduler"), + ) + verify( + {"should_run": True, "decision": "run"}, + Decision(outcome="wait", reason="capability_repair_bridge", source="capability"), + ) + + +def test_quota_source_must_match_exactly() -> None: + """When PolicyEngine consumed only the quota layer (source == quota), the + outcome must agree with the legacy ``should_run`` exactly. + """ + from loopx.control_plane.quota import live_decision as live_decision_module + + verify = live_decision_module._verify_policy_decision_consistency + # quota-source deny over a running quota is a genuine single-layer mismatch. + with pytest.raises(live_decision_module.PolicyIntegrationError): + verify( + {"should_run": True, "decision": "run"}, + Decision(outcome="deny", reason="forced_divergence", source="quota"), + ) + # quota-source run over a running quota is consistent. + verify( + {"should_run": True, "decision": "run"}, + Decision(outcome="run", reason="normal_delivery", source="quota"), + ) + + +def test_policy_integration_error_on_divergence(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A deliberately diverged unified decision must raise instead of silently + changing behavior.""" + from loopx.control_plane.quota import live_decision as live_decision_module + + class _DivergedEngine: + def decide_live(self, **_: object) -> Decision: + return Decision(outcome="deny", reason="forced_divergence", source="quota") + + monkeypatch.setattr(live_decision_module, "PolicyEngine", lambda: _DivergedEngine()) + with pytest.raises(PolicyIntegrationError): + build_live_quota_should_run_decision( + _payload(), + use_policy_engine=True, + **_common_kwargs(tmp_path), + ) + + +def _line_to_event(line: str) -> dict: + import json + + return json.loads(line) diff --git a/tests/control_plane/test_rich_decision.py b/tests/control_plane/test_rich_decision.py new file mode 100644 index 000000000..0dc5eac26 --- /dev/null +++ b/tests/control_plane/test_rich_decision.py @@ -0,0 +1,91 @@ +"""Tests for the rich Decision action vocabulary (plan/new_plan.md §5, P1). + +PolicyEngine decisions now expose an actionable verb +``ALLOW / DENY / DEFER / RETRY / BLOCK / CANCEL / ESCALATE`` alongside the +normalized outcome, plus optional scheduler-facing hints +(``max_attempts``, ``priority``, ``required_capability``, ``resource_class``). +""" + +from __future__ import annotations + +from loopx.control_plane.policy import PolicyEngine, Decision +from loopx.control_plane.policy.decision import ( + DECISION_ACTION_MAP, + DECISION_MAP, + default_action_for_outcome, + normalize_quota_decision, +) + + +def test_default_action_maps_from_outcome() -> None: + assert default_action_for_outcome("run") == "ALLOW" + assert default_action_for_outcome("deny") == "DENY" + assert default_action_for_outcome("wait") == "DEFER" + + +def test_decision_action_map_outcomes() -> None: + # Every rich action has a stable normalized outcome + reason. + assert DECISION_ACTION_MAP["ALLOW"] == ("run", "allow") + assert DECISION_ACTION_MAP["DENY"] == ("deny", "deny") + assert DECISION_ACTION_MAP["DEFER"][0] == "wait" + assert DECISION_ACTION_MAP["RETRY"][0] == "wait" + assert DECISION_ACTION_MAP["BLOCK"][0] == "wait" + assert DECISION_ACTION_MAP["CANCEL"] == ("deny", "cancel") + assert DECISION_ACTION_MAP["ESCALATE"][0] == "wait" + assert set(DECISION_ACTION_MAP) == { + "ALLOW", + "DENY", + "DEFER", + "RETRY", + "BLOCK", + "CANCEL", + "ESCALATE", + } + + +def test_decision_rich_action_derived_when_unspecified() -> None: + run = Decision(outcome="run", reason="ok", source="quota") + assert run.rich_action == "ALLOW" + deny = Decision(outcome="deny", reason="no", source="capability") + assert deny.rich_action == "DENY" + wait = Decision(outcome="wait", reason="later", source="quota") + assert wait.rich_action == "DEFER" + + +def test_decision_explicit_rich_action_and_hints() -> None: + decision = Decision( + outcome="wait", + reason="transient_error", + source="quota", + action="RETRY", + max_attempts=3, + priority=80, + required_capability="python", + resource_class="gpu", + ) + payload = decision.to_dict() + assert payload["action"] == "RETRY" + assert payload["max_attempts"] == 3 + assert payload["priority"] == 80 + assert payload["required_capability"] == "python" + assert payload["resource_class"] == "gpu" + # Round-trips losslessly. + restored = Decision.from_dict(payload) + assert restored == decision + + +def test_quota_decision_carries_rich_action() -> None: + decision = normalize_quota_decision("run") + assert decision.outcome == "run" + assert decision.rich_action == "ALLOW" + + +def test_policy_engine_decision_rich_action_via_normalization() -> None: + # Every normalized outcome exposes the matching rich action through + # ``rich_action``, so existing consumers keep working unchanged. + engine = PolicyEngine() + for raw_decision, (outcome, _reason) in DECISION_MAP.items(): + normalized = normalize_quota_decision(raw_decision) + assert normalized.outcome == outcome + assert normalized.rich_action == default_action_for_outcome(outcome) + assert engine is not None # engine facade imports cleanly diff --git a/tests/control_plane/test_scheduler_resident_merge.py b/tests/control_plane/test_scheduler_resident_merge.py new file mode 100644 index 000000000..07f40d7e1 --- /dev/null +++ b/tests/control_plane/test_scheduler_resident_merge.py @@ -0,0 +1,678 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from loopx.control_plane.scheduler.event_driven_dispatch import ( + EVENT_DRIVEN_DISPATCH_ENV, + QUEUE_STATUS_CLAIMED, + QUEUE_STATUS_PENDING, + enqueue_tasks, + load_task_queue, + task_queue_path, +) +from loopx.control_plane.scheduler.merge import ( + MERGE_PATH_ENV, + load_todo_items_from_rollout_log, + merge_enabled, + merge_event_driven_and_heartbeat, +) +from loopx.rollout_event_log import ( + build_rollout_event, + load_rollout_events, +) +from loopx.control_plane.scheduler.resident import ( + ResidentScheduler, + WorkerPool, + execute_claimed_task, + finalize_resident_execution, + run_resident_scheduler_bounded, +) +from loopx.event_sourced_state import ( + TODO_ADDED, + TODO_COMPLETED, + AppendOnlyStateEventStore, + make_state_event, +) +from loopx.rollout_event_log import load_rollout_events, rollout_event_log_path + + +def _seed_gate_events(tmp_path: Path, goal_id: str) -> None: + """todo_first done gate unlocks advancement todo_second.""" + store = AppendOnlyStateEventStore(tmp_path / "goals" / goal_id / "events.jsonl") + store.append( + make_state_event( + event_id="evt-gate-add", + goal_id=goal_id, + event_type=TODO_ADDED, + refs={"todo_id": "todo_first"}, + payload={ + "text": "setup done", + "role": "agent", + "excluded_agents": ["agent_worker"], + "unblocks_todo_id": "todo_second", + }, + recorded_at="2026-08-14T00:00:00Z", + ) + ) + store.append( + make_state_event( + event_id="evt-gate-complete", + goal_id=goal_id, + event_type=TODO_COMPLETED, + refs={"todo_id": "todo_first"}, + payload={"note": "done"}, + recorded_at="2026-08-14T00:00:01Z", + ) + ) + store.append( + make_state_event( + event_id="evt-succ-add", + goal_id=goal_id, + event_type=TODO_ADDED, + refs={"todo_id": "todo_second"}, + payload={ + "text": "followup advancement", + "role": "agent", + "task_class": "advancement_task", + "unblocks_todo_id": "todo_first", + }, + recorded_at="2026-08-14T00:00:02Z", + ) + ) + + +def _gate_items() -> list[dict[str, object]]: + return [ + { + "todo_id": "todo_first", + "text": "setup done", + "status": "done", + "excluded_agents": ["agent_worker"], + "unblocks_todo_id": "todo_second", + }, + { + "todo_id": "todo_second", + "text": "followup advancement", + "task_class": "advancement_task", + "unblocks_todo_id": "todo_first", + "status": "open", + }, + ] + + +# --------------------------------------------------------------------------- +# Worker Pool +# --------------------------------------------------------------------------- + + +def test_worker_pool_acquires_next_pending_task( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + queue = task_queue_path(tmp_path, goal_id="pool") + enqueue_tasks(queue, goal_id="pool", todo_ids=["todo_a", "todo_b"], recorded_at="t") + pool = WorkerPool(worker_ids=["worker_one", "worker_two"], runtime_root=tmp_path, goal_id="pool") + claimed = pool.acquire("worker_one") + assert claimed is not None + assert claimed["todo_id"] == "todo_a" + assert claimed["status"] == QUEUE_STATUS_CLAIMED + assert pool.acquired[0]["claimed_by"] == "worker_one" + view = load_task_queue(queue) + assert view["pending_todo_ids"] == ["todo_b"] + + +def test_worker_pool_drain_claims_for_idle_workers( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + queue = task_queue_path(tmp_path, goal_id="pool-drain") + enqueue_tasks(queue, goal_id="pool-drain", todo_ids=["a", "b", "c"], recorded_at="t") + pool = WorkerPool(worker_ids=["w1", "w2"], runtime_root=tmp_path, goal_id="pool-drain") + acquired = pool.drain() + assert [e["todo_id"] for e in acquired] == ["a", "b"] + # Second drain sees both workers busy -> no new acquisitions. + assert pool.drain() == [] + view = load_task_queue(queue) + assert view["claimed_todo_ids"] == ["a", "b"] + assert view["pending_todo_ids"] == ["c"] + + +def test_worker_pool_acquire_respects_opt_in( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv(EVENT_DRIVEN_DISPATCH_ENV, raising=False) + queue = task_queue_path(tmp_path, goal_id="pool-off") + enqueue_tasks(queue, goal_id="pool-off", todo_ids=["a"], recorded_at="t", use_event_driven=True) + pool = WorkerPool( + worker_ids=["w1"], runtime_root=tmp_path, goal_id="pool-off", use_event_driven=False + ) + assert pool.acquire("w1") is None + assert pool.acquired == [] + + +# --------------------------------------------------------------------------- +# Resident Scheduler +# --------------------------------------------------------------------------- + + +def test_resident_scheduler_tick_advances_ready_from_projection( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + goal_id = "resident-tick" + _seed_gate_events(tmp_path, goal_id) + scheduler = ResidentScheduler( + runtime_root=tmp_path, + goal_id=goal_id, + worker_ids=["worker_one"], + use_event_driven=True, + ) + payload = scheduler.tick() + assert payload.get("disabled") is not True + dispatch = payload["event_driven_dispatch"] + assert dispatch["ready_successors"] == ["todo_second"] + assert dispatch["newly_enqueued"] == ["todo_second"] + assert scheduler.tick_count == 1 + + +def test_resident_scheduler_bounded_run_writes_queue( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + goal_id = "resident-bounded" + _seed_gate_events(tmp_path, goal_id) + result = run_resident_scheduler_bounded( + runtime_root=tmp_path, + goal_id=goal_id, + worker_ids=["worker_one"], + max_iterations=1, + use_event_driven=True, + ) + assert result["ok"] is True + assert result["enabled"] is True + assert result["tick_count"] == 1 + queue = result["queue"] + assert "todo_second" in queue["claimed_todo_ids"] or "todo_second" in queue["pending_todo_ids"] + + +def test_resident_scheduler_tick_reconciles_zombie_lease( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """P0 closure: the resident tick runs lease/retry reconciliation each pass.""" + from loopx.control_plane.scheduler.task_lifecycle import ( + claim_next_eligible_task, + ) + + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + goal_id = "resident-reconcile" + _seed_gate_events(tmp_path, goal_id) + queue = task_queue_path(tmp_path, goal_id=goal_id) + # Seed a zombie: enqueue + claim with a short lease that is now expired. + enqueue_tasks(queue, goal_id=goal_id, todo_ids=["todo_zombie"], recorded_at="2026-08-14T00:00:00Z") + claim_next_eligible_task(queue, worker_id="worker_one", lease_seconds=10, now=1000.0) + scheduler = ResidentScheduler( + runtime_root=tmp_path, + goal_id=goal_id, + worker_ids=["worker_one"], + use_event_driven=True, + reconcile=True, + ) + payload = scheduler.tick() + reconcile = ((payload.get("event_driven_dispatch") or {}).get("reconcile")) or {} + assert reconcile.get("expired_count") == 1 + assert reconcile.get("expired_leases") == ["todo_zombie"] + # The reconcile summary is surfaced on the resident payload too. + assert (payload.get("resident_scheduler") or {}).get("reconcile") is not None + + +def test_resident_scheduler_reconcile_disabled_by_default_off( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from loopx.control_plane.scheduler.task_lifecycle import ( + claim_next_eligible_task, + ) + + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + goal_id = "resident-reconcile-off" + _seed_gate_events(tmp_path, goal_id) + queue = task_queue_path(tmp_path, goal_id=goal_id) + enqueue_tasks(queue, goal_id=goal_id, todo_ids=["todo_zombie"], recorded_at="2026-08-14T00:00:00Z") + claim_next_eligible_task(queue, worker_id="worker_one", lease_seconds=10, now=1000.0) + scheduler = ResidentScheduler( + runtime_root=tmp_path, + goal_id=goal_id, + worker_ids=["worker_one"], + use_event_driven=True, + reconcile=False, + ) + payload = scheduler.tick() + reconcile = ((payload.get("event_driven_dispatch") or {}).get("reconcile")) + assert reconcile is None + + +def test_resident_scheduler_bounded_disabled_returns_disabled( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv(EVENT_DRIVEN_DISPATCH_ENV, raising=False) + goal_id = "resident-off" + _seed_gate_events(tmp_path, goal_id) + result = run_resident_scheduler_bounded( + runtime_root=tmp_path, + goal_id=goal_id, + max_iterations=1, + use_event_driven=False, + ) + assert result["enabled"] is False + assert not task_queue_path(tmp_path, goal_id=goal_id).exists() + + +# --------------------------------------------------------------------------- +# Merge: event-driven + heartbeat wiring +# --------------------------------------------------------------------------- + + +def test_merge_enabled_requires_all_flags(monkeypatch: pytest.MonkeyPatch) -> None: + # The new architecture is ON by default (master switch): with all feature + # envs unset, merge inherits the master switch and is enabled. + monkeypatch.delenv(MERGE_PATH_ENV, raising=False) + monkeypatch.delenv(EVENT_DRIVEN_DISPATCH_ENV, raising=False) + monkeypatch.delenv("LOOPX_HEARTBEAT_EVENT_SOURCE", raising=False) + monkeypatch.delenv("LOOPX_NEW_ARCHITECTURE", raising=False) + assert merge_enabled() is True + # An explicit flag always wins over the master switch. + assert merge_enabled(use_merge=False) is False + assert merge_enabled(use_merge=True, use_event_driven=True, use_event_source=True) is True + # Merge still requires the eventing AND event-source layers; disabling any + # of them (explicitly) disables the merge. + assert merge_enabled(use_event_driven=False) is False + assert merge_enabled(use_event_source=False) is False + # The master switch can turn the whole new architecture off. + monkeypatch.setenv("LOOPX_NEW_ARCHITECTURE", "0") + assert merge_enabled() is False + # Merge requires all three layers; with the master switch off, all feature + # envs must be explicitly on to re-enable the merge. + monkeypatch.setenv(MERGE_PATH_ENV, "1") + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + monkeypatch.setenv("LOOPX_HEARTBEAT_EVENT_SOURCE", "1") + assert merge_enabled() is True + + +def test_merge_disabled_writes_nothing(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv(MERGE_PATH_ENV, raising=False) + monkeypatch.delenv(EVENT_DRIVEN_DISPATCH_ENV, raising=False) + goal_id = "merge-off" + payload = merge_event_driven_and_heartbeat( + runtime_root=tmp_path, + goal_id=goal_id, + items=_gate_items(), + completed_todo_id="todo_first", + use_merge=False, + ) + assert payload.get("disabled") is True + assert not rollout_event_log_path(tmp_path, goal_id).exists() + assert not task_queue_path(tmp_path, goal_id=goal_id).exists() + + +def test_merge_enabled_records_heartbeat_and_dispatch( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(MERGE_PATH_ENV, "1") + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + monkeypatch.setenv("LOOPX_HEARTBEAT_EVENT_SOURCE", "1") + goal_id = "merge-on" + payload = merge_event_driven_and_heartbeat( + runtime_root=tmp_path, + goal_id=goal_id, + agent_id="agent_one", + items=_gate_items(), + completed_todo_id="todo_first", + worker_id="worker_one", + use_event_driven=True, + use_event_source=True, + use_merge=True, + recorded_at="2026-08-14T00:00:00Z", + ) + assert payload.get("disabled") is not True + assert payload["heartbeat"]["event"]["event_kind"] == "heartbeat_observed" + dispatch = payload["event_driven_dispatch"] + assert dispatch["ready_successors"] == ["todo_second"] + assert dispatch["newly_enqueued"] == ["todo_second"] + + # Heartbeat fact + task_ready/task_enqueued/task_dispatched events. + events = load_rollout_events(rollout_event_log_path(tmp_path, goal_id)) + kinds = {e.get("event_kind") for e in events} + assert "heartbeat_observed" in kinds + assert "task_ready" in kinds + assert "task_enqueued" in kinds + + +def test_merge_policy_decision_attached_when_status_supplied( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(MERGE_PATH_ENV, "1") + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + monkeypatch.setenv("LOOPX_HEARTBEAT_EVENT_SOURCE", "1") + goal_id = "merge-policy" + status_payload = {"decision": "run", "should_run": True} + payload = merge_event_driven_and_heartbeat( + runtime_root=tmp_path, + goal_id=goal_id, + items=_gate_items(), + completed_todo_id="todo_first", + status_payload=status_payload, + use_event_driven=True, + use_event_source=True, + use_merge=True, + ) + decision = payload["policy_decision"] + assert decision is not None + assert decision["outcome"] in {"run", "wait", "deny"} + assert decision["source"] in {"quota", "scheduler", "capability"} + + +def test_merge_forwards_scheduler_context_to_policy_engine( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # Regression: the merged path must forward scheduler_execution_context from + # the status payload so PolicyEngine validates it instead of returning + # "missing required field" (invalid_scheduler_execution_context). + monkeypatch.setenv(MERGE_PATH_ENV, "1") + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + monkeypatch.setenv("LOOPX_HEARTBEAT_EVENT_SOURCE", "1") + goal_id = "merge-ctx-forward" + status_payload = { + "decision": "run", + "should_run": True, + "scheduler_execution_context": { + "host_surface": "codex_cli", + "scheduler_owner": "agent_cli_loop", + "execution_mode": "interactive", + }, + } + payload = merge_event_driven_and_heartbeat( + runtime_root=tmp_path, + goal_id=goal_id, + items=_gate_items(), + completed_todo_id="todo_first", + status_payload=status_payload, + use_event_driven=True, + use_event_source=True, + use_merge=True, + ) + decision = payload["policy_decision"] + assert decision is not None + # Scheduler context now passes validation; the decision proceeds to the + # quota gate (source != scheduler) instead of short-circuiting on a missing + # execution-context field. + assert decision["source"] != "scheduler" + assert decision["outcome"] in {"run", "wait", "deny"} + + +def test_load_todo_items_from_rollout_log_reconstructs_real_goal_items( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # Regression: real goals record task mutations in rollout-event-log.jsonl + # (todo_add / todo_complete) rather than a separate events.jsonl store, so + # event-driven dispatch must reconstruct items from the rollout log. + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + goal_id = "rollout-items" + log_path = tmp_path / "goals" / goal_id / "rollout-event-log.jsonl" + events = [ + ("todo_a", "todo_add", "open", "agent"), + ("todo_a", "todo_complete", "done", "agent"), + ("todo_b", "todo_add", "open", "user"), + ] + for index, (todo_id, kind, status, role) in enumerate(events): + event = build_rollout_event( + goal_id=goal_id, + event_kind=kind, + agent_id="agent_one", + status=status, + summary=f"{kind} {todo_id}", + details={"role": role, "todo_command": "add" if kind == "todo_add" else "complete"}, + recorded_at="2026-08-14T00:00:00Z", + ) + event["todo_id"] = todo_id + event["event_id"] = f"rollout-r{index}" + log_path.parent.mkdir(parents=True, exist_ok=True) + with log_path.open("a", encoding="utf-8") as fh: + fh.write(__import__("json").dumps(event, ensure_ascii=True) + "\n") + items = load_todo_items_from_rollout_log(tmp_path, goal_id) + by_id = {i["todo_id"]: i for i in items} + assert by_id["todo_a"]["status"] == "done" + assert by_id["todo_a"]["role"] == "agent" + assert by_id["todo_b"]["status"] == "open" + assert by_id["todo_b"]["role"] == "user" + + +def test_load_todo_items_reconstructs_dependencies_and_excluded_agents( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # Regression: dependencies live in causality.unblocks (build_rollout_event's + # ``unblocks`` param), and excluded_agents are stored as stringified lists + # in details. The reconstructor must recover both so handoff gates recompute. + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + goal_id = "rollout-deps" + log_path = tmp_path / "goals" / goal_id / "rollout-event-log.jsonl" + entries = [ + # gate: done, excludes agent_worker, unblocks successor (via causality) + ("todo_gate_abc", "todo_add", "open", "user", ["todo_followup_xyz"], + {"role": "user", "task_class": "user_gate", "excluded_agents": "agent_worker"}), + ("todo_gate_abc", "todo_complete", "done", "user", ["todo_followup_xyz"], + {"role": "user", "task_class": "user_gate", "excluded_agents": "['agent_worker']"}), + ("todo_followup_xyz", "todo_add", "open", "agent", ["todo_gate_abc"], + {"role": "agent", "task_class": "advancement_task", "excluded_agents": ""}), + ] + for index, (todo_id, kind, status, role, unblocks, details) in enumerate(entries): + event = build_rollout_event( + goal_id=goal_id, + event_kind=kind, + agent_id="agent_one", + status=status, + summary=f"{kind} {todo_id}", + unblocks=unblocks, + details=details, + recorded_at="2026-08-14T00:00:00Z", + ) + event["todo_id"] = todo_id + event["event_id"] = f"deps-r{index}" + log_path.parent.mkdir(parents=True, exist_ok=True) + with log_path.open("a", encoding="utf-8") as fh: + fh.write(__import__("json").dumps(event, ensure_ascii=True) + "\n") + items = load_todo_items_from_rollout_log(tmp_path, goal_id) + by_id = {i["todo_id"]: i for i in items} + assert by_id["todo_gate_abc"]["status"] == "done" + # Dependencies recovered from causality.unblocks. + assert by_id["todo_gate_abc"]["unblocks_todo_id"] == "todo_followup_xyz" + assert by_id["todo_followup_xyz"]["unblocks_todo_id"] == "todo_gate_abc" + # Excluded agents recovered from both comma-string and stringified repr list. + assert by_id["todo_gate_abc"]["excluded_agents"] == ["agent_worker"] + assert by_id["todo_followup_xyz"]["excluded_agents"] == [] + + +# --------------------------------------------------------------------------- +# Worker execution (new-architecture opt-in execution, mirrors original gates) +# --------------------------------------------------------------------------- + + +def _fake_runner(ok: bool = True): + def _run(command, **kwargs): + return {"ok": ok, "returncode": 0 if ok else 1, "timed_out": False, + "output_captured": False, "output": ""} + return _run + + +def test_execute_claimed_task_requires_command_guard_and_prefix() -> None: + entry = {"todo_id": "todo_x", "claimed_by": "worker_one"} + # No command + r = execute_claimed_task(entry) + assert r["executed"] is False and r["reason"] == "worker_command_missing" + # Command but no guard + r = execute_claimed_task(entry, worker_command="sed -i s/a/b/g file") + assert r["executed"] is False and r["reason"] == "fresh_quota_guard_confirmation_required" + # Guard but no prefix + r = execute_claimed_task(entry, worker_command="sed -i s/a/b/g file", guard_checked=True) + assert r["executed"] is False and r["reason"] == "worker_command_prefix_required" + # Prefix mismatch + r = execute_claimed_task(entry, worker_command="rm -rf /", guard_checked=True, + worker_command_prefixes=["sed"]) + assert r["executed"] is False and r["reason"] == "worker_command_prefix_mismatch" + + +def test_execute_claimed_task_runs_whitelisted_command() -> None: + entry = {"todo_id": "todo_x", "claimed_by": "worker_one"} + r = execute_claimed_task( + entry, + worker_command="sed -i s/Inter/Poppins/g index.html", + guard_checked=True, + worker_command_prefixes=["sed"], + runner=_fake_runner(ok=True), + ) + assert r["executed"] is True + assert r["reason"] == "executed" + assert r["todo_id"] == "todo_x" + + +def test_resident_bounded_runs_worker_exec_for_claimed_task( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + goal_id = "resident-exec" + _seed_gate_events(tmp_path, goal_id) + result = run_resident_scheduler_bounded( + runtime_root=tmp_path, + goal_id=goal_id, + worker_ids=["worker_one"], + max_iterations=1, + use_event_driven=True, + worker_exec_command="sed -i s/Inter/Poppins/g index.html", + worker_exec_command_prefixes=["sed"], + guard_checked=True, + runner=_fake_runner(ok=True), + ) + tick = result["ticks"][0] + executions = (tick.get("resident_scheduler") or {}).get("worker_executions") or [] + assert executions, "worker should have executed a claimed task" + assert executions[0]["executed"] is True + assert executions[0]["claimed_by"] == "worker_one" + + +def test_resident_closed_loop_complete_acceptance_close( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The full closed loop: claim -> execute -> task_completed -> acceptance -> goal_closed.""" + from loopx.rollout_event_log import load_rollout_events, rollout_event_log_path + from loopx.control_plane.goals.goal_acceptance import build_grep_evidence + + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + goal_id = "resident-closed-loop" + _seed_gate_events(tmp_path, goal_id) + result = run_resident_scheduler_bounded( + runtime_root=tmp_path, + goal_id=goal_id, + worker_ids=["worker_one"], + max_iterations=1, + use_event_driven=True, + worker_exec_command="sed -i s/Inter/Poppins/g index.html", + worker_exec_command_prefixes=["sed"], + guard_checked=True, + runner=_fake_runner(ok=True), + acceptance_criteria=[ + {"criterion_id": "font_poppins", "description": "font is Poppins"}, + ], + evidence=[ + build_grep_evidence( + ref="index.html", + pattern="Poppins", + match=True, + criterion_ids=["font_poppins"], + ) + ], + ) + finalize = result.get("finalize") + assert finalize is not None, "closed loop finalize must run when exec + acceptance declared" + # Task was completed. + assert finalize["task_results"], "completed task result expected" + assert finalize["task_results"][0]["executed"] is True + assert finalize["task_results"][0]["completed"] is True + # Acceptance satisfied. + assert finalize["acceptance"]["satisfied"] is True + # Goal closed. + assert finalize["closed"] is True + assert finalize["closure"]["ready"] is True + # Events: task_completed + goal_closure_ready + goal_closed recorded. + kinds = [e["event_kind"] for e in load_rollout_events( + rollout_event_log_path(tmp_path, goal_id), limit=100 + )] + assert "task_completed" in kinds + assert "goal_closure_ready" in kinds + assert "goal_closed" in kinds + + +def test_resident_closed_loop_acceptance_gap_holds_goal( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """With an acceptance gap, the goal is held (goal_acceptance_pending), not closed.""" + from loopx.rollout_event_log import load_rollout_events, rollout_event_log_path + from loopx.control_plane.goals.goal_acceptance import build_grep_evidence + + monkeypatch.setenv(EVENT_DRIVEN_DISPATCH_ENV, "1") + goal_id = "resident-closed-loop-gap" + _seed_gate_events(tmp_path, goal_id) + result = run_resident_scheduler_bounded( + runtime_root=tmp_path, + goal_id=goal_id, + worker_ids=["worker_one"], + max_iterations=1, + use_event_driven=True, + worker_exec_command="sed -i s/Inter/Poppins/g index.html", + worker_exec_command_prefixes=["sed"], + guard_checked=True, + runner=_fake_runner(ok=True), + acceptance_criteria=[ + {"criterion_id": "font_poppins", "description": "font is Poppins"}, + ], + # Evidence that FAILS (match=False) -> gap -> goal held, not closed. + evidence=[ + build_grep_evidence( + ref="index.html", pattern="Poppins", match=False, criterion_ids=["font_poppins"] + ) + ], + ) + finalize = result.get("finalize") + assert finalize is not None + assert finalize["acceptance"]["satisfied"] is False + assert finalize["closed"] is False + kinds = [e["event_kind"] for e in load_rollout_events( + rollout_event_log_path(tmp_path, goal_id), limit=100 + )] + assert "goal_acceptance_pending" in kinds + assert "goal_closed" not in kinds + + +def test_finalize_resident_execution_direct( + tmp_path: Path, +) -> None: + """Direct unit test of finalize_resident_execution.""" + from loopx.control_plane.scheduler.event_driven_dispatch import enqueue_tasks, task_queue_path + from loopx.control_plane.scheduler.task_lifecycle import claim_next_eligible_task + from loopx.control_plane.goals.goal_acceptance import build_manual_evidence + + goal_id = "finalize-direct" + queue = task_queue_path(tmp_path, goal_id=goal_id) + enqueue_tasks(queue, goal_id=goal_id, todo_ids=["todo_a"], recorded_at="2026-08-14T00:00:00Z") + claim_next_eligible_task(queue, worker_id="worker_one", lease_seconds=100, now=1000.0) + result = finalize_resident_execution( + runtime_root=tmp_path, + goal_id=goal_id, + event_log_path=rollout_event_log_path(tmp_path, goal_id), + executed=[{"todo_id": "todo_a", "claimed_by": "worker_one", "executed": True, "ok": True}], + worker_id="worker_one", + acceptance_criteria=[{"criterion_id": "c1", "description": "d"}], + evidence=[build_manual_evidence(ref="r", content="c", ok=True, criterion_ids=["c1"])], + ) + assert result["task_results"][0]["completed"] is True + assert result["acceptance"]["satisfied"] is True + assert result["closed"] is True diff --git a/tests/control_plane/test_start_goal_compact_projection.py b/tests/control_plane/test_start_goal_compact_projection.py index 0b605dbdd..1ce7006fa 100644 --- a/tests/control_plane/test_start_goal_compact_projection.py +++ b/tests/control_plane/test_start_goal_compact_projection.py @@ -14,8 +14,11 @@ GUIDED_COMMAND_PACK_PROJECTION_SCHEMA_VERSION, build_loopx_bootstrap_command_pack, build_start_goal_guided_packet, + inspect_bootstrap_connection, render_start_goal_guided_markdown, ) +from loopx.control_plane.goals.goal_closure import emit_goal_closed +from loopx.rollout_event_log import rollout_event_log_path from loopx.capabilities.issue_fix.candidate_preflight import ( build_issue_fix_candidate_preflight_packet, candidate_preflight_input_contract, @@ -898,6 +901,56 @@ def test_cli_codex_app_unbound_ambient_thread_defaults_to_fresh_registration( assert gate["fresh_agent_registration"]["recommended"] is True +def _write_connected_project_with_runtime(root: Path) -> tuple[Path, Path]: + """Variant of _write_connected_project that also sets common_runtime_root.""" + project = _write_connected_project(root) + runtime = root / "runtime" + runtime.mkdir() + registry_path = project / ".loopx" / "registry.json" + registry = json.loads(registry_path.read_text(encoding="utf-8")) + registry["common_runtime_root"] = str(runtime) + registry_path.write_text( + json.dumps(registry, indent=2) + "\n", encoding="utf-8" + ) + return project, runtime + + +def test_start_goal_requires_new_goal_when_existing_is_closed(tmp_path: Path) -> None: + """A closed goal must not be reused: start-goal should surface a fresh-goal + requirement instead of appending new todos to a finished goal (the website1 + yellow->purple state-pollution regression).""" + project, runtime = _write_connected_project_with_runtime(tmp_path) + emit_goal_closed( + log_path=rollout_event_log_path(runtime, GOAL_ID), + goal_id=GOAL_ID, + reason="done", + ) + + inspection = inspect_bootstrap_connection(project, goal_id=GOAL_ID) + assert inspection["connection_state"] == "goal_reuse_closed" + assert inspection["should_start_new_goal"] is True + + payload = build_start_goal_guided_packet( + project=project, + goal_id=GOAL_ID, + agent_id=AGENT_ID, + cli_bin="loopx", + host_surface="codex-app", + goal_text=GOAL_TEXT, + ) + step = payload.get("recommended_next_step", {}) + assert step["kind"] == "goal_reuse_closed_require_new_goal" + assert step["suggested_new_goal_id"] == f"{GOAL_ID}-2" + + +def test_start_goal_reuses_open_goal_without_new_goal_signal(tmp_path: Path) -> None: + """An open goal (no goal_closed event) keeps the normal connected path.""" + project, _runtime = _write_connected_project_with_runtime(tmp_path) + inspection = inspect_bootstrap_connection(project, goal_id=GOAL_ID) + assert inspection["connection_state"] == "connected" + assert inspection.get("should_start_new_goal") is None + + def test_start_goal_new_peer_explicitly_allows_fresh_registration(tmp_path: Path) -> None: project = _write_connected_project(tmp_path) diff --git a/tests/control_plane/test_task_lifecycle.py b/tests/control_plane/test_task_lifecycle.py new file mode 100644 index 000000000..afd3de772 --- /dev/null +++ b/tests/control_plane/test_task_lifecycle.py @@ -0,0 +1,362 @@ +"""Tests for the task lifecycle module (lease, retry, idempotency, capability). + +Covers ``plan/new_plan.md`` P0/P1: +* lease expiry re-enqueues zombie tasks (``claimed|running -> expired -> pending``); +* retry with backoff and ``max_attempts`` (``failed -> retry_wait -> pending``, + ``failed`` on exhausted attempts, ``dead_letter`` escalation); +* generation-aware ``task_id`` idempotency; +* capability-matched claiming. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from loopx.control_plane.scheduler.event_driven_dispatch import ( + QUEUE_STATUS_CLAIMED, + QUEUE_STATUS_DONE, + QUEUE_STATUS_PENDING, + enqueue_tasks, + task_queue_path, +) +from loopx.control_plane.scheduler.task_lifecycle import ( + QUEUE_STATUS_CANCELLED, + QUEUE_STATUS_DEAD_LETTER, + QUEUE_STATUS_FAILED, + QUEUE_STATUS_RETRY_WAIT, + TASK_ID_SEPARATOR, + build_task_id, + cancel_task, + claim_next_eligible_task, + complete_task, + dead_letter_exhausted, + eligible, + expire_stale_leases, + extended_queue_view, + fail_task, + is_expired, + parse_task_id, + promote_retry_ready, + reconcile_queue, + requeue_failed, + task_generation, + worker_satisfies_capabilities, +) + + +def _queue(tmp_path: Path, goal_id: str = "goal") -> Path: + return task_queue_path(tmp_path, goal_id=goal_id) + + +def _raw_entries(path: Path) -> list[dict[str, Any]]: + return [ + json.loads(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +# --------------------------------------------------------------------------- +# Generation-aware task id (idempotency) +# --------------------------------------------------------------------------- + + +def test_build_and_parse_task_id() -> None: + task_id = build_task_id("todo_a", generation=3) + assert task_id == f"todo_a{TASK_ID_SEPARATOR}3" + assert parse_task_id(task_id) == ("todo_a", 3) + assert task_generation(task_id) == 3 + + +def test_build_task_id_defaults_generation_zero() -> None: + assert build_task_id("todo_a") == f"todo_a{TASK_ID_SEPARATOR}0" + assert task_generation(build_task_id("todo_a")) == 0 + + +def test_parse_task_id_plain_todo_returns_none() -> None: + assert parse_task_id("todo_a") is None + assert task_generation("todo_a") == 0 + + +# --------------------------------------------------------------------------- +# Capability matching +# --------------------------------------------------------------------------- + + +def test_eligible_task_without_requirements_any_worker() -> None: + assert eligible({"capabilities": []}, {"todo_id": "t"}) is True + assert eligible({"capabilities": ["python"]}, {"todo_id": "t"}) is True + + +def test_eligible_matches_all_required_capabilities() -> None: + task = {"required_capabilities": ["python", "gpu"]} + assert eligible({"capabilities": ["python", "gpu"]}, task) is True + assert eligible({"capabilities": ["python"]}, task) is False + assert eligible({"capabilities": []}, task) is False + + +def test_worker_satisfies_capabilities_normalizes_tokens() -> None: + assert worker_satisfies_capabilities(["python", "gpu"], ["PYTHON"]) is True + assert worker_satisfies_capabilities("python, gpu", "python") is True + assert worker_satisfies_capabilities(["python"], ["latex"]) is False + + +# --------------------------------------------------------------------------- +# Claim with lease + capability matching +# --------------------------------------------------------------------------- + + +def test_claim_next_eligible_task_adds_lease_and_attempt( + tmp_path: Path, +) -> None: + path = _queue(tmp_path) + enqueue_tasks(path, goal_id="goal", todo_ids=["todo_a"], recorded_at="2026-08-14T00:00:00Z") + claimed = claim_next_eligible_task(path, worker_id="worker_one", lease_seconds=100, now=1000.0) + assert claimed is not None + assert claimed["status"] == QUEUE_STATUS_CLAIMED + assert claimed["claimed_by"] == "worker_one" + assert claimed["lease_until"] == 1100.0 + assert claimed["attempt"] == 1 + + +def test_claim_next_eligible_task_capability_gate(tmp_path: Path) -> None: + path = _queue(tmp_path) + enqueue_tasks( + path, + goal_id="goal", + todo_ids=["todo_gpu", "todo_py"], + recorded_at="2026-08-14T00:00:00Z", + ) + entries = _raw_entries(path) + gpu = entries[0] + gpu["required_capabilities"] = ["gpu"] + py = entries[1] + py["required_capabilities"] = ["python"] + path.write_text( + "".join(json.dumps(e, sort_keys=True) + "\n" for e in [gpu, py]), + encoding="utf-8", + ) + # Worker with only python cannot claim the gpu task; claims the python one. + claimed = claim_next_eligible_task(path, worker_id="w1", capabilities=["python"]) + assert claimed is not None + assert claimed["todo_id"] == "todo_py" + # A worker with gpu claims the gpu task next. + claimed2 = claim_next_eligible_task(path, worker_id="w2", capabilities=["gpu"]) + assert claimed2 is not None + assert claimed2["todo_id"] == "todo_gpu" + + +def test_claim_next_eligible_task_none_when_no_eligible(tmp_path: Path) -> None: + path = _queue(tmp_path) + enqueue_tasks( + path, + goal_id="goal", + todo_ids=["todo_gpu"], + recorded_at="2026-08-14T00:00:00Z", + ) + entries = _raw_entries(path) + entries[0]["required_capabilities"] = ["gpu"] + path.write_text(json.dumps(entries[0], sort_keys=True) + "\n", encoding="utf-8") + assert claim_next_eligible_task(path, worker_id="w1", capabilities=["python"]) is None + + +# --------------------------------------------------------------------------- +# Lease expiry (zombie recovery) +# --------------------------------------------------------------------------- + + +def test_is_expired_only_for_claimed_running() -> None: + claimed = {"status": QUEUE_STATUS_CLAIMED, "lease_until": 100.0} + assert is_expired(claimed, now=101.0) is True + assert is_expired(claimed, now=100.0) is True + assert is_expired(claimed, now=99.0) is False + pending = {"status": QUEUE_STATUS_PENDING, "lease_until": 1.0} + assert is_expired(pending, now=100.0) is False + + +def test_expire_stale_leases_reenqueues_zombie(tmp_path: Path) -> None: + path = _queue(tmp_path) + enqueue_tasks(path, goal_id="goal", todo_ids=["todo_zombie"], recorded_at="2026-08-14T00:00:00Z") + claimed = claim_next_eligible_task(path, worker_id="w1", lease_seconds=10, now=1000.0) + assert claimed is not None + assert is_expired(claimed, now=1005.0) is False + expired = expire_stale_leases(path, now=1011.0, worker_id="scheduler") + assert [e["todo_id"] for e in expired] == ["todo_zombie"] + view = extended_queue_view(path) + # The zombie task was re-enqueued to pending (not stuck in claimed). + entries = _raw_entries(path) + assert entries[0]["status"] == QUEUE_STATUS_PENDING + assert entries[0]["expired_at"] is not None + assert view["pending_count"] == 1 + assert view["extended"]["expired_count"] == 0 # expired is transient, not terminal + + +def test_expire_stale_leases_leaves_fresh_claimed_alone(tmp_path: Path) -> None: + path = _queue(tmp_path) + enqueue_tasks(path, goal_id="goal", todo_ids=["todo_fresh"], recorded_at="2026-08-14T00:00:00Z") + claim_next_eligible_task(path, worker_id="w1", lease_seconds=100, now=1000.0) + assert expire_stale_leases(path, now=1010.0) == [] + entries = _raw_entries(path) + assert entries[0]["status"] == QUEUE_STATUS_CLAIMED + + +def test_expire_stale_leases_recovers_legacy_unleased_zombie( + tmp_path: Path, +) -> None: + """Regression: claimed entries WITHOUT a lease_until (legacy/old-code zombies) + must be reclaimed, not stuck forever. + + This reproduces the website1 session's ``claimed: 4`` stale queue — four + tasks claimed by an older run without leases, which blocked the event-driven + path from making progress (pending=0, nothing claimable). + """ + path = _queue(tmp_path) + enqueue_tasks(path, goal_id="goal", todo_ids=["todo_stale"], recorded_at="2026-08-14T00:00:00Z") + # Simulate an old-code claim: status=claimed but NO lease_until written. + entries = _raw_entries(path) + entries[0]["status"] = QUEUE_STATUS_CLAIMED + entries[0]["claimed_by"] = "ghost_worker" + path.write_text( + "".join(json.dumps(e, sort_keys=True) + "\n" for e in entries), + encoding="utf-8", + ) + # A claimed entry without a lease is treated as an expired zombie. + assert is_expired(entries[0]) is True + reclaimed = expire_stale_leases(path, worker_id="scheduler") + assert [e["todo_id"] for e in reclaimed] == ["todo_stale"] + # The zombie is back to pending and claimable again. + re_entries = _raw_entries(path) + assert re_entries[0]["status"] == QUEUE_STATUS_PENDING + assert re_entries[0].get("expired_by") == "scheduler" + + +# --------------------------------------------------------------------------- +# Completion / failure / retry / dead letter +# --------------------------------------------------------------------------- + + +def test_complete_task_transitions_to_done(tmp_path: Path) -> None: + path = _queue(tmp_path) + enqueue_tasks(path, goal_id="goal", todo_ids=["todo_a"], recorded_at="2026-08-14T00:00:00Z") + claim_next_eligible_task(path, worker_id="w1", lease_seconds=100, now=1000.0) + completed = complete_task(path, task_id="todo_a", worker_id="w1") + assert completed is not None + assert completed["status"] == QUEUE_STATUS_DONE + assert completed["lease_until"] is None + # Re-completing the same task is a no-op (already done). + assert complete_task(path, task_id="todo_a", worker_id="w1") is None + + +def test_complete_task_respects_claimer(tmp_path: Path) -> None: + path = _queue(tmp_path) + enqueue_tasks(path, goal_id="goal", todo_ids=["todo_a"], recorded_at="2026-08-14T00:00:00Z") + claim_next_eligible_task(path, worker_id="w1", lease_seconds=100, now=1000.0) + # A different worker cannot complete someone else's claimed task. + assert complete_task(path, task_id="todo_a", worker_id="w2") is None + + +def test_fail_task_transient_retry_wait_then_promote(tmp_path: Path) -> None: + path = _queue(tmp_path) + enqueue_tasks(path, goal_id="goal", todo_ids=["todo_a"], recorded_at="2026-08-14T00:00:00Z") + claim_next_eligible_task(path, worker_id="w1", lease_seconds=100, now=1000.0) + failed = fail_task( + path, + task_id="todo_a", + worker_id="w1", + error="boom", + transient=True, + max_attempts=3, + retry_backoff_seconds=60, + now=1010.0, + ) + assert failed is not None + assert failed["status"] == QUEUE_STATUS_RETRY_WAIT + assert failed["retry_at"] == 1070.0 + # Backoff not elapsed -> stays in retry_wait. + assert promote_retry_ready(path, now=1069.0) == [] + # Backoff elapsed -> promoted to pending. + promoted = promote_retry_ready(path, now=1070.0) + assert [e["todo_id"] for e in promoted] == ["todo_a"] + entries = _raw_entries(path) + assert entries[0]["status"] == QUEUE_STATUS_PENDING + + +def test_fail_task_permanent_when_not_transient(tmp_path: Path) -> None: + path = _queue(tmp_path) + enqueue_tasks(path, goal_id="goal", todo_ids=["todo_a"], recorded_at="2026-08-14T00:00:00Z") + claim_next_eligible_task(path, worker_id="w1", lease_seconds=100, now=1000.0) + failed = fail_task(path, task_id="todo_a", worker_id="w1", transient=False) + assert failed is not None + assert failed["status"] == QUEUE_STATUS_FAILED + assert failed.get("retry_at") is None + + +def test_fail_task_dead_letter_when_attempts_exhausted(tmp_path: Path) -> None: + path = _queue(tmp_path) + enqueue_tasks(path, goal_id="goal", todo_ids=["todo_a"], recorded_at="2026-08-14T00:00:00Z") + # First claim attempts = 1; max_attempts=1 -> no retry allowed -> failed. + claim_next_eligible_task(path, worker_id="w1", lease_seconds=100, now=1000.0) + failed = fail_task( + path, + task_id="todo_a", + worker_id="w1", + transient=True, + max_attempts=1, + now=1001.0, + ) + assert failed["status"] == QUEUE_STATUS_FAILED + # Explicit dead-letter escalation. + dead = dead_letter_exhausted(path, task_id="todo_a", worker_id="scheduler") + assert dead is not None + assert dead["status"] == QUEUE_STATUS_DEAD_LETTER + + +def test_requeue_failed_returns_to_pending(tmp_path: Path) -> None: + path = _queue(tmp_path) + enqueue_tasks(path, goal_id="goal", todo_ids=["todo_a"], recorded_at="2026-08-14T00:00:00Z") + claim_next_eligible_task(path, worker_id="w1", lease_seconds=100, now=1000.0) + fail_task(path, task_id="todo_a", worker_id="w1", transient=False) + requeued = requeue_failed(path, task_id="todo_a", worker_id="scheduler") + assert requeued is not None + assert requeued["status"] == QUEUE_STATUS_PENDING + + +def test_cancel_task(tmp_path: Path) -> None: + path = _queue(tmp_path) + enqueue_tasks(path, goal_id="goal", todo_ids=["todo_a"], recorded_at="2026-08-14T00:00:00Z") + cancelled = cancel_task(path, task_id="todo_a", reason="owner nixed it") + assert cancelled is not None + assert cancelled["status"] == QUEUE_STATUS_CANCELLED + assert cancelled["cancel_reason"] == "owner nixed it" + view = extended_queue_view(path) + assert view["extended"]["cancelled_count"] == 1 + + +# --------------------------------------------------------------------------- +# Reconciliation +# --------------------------------------------------------------------------- + + +def test_reconcile_queue_handles_zombie_and_retry(tmp_path: Path) -> None: + path = _queue(tmp_path) + # Task A: zombie (claimed, lease expired). + enqueue_tasks(path, goal_id="goal", todo_ids=["todo_a", "todo_b"], recorded_at="2026-08-14T00:00:00Z") + claim_next_eligible_task(path, worker_id="w1", lease_seconds=10, now=1000.0) + # Task B: retry_wait whose backoff has elapsed. + claim_next_eligible_task(path, worker_id="w1", lease_seconds=100, now=1000.0) + fail_task(path, task_id="todo_b", worker_id="w1", transient=True, max_attempts=3, retry_backoff_seconds=5, now=1001.0) + result = reconcile_queue(path, now=1012.0) + assert result["expired_count"] == 1 + assert result["expired_leases"] == ["todo_a"] + assert result["retry_promoted_count"] == 1 + assert result["retry_promoted"] == ["todo_b"] + + +def test_extended_queue_view_counts_states(tmp_path: Path) -> None: + path = _queue(tmp_path) + enqueue_tasks(path, goal_id="goal", todo_ids=["todo_a"], recorded_at="2026-08-14T00:00:00Z") + view = extended_queue_view(path) + assert view["pending_count"] == 1 + assert view["extended"]["dead_letter_count"] == 0 diff --git a/tests/control_plane/test_todo_mutation_authority.py b/tests/control_plane/test_todo_mutation_authority.py index 82d27f616..831d60623 100644 --- a/tests/control_plane/test_todo_mutation_authority.py +++ b/tests/control_plane/test_todo_mutation_authority.py @@ -15,6 +15,8 @@ TaskLeaseError, acquire_task_lease, ) +from loopx.control_plane.scheduler.merge import load_todo_items_from_rollout_log +from loopx.rollout_event_log import load_rollout_events, rollout_event_log_path from loopx.event_sourced_state import ( TODO_ADDED, TODO_UPDATED, @@ -919,6 +921,29 @@ def test_delegated_orchestrator_requires_reason_before_state_write( assert state.read_text(encoding="utf-8") == before +def test_todo_add_bridges_rollout_event_for_event_driven_dispatch( + tmp_path: Path, +) -> None: + """A todo created via add_goal_todo must be visible to the event-driven + scheduler (load_todo_items_from_rollout_log), not just the markdown file.""" + registry, _state = _write_fixture(tmp_path) + todo = _add_agent_todo(registry) + + runtime_root = tmp_path / "runtime" + log_path = rollout_event_log_path(runtime_root, GOAL_ID) + assert log_path.exists(), "todo add must append a rollout todo_add event" + + kinds = [e["event_kind"] for e in load_rollout_events(log_path)] + assert "todo_add" in kinds + + items = load_todo_items_from_rollout_log(runtime_root, GOAL_ID) + added = next((it for it in items if it["todo_id"] == todo["todo_id"]), None) + assert added is not None + assert added["status"] == "open" + assert added["role"] == "agent" + assert added["task_class"] == "advancement_task" + + def test_delegated_orchestration_authority_is_action_scoped( tmp_path: Path, ) -> None: diff --git a/tests/opencode_goal_bridge_runtime.test.mjs b/tests/opencode_goal_bridge_runtime.test.mjs index 3bb844b29..9258ccd81 100644 --- a/tests/opencode_goal_bridge_runtime.test.mjs +++ b/tests/opencode_goal_bridge_runtime.test.mjs @@ -552,3 +552,105 @@ test("completes only after LoopX validates terminal no-follow-up", async () => { assert.equal(await fixture.store.read("session-terminal"), null) assert.equal(fixture.calls.event, 0) }) + + +function policyDecision({ outcome, retryAfterSeconds, retryAt }) { + const unified = { outcome, source: "quota" } + if (retryAfterSeconds !== undefined) unified.retry_after_seconds = retryAfterSeconds + if (retryAt !== undefined) unified.retry_at = retryAt + return { + policy_decision: unified, + should_run: outcome === "run", + effective_action: outcome === "run" ? "run_now" : "backoff_waiting_for_user", + } +} + +function policyTerminalDecision() { + return { + should_run: false, + effective_action: "terminal_no_followup", + policy_decision: { outcome: "deny", source: "quota" }, + goal_frontier_projection: { + terminal_state: { + schema_version: "goal_terminal_state_v0", + kind: "no_followup", + derived: true, + source: "validated_goal_closure", + }, + source_completeness: { + schema_version: "goal_terminal_source_completeness_v0", + user_todos: "valid", + agent_todos: "valid", + }, + }, + } +} + + +test("policy_decision run outcome continues immediately like legacy run_now", async () => { + const fixture = harness(policyDecision({ outcome: "run" })) + const hooks = await fixture.plugin({ directory: "/workspace", client: {} }) + await hooks.tool.loopx_goal_activate.execute( + { goalId: "goal-policy-run", objective: "LoopX task body" }, + { sessionID: "session-policy-run" }, + ) + await hooks.event({ event: { type: "session.idle", properties: { sessionID: "session-policy-run" } } }) + + assert.equal(fixture.calls.event, 1) + assert.equal(fixture.scheduled.length, 0) + const binding = await fixture.store.read("session-policy-run") + assert.equal(binding.autoResume, true) +}) + + +test("policy_decision deny without a validated terminal holds the loop", async () => { + // deny alone is not a validated goal closure; the loop must neither continue, + // schedule, nor self-close the goal. + const fixture = harness(policyDecision({ outcome: "deny" })) + const hooks = await fixture.plugin({ directory: "/workspace", client: {} }) + await hooks.tool.loopx_goal_activate.execute( + { goalId: "goal-policy-deny", objective: "LoopX task body" }, + { sessionID: "session-policy-deny" }, + ) + await hooks.event({ event: { type: "session.idle", properties: { sessionID: "session-policy-deny" } } }) + + assert.equal(fixture.calls.event, 0) + assert.equal(fixture.calls.complete, 0) + assert.equal(fixture.scheduled.length, 0) + const binding = await fixture.store.read("session-policy-deny") + assert.notEqual(binding, null) + assert.equal(binding.terminal, undefined) +}) + + +test("policy_decision wait uses retry_after_seconds for the backoff timer", async () => { + const fixture = harness(policyDecision({ outcome: "wait", retryAfterSeconds: 300 })) + const hooks = await fixture.plugin({ directory: "/workspace", client: {} }) + await hooks.tool.loopx_goal_activate.execute( + { goalId: "goal-policy-wait", objective: "LoopX task body" }, + { sessionID: "session-policy-wait" }, + ) + await hooks.event({ event: { type: "session.idle", properties: { sessionID: "session-policy-wait" } } }) + + assert.equal(fixture.calls.event, 0) + assert.equal(fixture.scheduled.length, 1) + assert.equal(fixture.scheduled[0].delay, 300_000) + assert.equal(fixture.scheduled[0].cleared, false) +}) + + +test("policy_decision deny plus validated terminal projection completes the goal", async () => { + const fixture = harness(policyTerminalDecision()) + const hooks = await fixture.plugin({ directory: "/workspace", client: {} }) + await hooks.tool.loopx_goal_activate.execute( + { goalId: "goal-policy-terminal", objective: "LoopX task body" }, + { sessionID: "session-policy-terminal" }, + ) + await hooks.event({ + event: { type: "session.idle", properties: { sessionID: "session-policy-terminal" } }, + }) + + assert.equal(fixture.calls.complete, 1) + assert.equal(await fixture.store.read("session-policy-terminal"), null) + assert.equal(fixture.calls.event, 0) +}) diff --git a/tests/pi_goal_loop_runtime.test.mjs b/tests/pi_goal_loop_runtime.test.mjs index b6d3e8224..c6e90b6c8 100644 --- a/tests/pi_goal_loop_runtime.test.mjs +++ b/tests/pi_goal_loop_runtime.test.mjs @@ -11,8 +11,11 @@ import { createEphemeralSessionIdentity, createGoalLoop, createMemoryBindingStore, + isTerminalNoFollowup, + policyDecisionOf, sanitizedKey, sessionKey, + shouldRunNow, waitPlan, } from "../loopx/pi_goal_mode/pi-goal-loop-runtime.mjs" @@ -105,6 +108,56 @@ function backoffDecision() { } +// Phase 5 unified policy-decision payloads (control_plane PolicyEngine). The +// loop must prefer `policy_decision.outcome` when present and fall back to the +// legacy quota fields otherwise. +function policyDecision({ outcome, retryAfterSeconds, retryAt, source }) { + const unified = { outcome, source: source || "quota" } + if (retryAfterSeconds !== undefined) unified.retry_after_seconds = retryAfterSeconds + if (retryAt !== undefined) unified.retry_at = retryAt + return { + policy_decision: unified, + // Legacy fields still travel alongside the unified decision. + should_run: outcome === "run", + effective_action: outcome === "run" ? "run_now" : "backoff_waiting_for_user", + } +} + +function policyRunDecision() { + return policyDecision({ outcome: "run" }) +} + +function policyWaitDecision(retryAfterSeconds) { + return policyDecision({ outcome: "wait", retryAfterSeconds }) +} + +function policyDenyDecision(retryAfterSeconds) { + return policyDecision({ outcome: "deny", retryAfterSeconds }) +} + +// A legacy terminal decision carrying the validated goal-closure projection. +function policyTerminalDecision() { + return { + should_run: false, + effective_action: "terminal_no_followup", + policy_decision: { outcome: "deny", source: "quota" }, + goal_frontier_projection: { + terminal_state: { + schema_version: "goal_terminal_state_v0", + kind: "no_followup", + derived: true, + source: "validated_goal_closure", + }, + source_completeness: { + schema_version: "goal_terminal_source_completeness_v0", + user_todos: "valid", + agent_todos: "valid", + }, + }, + } +} + + function gatedStore() { const inner = memoryBindingStore() const writes = [] @@ -414,6 +467,95 @@ test("wait plan resets the unchanged count when the scheduler token changes", () }) +test("policy_decision run outcome continues immediately like legacy run_now", async () => { + const fixture = harness(policyRunDecision()) + fixture.loop.bind("session-policy-run", fixture.services) + await fixture.activate("session-policy-run") + await fixture.loop.settle("session-policy-run") + + assert.equal(fixture.calls.quota, 1) + assert.equal(fixture.calls.send, 1) + assert.equal(fixture.calls.messages[0], "LoopX task body") + assert.equal(fixture.scheduled.length, 0) + const binding = await fixture.store.read("session-policy-run") + assert.equal(binding.lastInjectedPrompt, "LoopX task body") + assert.equal(binding.autoResume, true) +}) + + +test("policy_decision deny without a validated terminal holds the loop", async () => { + // deny alone is not a validated goal closure; the loop must neither send a + // message nor schedule a backoff timer, and must not self-close the goal. + const fixture = harness(policyDenyDecision()) + fixture.loop.bind("session-policy-deny", fixture.services) + await fixture.activate("session-policy-deny") + await fixture.loop.settle("session-policy-deny") + + assert.equal(fixture.calls.quota, 1) + assert.equal(fixture.calls.send, 0) + assert.equal(fixture.scheduled.length, 0) + const binding = await fixture.store.read("session-policy-deny") + assert.equal(binding.terminal, false) + assert.equal(binding.autoResume, true) +}) + + +test("policy_decision wait uses retry_after_seconds for the backoff timer", async () => { + const fixture = harness(policyWaitDecision(300)) // 300s -> 5 min + fixture.loop.bind("session-policy-wait", fixture.services) + await fixture.activate("session-policy-wait") + await fixture.loop.settle("session-policy-wait") + + assert.equal(fixture.calls.send, 0) + assert.equal(fixture.scheduled.length, 1) + assert.equal(fixture.scheduled[0].delayMs, 300_000) + assert.equal(fixture.scheduled[0].cleared, false) +}) + + +test("policy_decision deny plus validated terminal projection stops the loop", async () => { + const fixture = harness(policyTerminalDecision()) + fixture.loop.bind("session-policy-terminal", fixture.services) + await fixture.activate("session-policy-terminal") + const notifyAfterActivate = fixture.calls.notify + await fixture.loop.settle("session-policy-terminal") + + assert.equal(fixture.calls.send, 0) + assert.equal(fixture.calls.notify, notifyAfterActivate + 1) + assert.equal(fixture.scheduled.length, 0) + const binding = await fixture.store.read("session-policy-terminal") + assert.equal(binding.terminal, true) + assert.equal(binding.autoResume, false) +}) + + +test("policy_decisionOf rejects unknown outcomes and missing objects", () => { + assert.equal(policyDecisionOf({ policy_decision: { outcome: "run" } }).outcome, "run") + assert.equal(policyDecisionOf({ policy_decision: { outcome: "wait" } }).outcome, "wait") + assert.equal(policyDecisionOf({ policy_decision: { outcome: "deny" } }).outcome, "deny") + // Unknown / malformed unified decisions are ignored so the legacy path wins. + assert.equal(policyDecisionOf({ policy_decision: { outcome: "weird" } }), null) + assert.equal(policyDecisionOf({ policy_decision: "not-an-object" }), null) + assert.equal(policyDecisionOf({}), null) + assert.equal(policyDecisionOf(null), null) +}) + + +test("shouldRunNow falls back to legacy fields when no policy_decision", () => { + assert.equal(shouldRunNow({ should_run: true, scheduler_hint: { action: "run_now" } }), true) + assert.equal(shouldRunNow({ should_run: false, scheduler_hint: { action: "backoff" } }), false) + assert.equal(shouldRunNow({}), false) +}) + + +test("isTerminalNoFollowup requires the validated frontier even with deny", () => { + // deny without the closure projection is not terminal. + assert.equal(isTerminalNoFollowup(policyDenyDecision()), false) + // deny with the validated projection is terminal. + assert.equal(isTerminalNoFollowup(policyTerminalDecision()), true) +}) + + test("binding store round-trips through the filesystem and retires cleanly", async () => { const root = await fs.mkdtemp(path.join(os.tmpdir(), "pi-goal-loop-store-")) try { diff --git a/tests/test_claude_goal_policy.py b/tests/test_claude_goal_policy.py new file mode 100644 index 000000000..df213be48 --- /dev/null +++ b/tests/test_claude_goal_policy.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +# goal_policy.py is a standalone hook script that imports its sibling modules by +# bare name (e.g. `from goal_state import active_context`), so load it via the +# hooks directory rather than the package namespace. +_HOOKS = Path(__file__).resolve().parents[1] / "loopx" / "claude_goal_mode" / "hooks" +sys.path.insert(0, str(_HOOKS)) + +import goal_policy # noqa: E402 + +resolve_should_run = goal_policy.resolve_should_run + + +def test_resolve_should_run_prefers_policy_decision() -> None: + # New architecture (on by default): the unified decision is authoritative. + # A deny outcome over a permissive quota should_run must NOT run. + assert resolve_should_run({"should_run": True, "policy_decision": {"outcome": "deny"}}) is False + assert resolve_should_run({"should_run": True, "policy_decision": {"outcome": "wait"}}) is False + assert resolve_should_run({"should_run": True, "policy_decision": {"outcome": "run"}}) is True + # deny/wait over a non-running quota stays not-running. + assert resolve_should_run({"should_run": False, "policy_decision": {"outcome": "deny"}}) is False + + +def test_resolve_should_run_falls_back_to_legacy_should_run() -> None: + # No policy_decision (opt-out / legacy): the quota should_run bool decides. + assert resolve_should_run({"should_run": True}) is True + assert resolve_should_run({"should_run": False}) is False + assert resolve_should_run({"should_run": "not-a-bool"}) is None + assert resolve_should_run({}) is None + + +def test_resolve_should_run_ignores_malformed_policy_decision() -> None: + # A malformed policy_decision falls back to the legacy should_run. + assert resolve_should_run({"should_run": True, "policy_decision": "junk"}) is True + assert resolve_should_run({"should_run": True, "policy_decision": {"outcome": "weird"}}) is True + assert resolve_should_run({"should_run": False, "policy_decision": {}}) is False