diff --git a/.githooks/_resolve b/.githooks/_resolve index 171414d..0f79ef7 100644 --- a/.githooks/_resolve +++ b/.githooks/_resolve @@ -9,17 +9,26 @@ # $HARNESS is the path recorded by `record_harness`. # # Then, git hook does not require `uv` or `pip` or any specific environment layout. +# +# if is-agent-in-loop, check harness executable found via harness-path +[ "${RALPH_LOOP:-0}" = "1" ] || exit 0 recorded="$(git rev-parse --path-format=absolute --git-common-dir)/harness-path" -if [ ! -r "$recorded" ]; then - echo "loopgate: hooks are not installed. Run 'harness install' in this repo." >&2 - exit 1 +HARNESS="" + +if [ -r "$recorded" ]; then + HARNESS="$(cat "$recorded")" fi -HARNESS="$(cat "$recorded")" if [ ! -x "$HARNESS" ]; then - echo "loopgate: recorded harness '$HARNESS' is gone. Re-run 'harness install'." >&2 - exit 1 + HARNESS="$(command -v harness 2>/dev/null || true)" + if [ ! -x "$HARNESS" ]; then + echo "loopgate: hooks are not installed. Run 'harness install' in this repo." >&2 + exit 1 + fi + + echo "loopgate: harness installed but record of path '$HARNESS' is gone. Re-adding file 'harness-path'." >&2 + printf '%s\n' "$HARNESS" > "$recorded" fi export PATH="$(dirname "$HARNESS")${PATH:+:$PATH}" diff --git a/.github/workflows/mutation.yml b/.github/workflows/mutation.yml index 3199a73..ae8ba4e 100644 --- a/.github/workflows/mutation.yml +++ b/.github/workflows/mutation.yml @@ -24,10 +24,10 @@ jobs: steps: - name: Checkout Code - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Install uv - uses: astral-sh/setup-uv@v6 + uses: astral-sh/setup-uv@v7 with: enable-cache: true @@ -38,7 +38,7 @@ jobs: run: uv run --no-sync harness gate # need tests for Mutmut to target - name: Run Mutmut - run: uv run --no-sync mutmut run || true + run: uv run --no-sync mutmut run - name: Export CI/CD Stats run: uv run --no-sync mutmut export-cicd-stats @@ -48,3 +48,6 @@ jobs: with: name: mutmut-json-report path: mutants/mutmut-cicd-stats.json + + - name: Check Mutation Score + run: uv run --no-sync python mutation/check_mutmut.py diff --git a/.gitignore b/.gitignore index 16e3312..1079994 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,6 @@ __pycache__/ docs/launch.md test-results/ **/node_modules/ -harness-path .env .env.* diff --git a/README.md b/README.md index 60cc24a..3745489 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,8 @@ Blue infinity loop

L∞pGate

-

Run coding agents strictly andonly accept changes that pass your quality gates.

-

A coding-agent loop harness for Claude, Codex, Copilot, or any CLI agent. A dumb Ralph loop runner tells an agent to "Go!" and hands it a PROMPT. Agents can edit. Gates decide what lands. You set the plan in motion. The loops eat the prompt, and each agent iteration must update specs and commit through guardrails.

+

Run coding agents strictly and only accept changes that pass your quality gates.

+

A loop harness for Claude, Codex, Copilot, or any CLI agent. A loop runner hands each agent a prompt. Agents can edit. Gates decide what lands. You set the plan. Each agent iteration must update specs and commit through quality guardrails.

![Python](https://img.shields.io/badge/Python-3.11+-3776AB?logo=python&logoColor=white) ![Status](https://img.shields.io/badge/github-repo-blue?logo=github) @@ -17,6 +17,7 @@ [![](https://img.shields.io/badge/created%20an%20AGI%20by%20mistake-no-3C1)](https://github.com/sebmestrallet/absurd-badges) ![Claude](https://img.shields.io/badge/Claude-D97757?style=for-the-badge&logo=claude&logoColor=white) [![gate](https://github.com/rxdt/loopgate_harness/actions/workflows/ci.yml/badge.svg)](https://github.com/rxdt/loopgate_harness/actions/workflows/ci.yml) +[![mutation](https://img.shields.io/endpoint?url=https%3A%2F%2Fraw.githubusercontent.com%2Frxdt%2Floopgate_harness%2Fmutmut-diffsize%2Fmutation-score.json)](https://github.com/rxdt/loopgate_harness/actions/workflows/mutation.yml) @@ -24,27 +25,33 @@ ## TL;DR -1. `gh repo create / --template rxdt/loopgate_harness --private --clone && cd && uv run harness install && source .venv/bin/activate` +1. `gh repo create / --template rxdt/loopgate_harness --private --clone && cd && uv run harness install && source .venv/bin/activate && git add . && git commit --amend --no-edit` 2. `harness run codex` -**Requirements**: `pip`, `uv`, or `poetry`. Python 3.11. +**Requirements**: `pip`, `uv`, or `poetry`. Python 3.11. Linux or MacOS (Windows is experimental.) --- ## Features +Each run starts fresh, has clear limits, saves its logs, protects key files, and must pass checks you choose. + - **Quality-first**: Fight the AI slop with standards and style πŸ’― - **Worker-agnostic**: Claude, Codex, Copilot, Agy, or any prompt-reading CLI - **No lazy**: Agents work, _only if they pass the quality gates you set_ βœ… - **Repo-as-memory workflow**: specs/status/prompt are durable but code is king, leaving you free 😎 -- **Built-in stack**: Ruff, Pyright, Pylint, Semgrep, Complexipy, Hypothesis, Mutmut, 100% coverage β˜‘β˜‘β˜‘ +- **Built-in stack**: Linting, Format Check, Type-Checks, Dependancy Audit, Property-testing, Mutation-testing, 100% test coverage, Semgrep Security β˜‘β˜‘β˜‘ - **Progressive**: Preflight vs full gate split πŸ†— -- **Forbidden-path containment**: Don't touch that!-configurable πŸ›‘ +- **Forbidden-file containment**: Don't touch that!-configurable πŸ›‘ - **Installable project template**: `harness install ` gets the repo ready ▢️ - **No-rot**: Fresh-context agent iterations to reduce context rot πŸ”„ -- **Simple**: One command setup gets you git hooks and everything else -- **No-waste**: Timeouts and time-limits for all loops ⏸ -- **Agent containment prioritized**: Stop the madness (and [Semgrep](https://semgrep.dev/) for safety) πŸ”“ +- **Simple**: One-command setup gets you going +- **Hooks and CI ready-to-go:** Pre-commit, pre-push, and commit-message hooks + [`CI`](.github/workflows/ci.yml) are defined and already work with the checks +- **No-waste**: Timeouts and time-limits for all loops ⏰ +- **Diff size guardrails**: Agent changes warn at 300 lines and block at 400 πŸ“– +- **No empty work**: Agents blocked from empty commits +- **Agent containment prioritized**: Stop the madness +- **Industry-grade Security** Enabled with [Semgrep](https://semgrep.dev/) πŸ”“ --- @@ -53,17 +60,19 @@ > [!IMPORTANT] > Default configurations In [`pyproject.toml`](pyproject.toml) Update tool settings, add agent calls, remove or include checks... or leave as is. -`docs/PROMPT.md` tells each agent to pick a `spec` and build. `docs/specs/` say _what_ to build. The agent decides _what next_. You keep `docs/plan.md` current, and specs get rewritten from it (agent is told in `docs/PROMPT.md` to update the specs). Each iteration the agent updates its spec and `PROJECT_STATUS`. Ideas from [ghuntley](https://github.com/ghuntley), How to Ralph Wiggum. +[docs/plan.md](`docs/plan.md`) is where you define what you want the end product to be. You must be _very_ clear on what the finished product should and should **not** contain. Do **not** let agents guess. + +`docs/PROMPT.md` tells each agent to pick a `spec` and build. `docs/specs/` say _what_ to build. The agent decides _what next_. You keep `docs/plan.md` current, and specs get rewritten from it (agent is told in `docs/PROMPT.md` to update the specs). Each iteration the agent updates its spec and `PROJECT_STATUS`. > [!TIP] -> If you don't like _ANYTHING_ in this framework, update it. +> If you don't like _ANYTHING_ in this framework, [update it](#expanding-your-harness). ### Start a project -1. `gh repo create my-app-name --template /loopgate_harness --private --clone` **or** +1. `gh repo create / --template rxdt/loopgate_harness --private --clone` **or** ['Use This Template'](https://github.com/new?template_name=loopgate_harness&template_owner=rxdt) 2. Source your environment (if applicable) -3. From the root, run `harness install ` to name the project, install dependencies, set up the git hooks, and delete excess files. Install dependencies e.g. `uv sync && source .venv/bin/activate && harness install ` to name the project, install dependencies, set up the git hooks, and delete excess files. Install dependencies e.g. `uv sync && source .venv/bin/activate && harness install `. 4. `git commit` (the `install` command updates the repo) 5. Write your grand vision in [docs/plan.md](docs/plan.md) 6. Optionally add the first spec in `docs/specs/` (or leave it to the agents to draft the first specs based on your `plan.md`) @@ -73,11 +82,6 @@ 9. Not what you wanted? Refine [`docs/plan.md`](docs/plan.md) / [`docs/PROMPT.md`](docs/PROMPT.md) and re-run 10. Strict Ruff rules, type-checking Pyright, Complexipy, and Pytest coverage are set in [`pyproject.toml`](pyproject.toml). 11. Your coding quirks go in [`preferences/preferences.py`](preferences/preferences.py). -12. Loop!: - -```sh -harness run [max_iterations] [max_minutes] # agent: claude/codex/agy/copilot. ralph loop runner injects prompt -``` ### Works with `uv`, `poetry`, or `pip` @@ -85,11 +89,13 @@ harness run [max_iterations] [max_minutes] # agent: claude/codex/agy/co uv sync source .venv/bin/activate harness install +git add . && git commit harness gate harness run poetry install poetry run harness install +git add . && git commit poetry run harness gate poetry run harness run @@ -97,6 +103,7 @@ python -m venv .venv source .venv/bin/activate python -m pip install -r requirements.txt -e . harness install +git add . && git commit harness gate harness run ``` @@ -115,19 +122,39 @@ The repo is the only memory. Each iteration is a fresh-context agent. - every git push runs the full gate: lint, types, semgrep, tests, 100% coverage - the loop stops at `max_iterations`, a nonzero worker exit, or a timeout - Unspecified iterations/minutes β†’ default to 2 iterations Γ— 20 minutes each +- Each run streams agent 'thought' output live and is saved in a local scratchpad log - **The harness is worker-agnostic.** Any agent CLI that reads a prompt from stdin and can edit/commit works. ![L∞PS Agents](.loops_agents.svg) +## Safety + +`harness run` launches an autonomous LLM worker with the configured permissions, e.g. +`--permission-mode acceptEdits` or `--sandbox danger-full-access`. + +The gate bounds what any **commit** may touch, but the worker itself is **not** sandboxed to this repo unless you set that config. Consider the balance: without access it cannot do much. With machine access it can wreak havoc. Under a permissive mode it can run arbitrary shell. You are authorizing real changes. Choose the worker and permission mode deliberately. + +#### The Gate: Tiered Checks + +⚑ `harness preflight` (pre-commit) β†’ fast checks. +Ruff lint + check format for everyone, _plus_ **containment** for the agents. Self-heals by un-staging forbidden files. + +βœ… `harness gate` (CI/PR pre-push). Local checks mirror CI β†’ ruff lint + format report-only, pyright, pylint, semgrep, complexipy, hypothesis, pytest @ 100% cov. + +Only humans can bypass triggered gates and commit by adding flag `--no-verify`. + +
+ + ## Directory Layout ``` harness/ the gate, loop runner, CLI (πŸ€– forbidden directory) + tests/ the harness's own tests gate.py mirror the CI locally + preferences.py honored cli.py command-line entry point - tests/ the harness's own tests js-scaffold javascript example to build upon preferences/ user-defined preferences not covered by tools (πŸ€– forbidden directory) tests/ @@ -135,11 +162,8 @@ tests/ .githooks/ pre-commit / pre-push gate hooks (πŸ€– forbidden directory) .github/ CI that re-runs the gate (πŸ€– forbidden directory) pyproject.toml project + tooling config (πŸ€– forbidden) -AGENTS.md rules for agents working in the repo (πŸ€– forbidden) -docs/PROMPT.md the standing per-iteration instruction (human maintained) -docs/ PLAN, PROJECT_STATUS, PROMPT (human or agent maintained plan.md) +docs/ PROMPT, specs/, your plan (agent and human maintained) scratchpad/ scratch dir agents can use for temp files (For the πŸ€– to play) -docs/specs/ WHAT to build, one PRIORITY-bannered file per track (agent maintained) src/ your product/source code (add to coverage source) ``` @@ -149,26 +173,16 @@ If an agent edits a forbidden file, the file will be unstaged (not allowed to co
-A minimal `[tool.harness.gate]` snippet could look like: - -```toml -[tool.harness.forbidden] -dirs = ["harness/"] # agents may not commit changes here -iles = ["pyproject.toml"] -patterns = ["# noqa"] # banned in agent-authored diffs - -[tool.harness.gate] -pytest = "uv sync pytest" # one check command, run by the local gate AND CI -``` - ## Commands -Tool commands are defined once, in `[tool.harness.gate.checks]` in [pyproject.toml](pyproject.toml). The local gate and CI both derive them from there. +Tool commands are defined once, in `[tool.harness]` in [pyproject.toml](pyproject.toml). The local gate and CI both derive them from there. ```sh harness install # rewrite [project] name, uv sync, set core.hooksPath to .githooks harness preflight # fast checks: preferences, ruff lint + format (plus loop containment) -harness gate # full pass: preferences, ruff, format, pyright, pylint, complexipy, semgrep, pytest @ 100% cov, hypothesis +harness gate # full pass: preferences, ruff, format, pyright, pylint, complexipy, semgrep, pip-audit, pytest @ 100% cov, hypothesis +harness info # show configured agents, checks, and protected paths +harness status # count run logs and show the newest log RALPH_LOOP=1 harness gate # to run as if you are the agent in the loop harness run [max_iterations] [max_minutes] [verbose] # claude/codex/agy/copilot, defaults: 2 20 True @@ -187,7 +201,7 @@ To run LoopGate with Claude : harness run claude 2 20 ``` -Note: The worker must be installed and authenticated separately. **Claude Code exports env vars into every shell it spawns. `RALPH_LOOP=1` can be set globally.** +Note: The worker must be installed and authenticated separately.
@@ -195,7 +209,7 @@ Note: The worker must be installed and authenticated separately. **Claude Code e ## Expanding your harness - Edit rules at [pyproject.toml](pyproject.toml) for [ruff](https://docs.astral.sh/ruff/), [pylint](https://pypi.org/project/pylint/), [pydoclint](https://pypi.org/project/pydoclint/0.9.1/), [pyright](https://github.com/microsoft/pyright), [pytest](https://docs.pytest.org/en/stable/), [hypothesis](https://hypothesis.readthedocs.io/), [complexipy](https://github.com/rohaquinlop/complexipy), [mutmut](https://mutmut.readthedocs.io/) -- Add forbidden files, directories, or patterns in `[tool.harness.gate]` at [pyproject.toml](pyproject.toml) +- Add forbidden files, directories, or patterns in `[tool.harness]` at [pyproject.toml](pyproject.toml) - Add [Hypothesis](https://hypothesis.readthedocs.io/) tests in any test directory, examples at [test_properties.py](tests/preferences/test_properties.py). - Run [mutmut](https://mutmut.readthedocs.io/) by hand with `uv run mutmut run`, then `uv run mutmut browse`. A surviving mutant is a covered line no assertion checks. It is not a gate check: `mutmut run` exits 0 even with survivors and it should be run ~1x/week. Its mutants are cached in a JSON and should be used to identify weak tests. Example at [check_mutmut.py](https://github.com/rxdt/loopgate_harness/blob/main/mutation/check_mutmut.py). - [semgrep](https://docs.semgrep.dev/semgrep-ci/sample-ci-configs) has no repo config here. It uses registry configs / Semgrep's built-in defaults which ignore tests. @@ -239,15 +253,15 @@ You don’t have to. The loop runner, Ralph, and the CLI take a prompt, launch a The included [`harness/js-scaffold`](harness/js-scaffold/package.json) is a simple JavaScript **example** to expand on. Go to [pyproject.toml line 75](pyproject.toml#L75). Update checks. Put `js` into list `[tool.harness].languages`. Remove `py` if unused. -- **Why not just a shell loop?** - -A shell loop only reruns an agent. LoopGate ensures fresh context, durable repo state, time and iteration limits, protected paths, and quality gates that stop bad changes _before_ they land. - ``` npm run --prefix harness/js-scaffold gate npm run --prefix harness/js-scaffold preflight ``` +- **Why not just a shell loop?** + +A shell loop only reruns an agent. LoopGate ensures fresh context, durable repo state, time and iteration limits, protected paths, and quality gates that stop bad changes _before_ they land. +
@@ -256,7 +270,7 @@ npm run --prefix harness/js-scaffold preflight ## Coordination - Use `git log --oneline ..HEAD` to show what's unpushed. -- There is NO worktree/branch creation by design. You can create branches/trees and run a loop in each, then merge _(if you really feel like managing that)_ +- There is NO worktree/branch creation by design. You can create branches/trees and run a loop in each, then merge _(if you feel like managing that)_ - Agent duties can be contained to a part of the repo. e.g. Codex-1-frontend uses `docs/specs/frontend.md`, Claude-2-researcher `docs/specs/backend`... ### If you want to run a graph @@ -273,8 +287,7 @@ npm run --prefix harness/js-scaffold preflight ``` Other agents are working this repo. Before touching code, pick a spec whose claim line is - , replace it with your name, and commit that claim first. Own that spec's file and its - tests. Set the line back to on your last commit. + , replace it with your exact name `--/`, e.g. `claude-0003-backend-3/3`, and commit that claim first. Own that spec's file and its tests. Set the line back to on your last commit. ``` - What fails when agents do not claim specs/work: agents all pick the top-priority spec, duplicate work, and leave a half-staged git index. @@ -294,7 +307,7 @@ npm run --prefix harness/js-scaffold preflight - No branch/worktree creation in this harness was intentional: 1. For simplicity and maintainability of the framework. 2. Because a fresh iteration can't see the unmerged work in another worktree, so agents miss context and scramble to merge while conflicts pile up. - 3. Change this behavior if you're comfortable with granting agents machine access, feeding context to agents, and managing rapidly moving git history. + 3. Change this behavior as you like.
@@ -315,22 +328,3 @@ npm run --prefix harness/js-scaffold preflight 6. **100% coverage does not mean good tests.** That is quantity, not quality. Run `uv run mutmut run` to find covered lines that no assertion actually checks. 7. **Note**: `semgrep --config auto` needs network for semgrep registry rules. - -## Safety - -`harness run` launches an autonomous LLM worker with the configured permissions, e.g. -`--permission-mode acceptEdits` or `--sandbox danger-full-access`. - -The gate bounds what any **commit** may touch, but the worker itself is **not** sandboxed to this repo unless you set that config. Consider the balance: without access it cannot do much. With machine access it can wreak havoc. Under a permissive mode it can run arbitrary shell. You are authorizing real changes. Choose the worker and permission mode deliberately. - -#### The Gate: Tiered Checks - -⚑ `harness preflight` (pre-commit) β†’ fast checks. -Ruff lint + check format for everyone, _plus_ **containment** for the agents. Self-heals by un-staging forbidden files. - -βœ… `harness gate` (CI/PR pre-push). Local checks mirror CI β†’ ruff lint + format report-only, pyright, pylint, semgrep, complexipy, hypothesis, pytest @ 100% cov. - -Only humans can bypass triggered gates and commit by adding flag `--no-verify`. - -
- diff --git a/README.template.md b/README.template.md index 9d29f1c..931f976 100644 --- a/README.template.md +++ b/README.template.md @@ -2,7 +2,8 @@ Now that you have the template locally: -1. `uv sync` OR `poetry install` OR ` pip install -r requirements.txt`, then `harness install ` +1. `uv sync` OR `poetry install` OR ` pip install -r requirements.txt`, then `harness install +git add . && git commit` 2. Write your project goal in [docs/plan.md](docs/plan.md) 3. `harness run [max_iterations] [max_minutes]` 4. Not what you wanted? Refine [`docs/plan.md`](docs/plan.md) / [`docs/PROMPT.md`](docs/PROMPT.md) and re-run @@ -13,7 +14,9 @@ Now that you have the template locally: > [!IMPORTANT] > Default configurations In [`pyproject.toml`](pyproject.toml) Update tool settings, add agent calls, remove or include checks... or leave as is. -> If you don't like _ANYTHING_ in this framework, remove it. + +> [!TIP] +> If you don't like _ANYTHING_ in this framework, [update it](#expanding-your-harness). ### Start a project @@ -21,18 +24,20 @@ Now that you have the template locally: uv sync source .venv/bin/activate harness install +git add . && git commit harness gate harness run > poetry install poetry run harness install +git add . && git commit poetry run harness gate poetry run harness python -m venv .venv source .venv/bin/activate python -m pip install -r requirements.txt -e . -harness install +harness install && git add . && git commit harness gate harness run ``` @@ -157,12 +162,27 @@ The gate bounds what any **commit** may touch, but the worker itself is **not** ## Expanding your harness -- Edit rules at [pyproject.toml](pyproject.toml) for [ruff](https://docs.astral.sh/ruff/), [pylint](https://pypi.org/project/pylint/), [pydoclint](https://pypi.org/project/pydoclint/0.9.1/), [pyright](https://github.com/microsoft/pyright), [pytest](https://docs.pytest.org/en/stable/), [hypothesis](https://hypothesis.readthedocs.io/), [complexipy](https://github.com/rohaquinlop/complexipy) -- Add forbidden files, directories, or patterns in `[tool.harness.gate]` at [pyproject.toml](pyproject.toml) -- Add [Hypothesis](https://hypothesis.readthedocs.io/) tests when generated cases improve coverage beyond example-based tests. `tests/` -- [semgrep](https://docs.semgrep.dev/semgrep-ci/sample-ci-configs) has no repo config here. It uses registry configs plus Semgrep's built-in defaults which ignore tests. -- Edit `[tool.harness.gate.checks]` in [pyproject.toml](pyproject.toml). [ci.yml](.github/workflows/ci.yml) runs the same `harness gate`. -- Remove or add preferences not caught by Ruff, Pylint, etc. at [preferences.py](preferences/preferences.py). +- Edit rules at [pyproject.toml](pyproject.toml) for [ruff](https://docs.astral.sh/ruff/), [pylint](https://pypi.org/project/pylint/), [pydoclint](https://pypi.org/project/pydoclint/0.9.1/), [pyright](https://github.com/microsoft/pyright), [pytest](https://docs.pytest.org/en/stable/), [hypothesis](https://hypothesis.readthedocs.io/), [complexipy](https://github.com/rohaquinlop/complexipy), [mutmut](https://mutmut.readthedocs.io/) +- Add forbidden files, directories, or patterns in `[tool.harness]` at [pyproject.toml](pyproject.toml) +- Add [Hypothesis](https://hypothesis.readthedocs.io/) tests in any test directory, examples at [test_properties.py](tests/preferences/test_properties.py). +- Run [mutmut](https://mutmut.readthedocs.io/) by hand with `uv run mutmut run`, then `uv run mutmut browse`. A surviving mutant is a covered line no assertion checks. It is not a gate check: `mutmut run` exits 0 even with survivors and it should be run ~1x/week. Its mutants are cached in a JSON and should be used to identify weak tests. Example at [check_mutmut.py](https://github.com/rxdt/loopgate_harness/blob/main/mutation/check_mutmut.py). +- [semgrep](https://docs.semgrep.dev/semgrep-ci/sample-ci-configs) has no repo config here. It uses registry configs / Semgrep's built-in defaults which ignore tests. +- Update `[tool.harness.gate.checks]` in [pyproject.toml](pyproject.toml). [ci.yml](.github/workflows/ci.yml) runs those **same exact** `harness gate` checks. +- Add or remove coding preferences [preferences.py](preferences/preferences.py) that only agents in loops **must** respect. Current preferences: + +```py +function_argument_assignment_has_star # agents use non-specific `def fun(*)` +function_argument_assignment_underscore_lead # agents love over-using underscore names `def _fun()` +hidden_signature_star_args # Complain when a function uses *args or **kwargs (it hides function signatures) +dynamic_star_call # Calls to def fun(*items) breaks when you can't tell how many arguments f is getting +pointless_class # ensure classes are added for good reasons (carry state, values, methods) +lazy_assert # enforce real assertions, stronger tests +objects_injected_into_runtime_memory # finds calls that manipulate global state (dangerous, tricky) +lambda_found # abolish lambdas, make agents keep their code simple +lazy_any_type_hints # abolish type `Any` used to bypass strict type-checking +chaotic_continue_statements # abolish unecessary nested continue statements, clean code +complex_comprehension # no needlessly dense list/set/dict comprehensions, prefer linear code +```
@@ -209,11 +229,10 @@ npm run --prefix harness/js-scaffold preflight Spec claimed by agent: ``` -- **The agents:** paste this exact block into [PROMPT.md line 3](docs/PROMPT.md#L3): +- **This exact block** into [PROMPT.md line 3](docs/PROMPT.md#L3): ``` - Other agents are working this repo. Before touching code, pick a spec whose claim line is - , replace it with your name, and commit that claim first. Own that spec's file and its + Other agents are working this repo. Before touching code, pick a spec whose claim line is , replace it with your exact name `--/`, and commit that claim first. Own that spec's file and its tests. Set the line back to on your last commit. ``` diff --git a/docs/PROMPT.md b/docs/PROMPT.md index 1b1a718..612d876 100644 --- a/docs/PROMPT.md +++ b/docs/PROMPT.md @@ -1,36 +1,38 @@ -You are a fresh-context iteration in a loop. The repo is your memory. -Specs say what to build. You decide what is the next most useful change. +You are a fresh-context iteration in a loop. The repo `src/` and `docs/` are your memory. Specs say what to build. +You decide what is the next most useful change. 1. Read `docs/specs/*.md` and `docs/plan.md` and identify the most important unfinished items. -2. If a spec is wrong or missing, add or update the spec using `plan.md` instead of guessing. +2. If a spec is wrong or missing, add or update the spec using `plan.md` as a guide instead of guessing. 3. Inspect the relevant code and tests before editing. 4. Implement the scoped change that advances the specs. -5. Add or update tests that prove behavior and challenge the source; use durable, behavior-focused names and docstrings. -6. A milestone is not DONE until a test executes the entry point end-to-end and asserts observable output and exit code. Unit-testing an internal function is not sufficient. -7. Run `harness gate`. If `harness` is not on PATH, run `.venv/bin/harness gate`. -8. Fix failures without weakening tests, coverage, typing, security checks, or the gate. -9. Update the relevant spec and `docs/PROJECT_STATUS.md` to match what changed. -10. Commit on the current branch. -11. Push the current branch so the iteration is saved remotely. +5. If you are blocked, report it in `docs/PROJECT_STATUS.md` and exit: do not waste your turn and tokens pretending to work. +6. Verify existing 'blockers' before trusting them. Try to remove blockers. +7. Add or update tests that prove behavior and challenge the source; use durable, behavior-focused names and docstrings. +8. A milestone is not DONE until a test executes the entry point end-to-end and asserts observable output and exit code. Unit-testing an internal function is not sufficient. Prefer `hypothesis` property tests when possible. +9. Periodically run `mutmut run` and kill mutants. +10. Run `harness gate`. If `harness` is not on PATH, run `.venv/bin/harness gate`. +11. Update the relevant spec and `docs/PROJECT_STATUS.md` to match what changed. Keep `docs/PROJECT_STATUS.md` uncluttered: persist only actionable items. +12. Commit on the current branch. Rules: - Do not batch unrelated work. -- Keep history linear on the current branch: no branches, worktrees, merges, or rebases unless the human explicitly asked for one; commit only relevant current-branch work. +- Keep history linear on the current branch: no branches or worktrees unless the human explicitly asked for them. Commit only relevant current-branch work. - If forbidden paths block a commit, run `git restore --staged ` and leave those working-tree edits for human review. - Never delete tests or assertions to make checks pass. +- Fix failures without weakening tests, coverage, typing, security checks, or the gate. - Do not edit forbidden paths: `AGENTS.md`, `harness/`, `.githooks/`, `.github/`, `pyproject.toml`, `PROMPT.md`. -- Use tests for code behavior and API contracts. Do not test for `.md` contents. - -Commit message: +- Use tests for code output and contracts. Do not test for `.md` contents. + Commit message: ``` One sentence summary - concrete detail - concrete detail +... --- +-- # e.g. `codex-0006-frontend_ui-6/7` ``` Use the agent id the harness gave you (e.g. `0002-codex`); append the spec you worked and the diff --git a/harness/cli.py b/harness/cli.py index abdcf18..c38ab0f 100644 --- a/harness/cli.py +++ b/harness/cli.py @@ -20,27 +20,17 @@ from rich.json import JSON from rich.table import Table -from harness.gate import ( - AGENTS, - COMMIT_CHECKS, - FORBIDDEN, - REPO_ROOT, - gate_checks, - run_gate, - run_git, - run_preflight, -) -from harness.gate import prepare_commit_msg as commit_msg +from harness.gate import gates, run_git app = typer.Typer( name="loopgate", help="Commands to harness the loops", no_args_is_help=True, add_completion=False, - rich_markup_mode="rich", + rich_markup_mode=None if os.environ.get("RALPH_LOOP") else "rich", ) -console = Console(force_terminal=True) -REPO_ROOT_STR = str(REPO_ROOT) +console = Console(force_terminal=True, color_system=None if os.environ.get("RALPH_LOOP") else "256") +REPO_ROOT_STR = str(gates.repo_root) def setup_git_hooks(env_bin: Path, is_windows: bool) -> Path: @@ -58,20 +48,15 @@ def setup_git_hooks(env_bin: Path, is_windows: bool) -> Path: Returns: The path of the file that records the harness command. """ - rprint("\n[cyan2]Setting git hooks[/cyan2] with `git config core.hooksPath .githooks`:") - subprocess.run(("git", "config", "core.hooksPath", ".githooks"), cwd=REPO_ROOT_STR, check=True) + rprint("\n[cyan2]Setting git hooks[/cyan2] `git config core.hooksPath .githooks`") + subprocess.run(["git", "config", "core.hooksPath", ".githooks"], cwd=REPO_ROOT_STR, check=True) binary = env_bin / ("harness.exe" if is_windows else "harness") recorded = ( Path(run_git(["rev-parse", "--path-format=absolute", "--git-common-dir"]).strip()) / "harness-path" ) recorded.write_text(f"{binary.as_posix()}\n", encoding="utf-8", newline="\n") - typer.echo( - subprocess.run( - ("git", "config", "core.hooksPath"), cwd=REPO_ROOT_STR, capture_output=True, text=True, check=True - ).stdout.strip() - ) if is_windows: - rprint("Windows is experimental. Reoprt issues at https://github.com/rxdt/loopgate_harness/issues") + rprint("Windows is experimental. Reoprt issues https://github.com/rxdt/loopgate_harness/issues") else: subprocess.run(("ls", "-l", ".githooks"), cwd=REPO_ROOT_STR, check=True) return recorded @@ -138,13 +123,13 @@ def check(name: str, command: Callable[[], dict[str, list[str]]]) -> dict[str, l @app.command(help="Fast pre-commit checks (lint/format) plus agent containment") def preflight() -> None: """Dumb pass-through to the fast pre-commit gate.""" - check("preflight", run_preflight) + check("preflight", gates.run_preflight) @app.command(help="Pre-push checks match the CI gate exactly (lint, types, security, etc.)") def gate() -> None: """Dumb pass-through to the full pre-push gate; exit nonzero if anything fails.""" - check("gate", run_gate) + check("gate", gates.run_gate) @app.command(hidden=True, help="Git prepare-commit-msg hook. Called by .githooks, not by people.") @@ -159,7 +144,7 @@ def prepare_commit_msg( Raises: typer.Exit: the hook's status; git aborts the commit on 1. """ - raise typer.Exit(code=commit_msg(["prepare-commit-msg", *(args or [])])) + raise typer.Exit(code=gates.prepare_commit_msg(["prepare-commit-msg", *(args or [])])) @app.command(help="Show harness configuration and capabilitie in pyproject.toml") @@ -171,10 +156,10 @@ def info() -> None: padding=(0, 2), ) phases = ( - ("agents", AGENTS), - ("preflight", COMMIT_CHECKS), - ("gate", gate_checks), - ("forbidden", FORBIDDEN), + ("agents", gates.agents), + ("preflight", gates.commit_checks), + ("gate", gates.full_checks), + ("forbidden", gates.forbidden), ) for title, checks in phases: table.add_row(f"[bold cyan]{title}[/]", "") @@ -186,7 +171,7 @@ def info() -> None: @app.command(help="Count agent run logs under scratchpad/runs") def status() -> None: """Count run logs and point at the newest one.""" - runs = REPO_ROOT / "scratchpad" / "runs" + runs = gates.repo_root / "scratchpad" / "runs" logs = sorted(runs.glob("*.jsonl")) if runs.is_dir() else [] typer.secho(f"{len(logs)} run log(s) in {runs}", fg=typer.colors.CYAN, bold=True) if logs: @@ -213,6 +198,7 @@ def cleanup(cwd: Path, name: str | None) -> bool: ".loops.svg", ".github/workflows/publish.yml", "CONTRIBUTING.md", + "LICENSE.md", ): (cwd / file_name).unlink(missing_ok=True) for directory in (cwd / "dist", cwd / "harness" / "tests"): @@ -226,19 +212,22 @@ def cleanup(cwd: Path, name: str | None) -> bool: "name": canonicalize_name(name) if name and is_normalized_name(name) else "my-app-name", "version": "0.0.0", }) + paths = ["src", "preferences", "mutation"] tool = document.setdefault("tool", tomlkit.table()) - tool.setdefault("pyright", tomlkit.table()).update({"include": ["src", "preferences"]}) + tool.setdefault("pyright", tomlkit.table()).update({"include": paths}) + tool.setdefault("mutmut", tomlkit.table()).update({"source_paths": paths}) tool.setdefault("pytest", tomlkit.table()).setdefault("ini_options", tomlkit.table()).update({ "testpaths": ["tests"], "pythonpath": [".", "src"], }) coverage = tool.setdefault("coverage", tomlkit.table()) - coverage.setdefault("run", tomlkit.table()).update({"source": ["src", "preferences"]}) - tool.setdefault("complexipy", tomlkit.table()).update({"paths": ["src", "preferences"]}) + coverage.setdefault("run", tomlkit.table()).update({"source": paths}) + tool.setdefault("complexipy", tomlkit.table()).update({"paths": paths}) tool.setdefault("ruff", tomlkit.table()).setdefault("exclude", tomlkit.array()).append("harness") tool.setdefault("pylint", tomlkit.table()).setdefault("main", tomlkit.table()).setdefault( "ignore", tomlkit.array() ).append("harness") + rprint(f"\n[cyan2]project name[/cyan2] '{project['name']}' set in `pyproject.toml`") (cwd / "pyproject.toml").write_text(tomlkit.dumps(document), encoding="utf-8") return True @@ -259,10 +248,10 @@ def install(name: Annotated[str | None, typer.Argument(help="Set up project for rprint("\n[cyan2]installing dependencies[/cyan2]") is_windows = sys.platform == "win32" # Record the env the manager just filled that holds the harness executable - if (REPO_ROOT / "uv.lock").is_file(): + if (gates.repo_root / "uv.lock").is_file(): subprocess.run(("uv", "sync"), cwd=REPO_ROOT_STR, check=True) - env_bin = REPO_ROOT / ".venv" / ("Scripts" if is_windows else "bin") - elif (REPO_ROOT / "poetry.lock").is_file(): + env_bin = gates.repo_root / ".venv" / ("Scripts" if is_windows else "bin") + elif (gates.repo_root / "poetry.lock").is_file(): subprocess.run(("poetry", "install"), cwd=REPO_ROOT_STR, check=True) poetry_env = subprocess.run( ("poetry", "env", "info", "--executable"), capture_output=True, text=True, check=True @@ -276,7 +265,7 @@ def install(name: Annotated[str | None, typer.Argument(help="Set up project for check=True, ) env_bin = Path(sys.executable).parent - cleanup(REPO_ROOT, name) + cleanup(gates.repo_root, name) recorded = setup_git_hooks(env_bin, is_windows) if not is_windows: check_for_timeout_and_prompt(env_bin) @@ -308,7 +297,7 @@ def check_for_timeout_and_prompt(env_bin: Path) -> None: @app.command( help="Run one harnessed ralph loop with , e.g. harness run claude 3 20.\n\n" - f"Integrated agents (from tool.harness.agents): {', '.join(AGENTS)}" + f"Integrated agents (from tool.harness.agents): {', '.join(gates.agents)}" ) def run( agent: str, @@ -330,7 +319,7 @@ def run( typer.Exit: code 2 for an unknown agent or non-positive counts, else the worker's exit code. """ agent = agent.casefold() - if agent not in AGENTS: + if agent not in gates.agents: typer.secho(f"Unknown agent name '{agent}'", err=True, fg=typer.colors.MAGENTA, bold=True) raise typer.Exit(code=2) if num_iterations < 1 or max_minutes < 1: @@ -344,7 +333,7 @@ def run( worker_id = f"{max((int(p.stem) for p in runs.glob('[0-9][0-9][0-9][0-9].jsonl')), default=0) + 1:04d}" # Hand the agent a fixed identity to use in claims and commits prompt = (cwd / "docs" / "PROMPT.md").read_text(encoding="utf-8").rstrip("\n") - os.environ["RALPH_PROMPT"] = f"Your agent id is `{worker_id}`\n\n{prompt}" + os.environ["RALPH_PROMPT"] = f"Your agent id prefix is `{agent}-{worker_id}`\n\n{prompt}" log = runs / f"{worker_id}.jsonl" # each log file is one run / ralph invocation, not one iteration loop_dir = Path(__file__).resolve().parent # Windows has no POSIX shell/timeout so run PowerShell ralph.ps1 twin @@ -353,7 +342,7 @@ def run( if sys.platform == "win32" # support windows else [str(loop_dir / "ralph.sh")] ) - agent_argv = [tok.replace("{log_path}", str(log)) for tok in AGENTS[agent]] + agent_argv = [tok.replace("{log_path}", str(log)) for tok in gates.agents[agent]] if model: agent_argv[agent_argv.index("--model") + 1] = model command = [*launcher, str(num_iterations), str(max_minutes), *agent_argv] diff --git a/harness/gate.py b/harness/gate.py index bdcaef4..2d564b2 100644 --- a/harness/gate.py +++ b/harness/gate.py @@ -19,13 +19,217 @@ import typer from rich.console import Console -console = Console(force_terminal=True) +console = Console(force_terminal=True, color_system=None if os.environ.get("RALPH_LOOP") else "256") + try: from preferences.preferences import preferences_violations as prefs -except ImportError: # humans do what they want with preferences.py +except ImportError: # humans can delete preferences.py prefs = None -EMPTY_TREE = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" # universal empty tree hash + +class Gate: + """Contains Gate configuration values and methods.""" + + EMPTY_TREE = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" # universal empty tree hash + + def __init__(self, root: Path) -> None: + self.repo_root = root + toml = tomllib.loads((self.repo_root / "pyproject.toml").read_bytes().decode()).get("tool", {}) + harness = toml["harness"] + self.forbidden: dict[str, list[str]] = harness.get("FORBIDDEN", {}) + self.languages: tuple[str, ...] = harness.get("languages", {}) + self.agents: dict[str, list[str]] = harness.get("agents", {}) + self.commit_checks: dict[str, list[str]] = harness.get("preflight", {}) + self.full_checks: dict[str, list[str]] = self.commit_checks | harness.get("gate", {}) + self.forbidden_files: tuple[str, ...] = tuple(self.forbidden.get("FILES", [])) + self.forbidden_dirs: tuple[str, ...] = tuple(self.forbidden.get("DIRS", [])) + self.forbidden_patterns: tuple[str, ...] = tuple(self.forbidden.get("PATTERNS", [])) + self.error_diff_lines: int = harness.get("error_diff_lines") + + def run_checks(self, checks: dict[str, list[str]]) -> dict[str, list[str]]: + """Run each named command, streaming its output live under a phase header. + Reports what each command did and leaves the verdict to the caller. + + Args: + checks: Mapping of check name to the argv that runs it. + + Returns: + { "pass": [...], "warn": [...], "fail": [ problems ] } bucketing each check name by exit code. + If anything is in "fail", a commit is not allowed. + """ + clean_env = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")} + if not os.environ.get("RALPH_LOOP"): + clean_env.update({"FORCE_COLOR": "1", "CLICOLOR_FORCE": "1", "SEMGREP_FORCE_COLOR": "1"}) + results: dict[str, list[str]] = {"pass": [], "fail": [], "warn": []} + for name, command in checks.items(): + colorize(name, " ".join(command)) + sys.stdout.flush() + with subprocess.Popen(command, cwd=self.repo_root, env=clean_env) as process: + exit_code = process.wait() + if exit_code == 0: + results["pass"].append(name) + elif "format" in name: + results["warn"].append(name) + else: + results["fail"].append(name) + if os.environ.get("RALPH_LOOP"): + self._run_non_human_checks(results) + + return results + + def _run_non_human_checks(self, results: dict[str, list[str]]): + """Runs checks on non-humans only. Checks things that linters or other chekcs to do not check. + Unstages files that should never be touched. + + Arguments: + results: The original bucketing of each check name into "pass"/"fail"/"warn" lists. + """ + colorize("AGENT CHECKs", "running non-human agent checks") + ref = "HEAD" if run_git(["rev-parse", "--verify", "HEAD"], check=False).strip() else self.EMPTY_TREE + stats = run_git(["diff", ref, "--numstat", "--find-renames"]).splitlines() + self._check_diff_size(stats, results) + staged = run_git([ + "diff", + "--cached", + "--name-only", + "--no-renames", + "--diff-filter=ACMRD", + ]).splitlines() + if not staged: + colorize("EMPTY COMMIT", "nothing staged: do real work, do not commit empty") + else: + forbidden: list[str] = [ + path + for path in staged + if path.casefold() in self.forbidden_files or path.casefold().startswith(self.forbidden_dirs) + ] + if forbidden: + run_git(["reset", "-q", "HEAD", "--", *forbidden]) + colorize("EJECTED", f"kept forbidden paths out of the commit: {forbidden}") + results["fail"].extend(self._check_for_bad_patterns()) + results["fail"].extend(filter(None, self._check_for_preferences())) + + def _check_for_bad_patterns(self) -> list[str]: + """Check staged files for banned patterns (agent-in-loop containment). + Does not unstage anything. Later, if any problem lands in { "fail": ... } the commit is blocked. + + Banned patterns are flagged only on ADDED diff lines (a '+' line, never a '+++' header). + + Returns: + The banned-pattern hits plus any preference violations found in the staged files. + """ + colorize("BANNED PATTERNS CHECK", "checking for banned patterns in staged files") + diff_args = ["diff", "--cached", "--unified=0", "--output-indicator-new=a", "--"] + staged_lines = run_git(diff_args).splitlines() + problems: list[str] = [] + for line in staged_lines: + if line.startswith("a"): + for pattern in self.forbidden_patterns: + pattern_and_bare_line = f"'{pattern}' line: {line[1:].strip()}" + if pattern.casefold() in line.casefold(): + problems.append(pattern_and_bare_line) + return problems + + def _check_diff_size(self, stats: list[str], results: dict[str, list[str]]): + """Report size of pending diff and block a bloated commit if past Lines Of Code (LOC) review cap. + + LOC = added + deleted. Count diff lines, staged and unstaged. An edit is + one deletion plus one addition. Docs, lockfiles and binaries are excluded. + + Arguments: + stats: Git numstat rows to parse to get total Lines Of Code + results: The full-checks result bucketing each check name into "pass"/"fail"/"warn" lists. + """ + warn_at_75: int = round(self.error_diff_lines * 0.75) + total = 0 + for line in stats: + inserted, deleted, path = line.split("\t", 2) + if not (inserted == "-" or path.endswith(".lock")): # binary or lockfile + total += int(inserted) + int(deleted) + if total < warn_at_75: + return + msg = ( + f"{total} lines modified. WARN at 75% {warn_at_75} lines, ERROR at {self.error_diff_lines}." + "\nSuggestion: Refactor bloat, inline helpers, reduce mis-direction, re-use fixtures, cut " + "duplication, slim down if-elif-else blocks." + ) + colorize("DIFF SIZE", msg) + if total > self.error_diff_lines: + results["fail"].append(msg) + elif total > warn_at_75: + results["warn"].append(msg) + + def _check_for_preferences(self) -> list[str]: + """Checks user preferences honored. Currently only a preferences.py file exists. New languages should + add their own. + + Returns: + The banned-pattern hits plus any preference violations found in the staged files. + """ + colorize("USER PREFERENCES", "checking that user's preferences are respected") + if "py" in self.languages: + staged = run_git([ + "diff", + "--cached", + "--name-only", + "--diff-filter=d", + "--", + "*.py", + ]).splitlines() + if staged and prefs: + return [prefs(path, run_git(["show", f":{path}"])) for path in staged] + return [] + + def run_preflight(self) -> dict[str, list[str]]: + """Pre-commit: lint (blocking) plus an informational format report. For agents in the loop also + unstages forbidden filepaths and flags banned patterns + human-preferences not honored. + + Returns: + The commit-checks result with any containment problems appended to "fail" list. + """ + return self.run_checks(self.commit_checks) + + def run_gate(self) -> dict[str, list[str]]: + """Pre-push / CI: lint, types, pylint, security, pytest/hypothesis (blocking), complexipy, plus an + informational format report. + + Returns: + results: The full-checks result bucketing each check name into "pass"/"fail"/"warn" lists. + """ + return self.run_checks(self.full_checks) + + def prepare_commit_msg(self, argv: list[str]) -> int: + """Logic for the git prepare-commit-msg hook applicable to agents in the loop. + + Args: + argv: arguments used to invoke `git commit` + + Returns: + Status code integer 0 or 1 (git blocks commit on code 1) + """ + if not os.environ.get("RALPH_LOOP"): + return 0 + commit_msg_file: str = argv[1] if len(argv) > 1 else "" + command = argv[2] if len(argv) > 2 else "" + msg = "" + if command in {"merge", "squash", "rebase", "reset", "clean", "filter-branch"}: + msg = f"You cannot use that git command `{command}`.\n" + ref = "HEAD" if run_git(["rev-parse", "--verify", "HEAD"], check=False).strip() else self.EMPTY_TREE + if not run_git(["diff-index", "--cached", "--name-only", ref]): + msg += ( + "Empty commit detected. Stage real work, Don't use --allow-empty. Or say if you're blocked\n" + ) + if Path(commit_msg_file).exists(): + content = Path(commit_msg_file).read_text(encoding="utf-8") + actual_text = "\n".join([ + line for line in content.splitlines() if not line.startswith("#") + ]).strip() + if not actual_text: + msg += "Commit message is blank. Provide an informative message with your agent ID.\n" + if msg: + colorize("PRE COMMIT MESSAGE", msg) + return 1 # Intercepts git + return 0 def run_git(args: list[str], repo: Path | None = None, check: bool = True) -> str: @@ -33,37 +237,19 @@ def run_git(args: list[str], repo: Path | None = None, check: bool = True) -> st Arguments: args: Git subcommand and its arguments - repo: the repository directory to run the git command from; defaults to REPO_ROOT - check: If check is True and the exit code was non-zero, it raises a CalledProcessError which has - returncode attribute, and output attribute + repo: the repository directory to run the git command from. Defaults to REPO_ROOT. + check: If check is True and the exit code was non-zero, it raises a CalledProcessError. Returns: The command's raw stdout string (callers will .splitlines() as needed) """ - target = REPO_ROOT if repo is None else repo + target = gates.repo_root if repo is None else repo command = ["git", "-C", str(target), *args] git_env = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")} result = subprocess.run(command, capture_output=True, text=True, check=check, env=git_env) return result.stdout -# GATE AND PREFLIGHT RUN FROM PROJECT LEVEL DIRECTORY -REPO_ROOT = Path(run_git(["rev-parse", "--show-toplevel"], repo=Path.cwd()).strip()).resolve() - - -raw_toml = tomllib.loads((REPO_ROOT / "pyproject.toml").read_bytes().decode()) -HARNESS = raw_toml.get("tool", {}).get("harness", {}) -languages = HARNESS.get("languages", {}) -gate_checks = HARNESS.get("gate", {}) -AGENTS = HARNESS.get("agents", {}) -COMMIT_CHECKS = HARNESS.get("preflight", {}) -FULL_CHECKS = COMMIT_CHECKS | gate_checks -FORBIDDEN = HARNESS.get("FORBIDDEN", {}) -FORBIDDEN_FILES = FORBIDDEN.get("FILES", []) -FORBIDDEN_DIRS = tuple(FORBIDDEN.get("DIRS", [])) -FORBIDDEN_PATTERNS = FORBIDDEN.get("PATTERNS", []) - - def colorize(name: str, command: str) -> None: """Rich consosle printing to signpost checks. @@ -78,145 +264,4 @@ def colorize(name: str, command: str) -> None: console.print(f"[dim italic]{command}[/dim italic]\n", justify="center") -def run_checks(checks: dict[str, list[str]]) -> dict[str, list[str]]: - """Run each named command, streaming its output live under a phase header. - Reports what each command did and leaves the verdict to the caller. - - Args: - checks: Mapping of check name to the argv that runs it. - - Returns: - { "pass": [...], "warn": [...], "fail": [ problems ] } bucketing each check name by exit code. - If anything is in "fail", a commit is not allowed. - """ - clean_env = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")} - if not os.environ.get("RALPH_LOOP"): - clean_env.update({"FORCE_COLOR": "1", "CLICOLOR_FORCE": "1", "SEMGREP_FORCE_COLOR": "1"}) - results: dict[str, list[str]] = {"pass": [], "fail": [], "warn": []} - for name, command in checks.items(): - colorize(name, " ".join(command)) - sys.stdout.flush() - with subprocess.Popen(command, cwd=REPO_ROOT, env=clean_env) as process: - exit_code = process.wait() - if exit_code == 0: - results["pass"].append(name) - elif "format" in name: - results["warn"].append(name) - else: - results["fail"].append(name) - if os.environ.get("RALPH_LOOP"): - results["fail"].extend(run_non_human_checks()) - - return results - - -def run_non_human_checks() -> list[str]: - """Runs checks on non-humans only. Checks things that linters or other chekcs to do not check. - Unstages files that should never be touched. - - Returns: - list of problems not caught by lint, type-checking, testing - """ - problems: list[str] = [] - staged = run_git(["diff", "--cached", "--name-only", "--no-renames", "--diff-filter=ACMRD"]).splitlines() - if not staged: - colorize("EMPTY COMMIT", "nothing staged: do real work, do not commit empty") - return problems # yell, but don't block - problems.extend(check_for_bad_patterns()) - forbidden: list[str] = [ - path - for path in staged - if path.casefold() in FORBIDDEN_FILES or path.casefold().startswith(FORBIDDEN_DIRS) - ] - if forbidden: - run_git(["reset", "-q", "HEAD", "--", *forbidden]) - colorize("EJECTED", f"kept forbidden paths out of the commit: {', '.join(forbidden)}") - return ["problems:\n" + "\n".join(problems)] if problems else [] - - -def check_for_bad_patterns() -> list[str]: - """Check staged files for banned patterns and user-preference breaks (agent-in-loop containment). - Does not unstage anything. Later, if any problem lands in { "fail": ... } the commit is blocked. - - Banned patterns are flagged only on ADDED diff lines (a '+' line, never a '+++' header). - - Returns: - The banned-pattern hits plus any preference violations found in the staged files. - """ - colorize("BANNED PATTERNS CHECK", "checking for banned patterns in staged files") - diff_args = ["diff", "--cached", "--unified=0", "--output-indicator-new=a", "--", ".", ":(exclude)*.md"] - staged_lines = run_git(diff_args).splitlines() - problems: list[str] = [] - for line in staged_lines: - if line.startswith("a"): - for pattern in FORBIDDEN_PATTERNS: - pattern_and_bare_line = f"'{pattern}' line: {line[1:].strip()}" - if pattern.casefold() in line.casefold(): - problems.append(pattern_and_bare_line) - problems.extend(filter(None, check_for_preferences())) - return problems - - -def check_for_preferences() -> list[str]: - """Checks user preferences honored. Currently only a preferences.py file exists. New languages should add - their own. - - Returns: - The banned-pattern hits plus any preference violations found in the staged files. - """ - colorize("USER PREFERENCES", "checking that user's preferences are respected") - if "py" in languages: - staged = run_git(["diff", "--cached", "--name-only", "--diff-filter=d", "--", "*.py"]).splitlines() - if staged and prefs: - return [prefs(path, run_git(["show", f":{path}"])) for path in staged] - return [] - - -def run_preflight() -> dict[str, list[str]]: - """Pre-commit: lint (blocking) plus an informational format report. For agents in the loop also unstages - forbidden filepaths and flags banned patterns + human-preferences not honored. - - Returns: - The commit-checks result with any containment problems appended to "fail" list. - """ - return run_checks(COMMIT_CHECKS) - - -def run_gate() -> dict[str, list[str]]: - """Pre-push / CI: lint, types, pylint, security, pytest/hypothesis (blocking), complexipy, plus an - informational format report. - - Returns: - The full-checks result bucketing each check name into "pass"/"fail" lists. - """ - return run_checks(FULL_CHECKS) - - -def prepare_commit_msg(argv: list[str]) -> int: - """Logic for the git prepare-commit-msg hook applicable to agents in the loop. - - Args: - argv: arguments used to invoke `git commit` - - Returns: - Status code integer 0 or 1 (git blocks commit on code 1) - """ - if not os.environ.get("RALPH_LOOP"): - return 0 - commit_msg_file: str = argv[1] if len(argv) > 1 else "" - command = argv[2] if len(argv) > 2 else "" - msg = "" - ref = "HEAD" if run_git(["rev-parse", "--verify", "HEAD"], check=False).strip() else EMPTY_TREE - if command in {"merge", "squash", "rebase", "reset", "clean", "filter-branch"}: - msg = f"You cannot use that git command `{command}`.\n" - if not run_git(["diff-index", "--cached", "--name-only", f"{ref}"]): - msg += "Empty-tree commit detected. Stage real work and don't use --allow-empty. Lazy.\n" - if Path(commit_msg_file).exists(): - content = Path(commit_msg_file).read_text(encoding="utf-8") - actual_text = "\n".join([line for line in content.splitlines() if not line.startswith("#")]).strip() - if not actual_text: - msg += "Commit message is blank. Provide an informative message with your agent ID.\n" - if msg: - sys.stdout.write(f"\n[COMMIT BLOCKED]:\n{msg}\n") - return 1 # Intercepts git - return 0 +gates = Gate(Path(run_git(["rev-parse", "--show-toplevel"], repo=Path.cwd()).strip()).resolve()) diff --git a/harness/ralph.ps1 b/harness/ralph.ps1 index 617adf1..625e8ce 100644 --- a/harness/ralph.ps1 +++ b/harness/ralph.ps1 @@ -36,6 +36,11 @@ if ($maxIterations -lt 1 -or $maxMinutes -le 0) { for ($i = 1; $i -le $maxIterations; $i++) { [Console]::Error.WriteLine("ralph: iteration $i/$maxIterations") + # Receipt line, same stdout contract as ralph.sh: `harness run` saves stdout as the run's .jsonl. + # The worker inherits this handle, so flush before starting it or the records interleave. + $timestamp = (Get-Date).ToString("yyyy-MM-ddTHH:mm") + [Console]::Out.WriteLine("{""type"":""ralph"",""iteration"":$i,""max_iterations"":$maxIterations,""timestamp"":""$timestamp""}") + [Console]::Out.Flush() $stdin = "$($env:RALPH_PROMPT)`n`nRALPH_ITERATION=$i/$maxIterations`n" $psi = [System.Diagnostics.ProcessStartInfo]::new() $psi.FileName = $rest[0] @@ -60,4 +65,7 @@ for ($i = 1; $i -le $maxIterations; $i++) { } } +$timestamp = (Get-Date).ToString("yyyy-MM-ddTHH:mm") +[Console]::Out.WriteLine("{""type"":""ralph"",""completed"":$maxIterations, ""timestamp"":""$timestamp""}") +[Console]::Out.Flush() [Console]::Error.WriteLine("ralph: completed $maxIterations iteration(s)") diff --git a/harness/ralph.sh b/harness/ralph.sh index fc8b8f0..316bd35 100755 --- a/harness/ralph.sh +++ b/harness/ralph.sh @@ -51,10 +51,12 @@ fi i=1 while [ "$i" -le "$MAX_ITERATIONS" ]; do - echo "ralph: iteration $i/$MAX_ITERATIONS" >&2 + printf '{"type":"ralph","iteration":%s,"max_iterations":%s,"timestamp":"%s"}\n' \ + "$i" "$MAX_ITERATIONS" "$(date '+%Y-%m-%dT%H:%M')" + printf '%s\n\nRALPH_ITERATION=%s/%s\n' "$RALPH_PROMPT" "$i" "$MAX_ITERATIONS" \ | "$TIMEOUT" "$((MAX_MINUTES * 60))" "$@" i=$((i + 1)) done -echo "ralph: completed $MAX_ITERATIONS iteration(s)" >&2 +printf '{"type":"ralph","completed":%s, "timestamp":"%s"}\n' "$MAX_ITERATIONS" "$(date '+%Y-%m-%dT%H:%M')" diff --git a/harness/tests/conftest.py b/harness/tests/conftest.py index e1b454a..32c1b35 100644 --- a/harness/tests/conftest.py +++ b/harness/tests/conftest.py @@ -2,8 +2,10 @@ from __future__ import annotations +import shutil import subprocess import sys +from collections.abc import Iterator from pathlib import Path from subprocess import PIPE from typing import Self @@ -11,6 +13,7 @@ import pytest from harness import cli, gate +from harness.gate import gates REPO_ROOT = Path(__file__).resolve().parents[2] collect_ignore = ["test_ralph.py"] if sys.platform == "win32" else ["test_ralph_ps1.py"] @@ -39,7 +42,7 @@ def fake_popen( Git is never faked. run_git reaches Popen through subprocess.run, so a git command is handed straight to the real Popen and the real gate.run_git keeps working against the temp repo the - test points REPO_ROOT at. Only the checks around it are stand-ins. + test points gates.repo_root at. Only the checks around it are stand-ins. Every faked check reports exit 0 (pass) unless its exact argv is in fails, which reports exit 1. Every faked launch is recorded (command, cwd, env) so dispatch tests can assert what run_checks ran. @@ -76,6 +79,77 @@ def git_repo(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: (tmp_path / ".gitignore").write_text("existing\n", encoding="utf-8") gate.run_git(["add", ".gitignore", "README.md", "README.template.md"], tmp_path) gate.run_git(["commit", "-q", "-m", "seed"], tmp_path) - monkeypatch.setattr(cli, "REPO_ROOT", tmp_path) - monkeypatch.setattr(gate, "REPO_ROOT", tmp_path) + monkeypatch.setattr(cli, "REPO_ROOT_STR", str(tmp_path)) + monkeypatch.setattr(gates, "repo_root", tmp_path) return tmp_path + + +@pytest.fixture +def real_hook_repo(request: pytest.FixtureRequest, git_repo: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Wire selected tracked hooks to a recorded executable in the disposable repository.""" + hooks = git_repo / ".active-hooks" + hooks.mkdir() + for name in (*request.param, "_resolve"): + shutil.copy2(REPO_ROOT / ".githooks" / name, hooks / name) + gate.run_git(["config", "core.hooksPath", ".active-hooks"], git_repo) + + executable = git_repo / "recorded-harness" + executable.write_text( + f"""#!{Path(sys.executable).as_posix()} +import json +import os +import sys +from pathlib import Path + +repo = Path.cwd() +arguments = sys.argv[1:] +command = arguments[0] if arguments else "" +recorded = arguments.copy() +if command == "prepare-commit-msg" and len(recorded) > 1: + recorded[1] = Path(recorded[1]).name +with (repo / "harness.calls").open("a", encoding="utf-8") as handle: + handle.write(json.dumps({{"arguments": recorded, "RALPH_LOOP": os.environ.get("RALPH_LOOP")}}) + "\\n") +real_file = repo / "harness.real" +real_commands = real_file.read_text(encoding="utf-8").splitlines() if real_file.exists() else [] +if command == "prepare-commit-msg" or command in real_commands: + sys.path.insert(0, {str(REPO_ROOT)!r}) + os.chdir({str(REPO_ROOT)!r}) + from harness import cli + from harness.gate import gates + os.chdir(repo) + gates.repo_root = repo + cli.REPO_ROOT_STR = str(repo) + if command == "preflight": + gates.commit_checks = {{}} + cli.main(arguments) +status_file = repo / "harness.exit" +raise SystemExit(int(status_file.read_text(encoding="utf-8")) if status_file.exists() else 0) +""", + encoding="utf-8", + ) + executable.chmod(0o755) + (git_repo / ".git" / "harness-path").write_text(f"{executable}\n", encoding="utf-8") + monkeypatch.setenv("RALPH_LOOP", "1") + return git_repo + + +def seed_repo(directory: Path) -> Path: + """Create a temp git repository with one commit and point gate's git calls at it.""" + gate.run_git(["init", "-q"], directory) + gate.run_git(["config", "user.email", "harness@test.local"], directory) + gate.run_git(["config", "user.name", "harness-test"], directory) + (directory / "README.md").write_text("seed\n", encoding="utf-8") + gate.run_git(["add", "README.md"], directory) + gate.run_git(["commit", "-q", "-m", "seed"], directory) + return directory + + +@pytest.fixture(scope="module") +def scan_repo(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Path]: + """A temp repo shared by the generated examples, since @given cannot take a per-test fixture.""" + repo = seed_repo(tmp_path_factory.mktemp("banned-patterns")) + with pytest.MonkeyPatch.context() as patch: + patch.setenv("RALPH_LOOP", "1") + patch.setattr(gates, "repo_root", repo) + patch.setattr(gates, "commit_checks", {}) + yield repo diff --git a/harness/tests/test_cli.py b/harness/tests/test_cli.py index 9d6f7fd..22aa62e 100644 --- a/harness/tests/test_cli.py +++ b/harness/tests/test_cli.py @@ -5,8 +5,8 @@ from __future__ import annotations import io +import json import os -import shutil import subprocess import sys import tomllib @@ -21,7 +21,7 @@ from typer.testing import CliRunner from harness import cli, gate -from harness.gate import AGENTS, FORBIDDEN_DIRS, FORBIDDEN_FILES, FORBIDDEN_PATTERNS, FULL_CHECKS +from harness.gate import Gate, gates from harness.tests.conftest import REPO_ROOT, fake_popen if TYPE_CHECKING: @@ -111,7 +111,7 @@ def test_entry_point_propagates_exit_codes_and_rejects_unknown_commands( assert runner.invoke(cli.app, ["bogus"]).exit_code == 2 assert runner.invoke(cli.app, []).exit_code == 2 - fake_popen(monkeypatch, fails=[gate.COMMIT_CHECKS["lint"], gate.COMMIT_CHECKS["format"]]) + fake_popen(monkeypatch, fails=[gates.commit_checks["lint"], gates.commit_checks["format"]]) rejected = runner.invoke(cli.app, ["preflight"]) summary = " ".join(unstyle(rejected.stdout).split()) @@ -134,20 +134,20 @@ def test_help_and_info_surface_every_check_agent_and_containment_rule( flat = " ".join(unstyle(info.output).split()) for phase in ("preflight", "gate"): assert phase in flat - for name, command in FULL_CHECKS.items(): + for name, command in gates.full_checks.items(): assert name in flat assert command[0] in flat - for pattern in FORBIDDEN_PATTERNS: + for pattern in gates.forbidden_patterns: assert pattern in flat - for path in (*FORBIDDEN_DIRS, *FORBIDDEN_FILES): + for path in (*gates.forbidden_dirs, *gates.forbidden_files): assert path in flat - for agent in AGENTS: + for agent in gates.agents: assert agent in flat run_help = runner.invoke(cli.app, ["run", "--help"]) assert run_help.exit_code == 0 - for agent in AGENTS: + for agent in gates.agents: assert agent in run_help.output assert "verbose" in run_help.output assert "--verbose" not in run_help.output @@ -168,7 +168,7 @@ def test_every_supported_agent_has_a_nonempty_command() -> None: "tool" ]["harness"]["agents"] - assert agents == AGENTS + assert agents == gates.agents assert set(agents) == {"claude", "codex", "agy", "copilot"} assert all(isinstance(command, list) and bool(command) for command in agents.values()) assert all( @@ -176,60 +176,56 @@ def test_every_supported_agent_has_a_nonempty_command() -> None: ) -def test_preflight_summary_names_every_check_for_agents( - monkeypatch: pytest.MonkeyPatch, git_repo: Path +@pytest.mark.parametrize( + ("command", "source", "expected"), + [ + pytest.param( + "preflight", + "value = 1\n", + (0, "Harness Summary RESULT CHECK ok: preflight pass"), + id="preflight-passes", + ), + pytest.param( + "gate", + "_bad = 1\n", + ( + 1, + ( + "Harness Summary RESULT CHECK FAILED " + "src/mod.py:1: Name '_bad' starts with underscore rejected by harness" + ), + ), + id="gate-rejects", + ), + ], +) +def test_cli_summaries_report_complete_agent_check_results( + command: str, + source: str, + expected: tuple[int, str], + monkeypatch: pytest.MonkeyPatch, + git_repo: Path, ) -> None: - """The preflight summary names configured checks and the separate preferences containment result.""" + """Preflight and gate render every containment phase and preserve the final verdict exactly.""" + exit_code, summary = expected monkeypatch.setenv("RALPH_LOOP", "1") - source = git_repo / "src" / "mod.py" - source.parent.mkdir() - source.write_text("value = 1\n", encoding="utf-8") + monkeypatch.setattr(gates, "commit_checks" if command == "preflight" else "full_checks", {}) + source_path = git_repo / "src" / "mod.py" + source_path.parent.mkdir() + source_path.write_text(source, encoding="utf-8") gate.run_git(["add", "src/mod.py"], git_repo) - fake_popen(monkeypatch) - result = runner.invoke(cli.app, ["preflight"]) - assert result.exit_code == 0 + result = runner.invoke(cli.app, [command]) output = " ".join(unstyle(result.stdout).split()) - assert output == ( - "PHASE: LINT ruff check --no-cache --show-fixes . PHASE: PYLINT pylint . " - "PHASE: FORMAT ruff format --no-cache --check PHASE: COMPLEXIPY complexipy . " - "PHASE: BANNED PATTERNS CHECK checking for banned patterns in staged files " - "PHASE: USER PREFERENCES checking that user's preferences are respected " - "Harness Summary RESULT CHECK PASSED lint PASSED pylint PASSED format PASSED complexipy " - "ok: preflight pass" - ) - for name, command in gate.COMMIT_CHECKS.items(): - assert name in output - assert " ".join(command) in output - -def test_gate_summary_names_every_check_for_agents(monkeypatch: pytest.MonkeyPatch, git_repo: Path) -> None: - """The gate summary names configured checks and the separate preferences containment result.""" - monkeypatch.setenv("RALPH_LOOP", "1") - source = git_repo / "src" / "mod.py" - source.parent.mkdir() - source.write_text("value = 1\n", encoding="utf-8") - gate.run_git(["add", "src/mod.py"], git_repo) - fake_popen(monkeypatch) - result = runner.invoke(cli.app, ["gate"]) - assert result.exit_code == 0 - output = " ".join(unstyle(result.stdout).split()) - assert output == ( - "PHASE: LINT ruff check --no-cache --show-fixes . PHASE: PYLINT pylint . " - "PHASE: FORMAT ruff format --no-cache --check " - "PHASE: COMPLEXIPY complexipy . " - "PHASE: SECURITY semgrep scan --error --config auto --config p/secrets --exclude-rule " - "yaml.github-actions.security.github-actions-mutable-action-tag.github-actions-mutable-action-tag . " - "PHASE: TYPES pyright --outputjson " - "PHASE: PYTEST pytest -p no:cacheprovider -n auto --cov " - "--cov-report=term-missing --cov-fail-under=100 " - "PHASE: BANNED PATTERNS CHECK checking for banned patterns in staged files " - "PHASE: USER PREFERENCES checking that user's preferences are respected " - "Harness Summary RESULT CHECK PASSED lint PASSED pylint PASSED format PASSED complexipy " - "PASSED security PASSED types PASSED pytest ok: gate pass" + assert (result.exit_code, output) == ( + exit_code, + ( + "PHASE: AGENT CHECKS running non-human agent checks " + "PHASE: BANNED PATTERNS CHECK checking for banned patterns in staged files " + "PHASE: USER PREFERENCES checking that user's preferences are respected " + f"{summary}" + ), ) - for name, command in gate.FULL_CHECKS.items(): - assert name in output - assert " ".join(command) in output def test_status_counts_run_receipts_and_names_the_newest(git_repo: Path) -> None: @@ -288,7 +284,13 @@ def test_installing_the_template_cleans_the_repo_sets_hooks_and_reruns_cleanly( encoding="utf-8", ) (git_repo / "uv.lock").touch() - template_files = (".banner.svg", ".diagram.png", ".infin.png", ".loops_agents.svg", ".loops.svg") + template_files = ( + ".banner.svg", + ".diagram.png", + ".infin.png", + ".loops_agents.svg", + ".loops.svg", + ) for file_name in template_files: (git_repo / file_name).touch() (git_repo / ".github" / "workflows").mkdir(parents=True) @@ -316,17 +318,24 @@ def test_installing_the_template_cleans_the_repo_sets_hooks_and_reruns_cleanly( "requires-python": ">=3.11", "scripts": {"harness": "harness.cli:main"}, } - assert document["tool"]["pyright"] == {"typeCheckingMode": "strict", "include": ["src", "preferences"]} + assert document["tool"]["pyright"] == { + "typeCheckingMode": "strict", + "include": ["src", "preferences", "mutation"], + } + assert document["tool"]["mutmut"] == {"source_paths": ["src", "preferences", "mutation"]} assert document["tool"]["pytest"]["ini_options"] == { "addopts": ["-ra"], "testpaths": ["tests"], "pythonpath": [".", "src"], } assert document["tool"]["coverage"] == { - "run": {"source": ["src", "preferences"]}, + "run": {"source": ["src", "preferences", "mutation"]}, "report": {"fail_under": 100}, } - assert document["tool"]["complexipy"] == {"paths": ["src", "preferences"], "max-complexity-allowed": 10} + assert document["tool"]["complexipy"] == { + "paths": ["src", "preferences", "mutation"], + "max-complexity-allowed": 10, + } assert document["tool"]["ruff"]["exclude"] == [".git", "harness"] assert document["tool"]["pylint"]["main"]["ignore"] == [".git", "harness"] assert (git_repo / "README.md").read_text(encoding="utf-8") == "seed\n" @@ -376,7 +385,9 @@ def test_install_picks_the_package_manager_from_the_lockfile( (git_repo / lockfile).touch() calls: list[tuple[str, ...]] = [] monkeypatch.setattr( - subprocess, "run", stub_toolchain(subprocess.run, calls, str(poetry_bin / python_name)) + subprocess, + "run", + stub_toolchain(subprocess.run, calls, str(poetry_bin / python_name)), ) assert runner.invoke(cli.app, ["install"]).exit_code == 0 @@ -384,7 +395,16 @@ def test_install_picks_the_package_manager_from_the_lockfile( managers = { "uv": ("uv", "sync"), "poetry": ("poetry", "install"), - "pip": (str(interpreter / python_name), "-m", "pip", "install", "-r", "requirements.txt", "-e", "."), + "pip": ( + str(interpreter / python_name), + "-m", + "pip", + "install", + "-r", + "requirements.txt", + "-e", + ".", + ), } recorded = { "uv": harness_executable(git_repo / ".venv" / scripts), @@ -481,7 +501,7 @@ def capture_worker(command: list[str], log: Path, verbose: bool) -> int: assert launched[0][:3] == ["powershell.exe", "-NoProfile", "-File"] assert launched[0][3].endswith("ralph.ps1") assert launched[0][4:6] == ["2", "5"] - assert launched[0][6:] == list(AGENTS["claude"]) + assert launched[0][6:] == list(gates.agents["claude"]) def test_windows_run_uses_powershell_without_path_lookup( @@ -519,7 +539,10 @@ def test_windows_run_uses_powershell_without_path_lookup( ], ) def test_cleanup_applies_the_project_name_rules( - tmp_path: Path, initial_name: str | None, requested_name: str | None, expected_name: str + tmp_path: Path, + initial_name: str | None, + requested_name: str | None, + expected_name: str, ) -> None: """Cleanup starts the project at v0 and only accepts a name that is already PEP 503 normalized.""" (tmp_path / "README.md").write_text("old\n", encoding="utf-8") @@ -559,36 +582,6 @@ def test_tracked_hooks_call_registered_commands_without_venv_paths(hook: str) -> assert "uv" not in text -@pytest.mark.parametrize( - ("recorded", "message"), - [ - pytest.param(None, "hooks are not installed. Run 'harness install' in this repo.", id="never-ran"), - pytest.param("missing-harness", "is gone. Re-run 'harness install'.", id="stale-record"), - ], -) -def test_hooks_name_the_fix_when_the_recorded_harness_is_missing_or_stale( - recorded: str | None, message: str, git_repo: Path -) -> None: - """A repo without the install record, or one whose environment was rebuilt, gets instructions.""" - shutil.copytree(REPO_ROOT / ".githooks", git_repo / ".githooks", dirs_exist_ok=True) - if recorded: - (git_repo / ".git" / "harness-path").write_text(f"{git_repo / recorded}\n", encoding="utf-8") - env = {key: value for key, value in os.environ.items() if not key.startswith("GIT_")} - gate.run_git(["config", "core.hooksPath", ".githooks"], git_repo) - - result = subprocess.run( - ["git", "commit", "--allow-empty", "-m", "exercise pre-commit"], - cwd=git_repo, - capture_output=True, - text=True, - check=False, - env=env, - ) - - assert result.returncode == 1 - assert message in result.stderr - - @pytest.mark.parametrize( ("arguments", "code"), [ @@ -602,11 +595,11 @@ def test_prepare_commit_msg_forwards_gits_own_arguments( """The hook command hands git's own arguments to the gate logic and exits with its status.""" seen: list[list[str]] = [] - def commit_msg(argv: list[str]) -> int: + def commit_msg(_gate: Gate, argv: list[str]) -> int: seen.append(argv) return code - monkeypatch.setattr(cli, "commit_msg", commit_msg) + monkeypatch.setattr(Gate, "prepare_commit_msg", commit_msg) result = runner.invoke(cli.app, ["prepare-commit-msg", *arguments]) @@ -647,15 +640,17 @@ def test_a_harnessed_run_writes_numbered_receipts_and_propagates_exit_codes( assert not first.stdout assert launched[0][0].endswith("ralph.sh") assert launched[0][1:3] == ["1", "2"] - assert launched[0][3:] == list(AGENTS["claude"]) + assert launched[0][3:] == list(gates.agents["claude"]) receipts = git_repo / "scratchpad" / "runs" / "20990102" / "claude" assert (receipts / "0001.jsonl").read_text(encoding="utf-8") == '{"type":"result","result":"ok"}\n' - assert os.environ["RALPH_PROMPT"] == "Your agent id is `0001`\n\ndo the most important thing" + assert os.environ["RALPH_PROMPT"] == ( + "Your agent id prefix is `claude-0001`\n\ndo the most important thing" + ) second = runner.invoke(cli.app, ["run", "claude", "1", "2", "False", "--model", "haiku"]) assert second.exit_code == 0 - swapped = list(AGENTS["claude"]) + swapped = list(gates.agents["claude"]) swapped[swapped.index("--model") + 1] = "haiku" assert launched[1][3:] == swapped assert launched[1].count("--model") == 1 @@ -679,7 +674,11 @@ def test_run_worker_logs_every_line_and_streams_only_when_verbose( """ monkeypatch.setattr(cli, "REPO_ROOT_STR", str(tmp_path)) log = tmp_path / "out.jsonl" - streaming_worker = [sys.executable, "-c", 'print(\'{ "type" : "result" }\'); print("not json")'] + streaming_worker = [ + sys.executable, + "-c", + 'print(\'{ "type" : "result" }\'); print("not json")', + ] assert cli.run_worker(streaming_worker, log, verbose=True) == 0 @@ -689,7 +688,11 @@ def test_run_worker_logs_every_line_and_streams_only_when_verbose( assert "not json" in streamed assert log.read_text(encoding="utf-8") == '{ "type" : "result" }\nnot json\n' - failing_worker = [sys.executable, "-c", 'print("worker output"); raise SystemExit(3)'] + failing_worker = [ + sys.executable, + "-c", + 'print("worker output"); raise SystemExit(3)', + ] assert cli.run_worker(failing_worker, log, verbose=False) == 3 @@ -717,7 +720,7 @@ def test_claude_preset_runs_two_real_loop_iterations(monkeypatch: pytest.MonkeyP encoding="utf-8", ) preset = [sys.executable, str(worker), "--model", "opus", "-p"] - monkeypatch.setitem(cli.AGENTS, "claude", preset) + monkeypatch.setitem(gates.agents, "claude", preset) monkeypatch.chdir(git_repo) monkeypatch.setenv("PATH", f"{bin_dir}{os.pathsep}{os.environ['PATH']}") monkeypatch.setattr(cli, "REPO_ROOT_STR", str(git_repo)) @@ -729,7 +732,7 @@ def test_claude_preset_runs_two_real_loop_iterations(monkeypatch: pytest.MonkeyP assert result.exit_code == 0 assert (git_repo / "claude-count").read_text(encoding="utf-8") == "2" - identity = "Your agent id is `0001`\n\n" + identity = "Your agent id prefix is `claude-0001`\n\n" assert (git_repo / "prompt-1.txt").read_text(encoding="utf-8") == ( f"{identity}build from specs\n\nRALPH_ITERATION=1/2\n" ) @@ -741,6 +744,8 @@ def test_claude_preset_runs_two_real_loop_iterations(monkeypatch: pytest.MonkeyP *preset_args, *preset_args, ] - assert (git_repo / "scratchpad" / "runs" / "20990102" / "claude" / "0001.jsonl").read_text( - encoding="utf-8" - ) == '{"type": "result", "result": "ok"}\n{"type": "result", "result": "ok"}\n' + receipt = git_repo / "scratchpad" / "runs" / "20990102" / "claude" / "0001.jsonl" + events = [json.loads(line) for line in receipt.read_text(encoding="utf-8").splitlines()] + assert [event["type"] for event in events] == ["ralph", "result", "ralph", "result", "ralph"] + assert [event.get("iteration") for event in events] == [1, None, 2, None, None] + assert events[-1]["completed"] == 2 diff --git a/harness/tests/test_gate.py b/harness/tests/test_gate.py index 5672220..d25f952 100644 --- a/harness/tests/test_gate.py +++ b/harness/tests/test_gate.py @@ -2,9 +2,9 @@ from __future__ import annotations -import importlib import json import os +import runpy import shutil import subprocess import sys @@ -15,8 +15,11 @@ import pytest from harness import gate +from harness.gate import Gate, gates from harness.tests.conftest import REPO_ROOT, fake_popen +WARNING_THRESHOLD = round(gates.error_diff_lines * 0.75) + def stage(repo: Path, name: str, content: str) -> None: """Write a file inside the repo and stage it.""" @@ -44,6 +47,7 @@ def stage_a_bad_iteration(repo: Path) -> None: stage(repo, "PyProject.TOML", "[tool.harness]\n") stage(repo, ".github/workflows/ci.yml", "jobs:\n gate:\n steps: []\n") stage(repo, ".githooks/pre-commit", "#!/bin/sh\nexit 0\n") + stage(repo, "docs/notes.md", "Run with `# noqa` to silence the linter.\n") stage(repo, "src/sloppy.py", "import os # noqa\n") stage(repo, "release.sh", "git commit --no-verify -m ship\n") stage(repo, "src/named.py", "_bad = 1\n") @@ -67,76 +71,106 @@ def get_logged_calls_and_clear(repo: Path) -> list[object]: return calls -@pytest.fixture -def real_hook_repo(request: pytest.FixtureRequest, git_repo: Path, monkeypatch: pytest.MonkeyPatch) -> Path: - """Wire selected tracked hooks to a recorded executable in the disposable repository.""" - hooks = git_repo / ".active-hooks" - hooks.mkdir() - for name in (*request.param, "_resolve"): - shutil.copy2(REPO_ROOT / ".githooks" / name, hooks / name) - gate.run_git(["config", "core.hooksPath", ".active-hooks"], git_repo) - - executable = git_repo / "recorded-harness" - executable.write_text( - f"""#!{Path(sys.executable).as_posix()} -import json -import os -import sys -from pathlib import Path - -repo = Path.cwd() -arguments = sys.argv[1:] -command = arguments[0] if arguments else "" -recorded = arguments.copy() -if command == "prepare-commit-msg" and len(recorded) > 1: - recorded[1] = Path(recorded[1]).name -with (repo / "harness.calls").open("a", encoding="utf-8") as handle: - handle.write(json.dumps({{"arguments": recorded, "RALPH_LOOP": os.environ.get("RALPH_LOOP")}}) + "\\n") -real_file = repo / "harness.real" -real_commands = real_file.read_text(encoding="utf-8").splitlines() if real_file.exists() else [] -if command == "prepare-commit-msg" or command in real_commands: - os.chdir({str(REPO_ROOT)!r}) - from harness import cli, gate - os.chdir(repo) - gate.REPO_ROOT = repo - if command == "preflight": - gate.COMMIT_CHECKS = {{}} - cli.main(arguments) -status_file = repo / "harness.exit" -raise SystemExit(int(status_file.read_text(encoding="utf-8")) if status_file.exists() else 0) -""", - encoding="utf-8", - ) - executable.chmod(0o755) - (git_repo / ".git" / "harness-path").write_text(f"{executable}\n", encoding="utf-8") - monkeypatch.setenv("RALPH_LOOP", "1") - return git_repo - - @pytest.mark.parametrize("real_hook_repo", [("pre-commit",)], indirect=True) @pytest.mark.parametrize( - ("exit_code", "lands"), [pytest.param(0, True, id="passing"), pytest.param(1, False, id="blocking")] + "case", + [ + pytest.param(("valid", 0, True, True), id="passing"), + pytest.param(("valid", 1, False, True), id="blocking"), + pytest.param(("missing", 0, True, True), id="missing-record"), + pytest.param(("stale", 0, True, True), id="stale-record"), + pytest.param(("unavailable", 0, False, False), id="unavailable"), + ], ) def test_pre_commit_hook_dispatches_preflight_and_controls_commit( - exit_code: int, lands: bool, real_hook_repo: Path + case: tuple[str, int, bool, bool], real_hook_repo: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - """The tracked pre-commit hook runs the recorded preflight and owns the commit verdict.""" + """The tracked pre-commit hook resolves preflight and owns the commit verdict.""" + recorded = real_hook_repo / ".git" / "harness-path" + expected_record = recorded.read_text(encoding="utf-8") + if case[0] in {"missing", "stale"}: + if case[0] == "missing": + recorded.unlink() + bin_dir = real_hook_repo / "bin" + bin_dir.mkdir() + executable = bin_dir / "harness" + (real_hook_repo / "recorded-harness").rename(executable) + monkeypatch.setenv("PATH", f"{bin_dir}{os.pathsep}{os.environ['PATH']}") + expected_record = f"{executable}\n" + elif case[0] == "unavailable": + recorded.unlink() + tool_dir = real_hook_repo / "tool-bin" + tool_dir.mkdir() + for name in ("git", "dirname"): + executable = shutil.which(name) + assert executable + (tool_dir / Path(executable).name).symlink_to(executable) + monkeypatch.setenv("PATH", str(tool_dir)) + stage(real_hook_repo, "feature.py", "value = 1\n") - (real_hook_repo / "harness.exit").write_text(str(exit_code), encoding="utf-8") + (real_hook_repo / "harness.exit").write_text(str(case[1]), encoding="utf-8") before = gate.run_git(["rev-parse", "HEAD"], real_hook_repo).strip() result = git_process(real_hook_repo, "commit", "-q", "-m", "exercise pre-commit") - after = gate.run_git(["rev-parse", "HEAD"], real_hook_repo).strip() + assert ( result.returncode == 0, - get_logged_calls_and_clear(real_hook_repo), - after != before, + get_logged_calls_and_clear(real_hook_repo) if case[3] else [], + gate.run_git(["rev-parse", "HEAD"], real_hook_repo).strip() != before, gate.run_git(["show", "--name-only", "--format=", "HEAD"], real_hook_repo).splitlines(), ) == ( - lands, - [{"arguments": ["preflight"], "RALPH_LOOP": "1"}], - lands, - ["feature.py"] if lands else [".gitignore", "README.md", "README.template.md"], + case[2], + [{"arguments": ["preflight"], "RALPH_LOOP": "1"}] if case[3] else [], + case[2], + ["feature.py"] if case[2] else [".gitignore", "README.md", "README.template.md"], ) + if case[0] == "unavailable": + assert not recorded.exists() + assert result.stderr == "loopgate: hooks are not installed. Run 'harness install' in this repo.\n" + else: + # Git Bash records `command -v` hits as MSYS paths (/c/Users/...); compare tail, not drive + recorded_path = Path(recorded.read_text(encoding="utf-8").strip()) + assert recorded_path.parts[-3:] == Path(expected_record.strip()).parts[-3:] + + +@pytest.mark.parametrize("real_hook_repo", [("pre-commit",)], indirect=True) +@pytest.mark.parametrize( + "case", + [ + pytest.param((WARNING_THRESHOLD, WARNING_THRESHOLD, True, None), id="at-warning-threshold"), + pytest.param((WARNING_THRESHOLD + 1, WARNING_THRESHOLD + 1, True, "WARNED"), id="warn"), + pytest.param( + (gates.error_diff_lines, gates.error_diff_lines, True, "WARNED"), id="at-error-threshold" + ), + pytest.param((gates.error_diff_lines + 1, WARNING_THRESHOLD, False, "FAILED"), id="fail"), + ], +) +def test_pre_commit_hook_warns_then_blocks_on_combined_diff_size( + case: tuple[int, int, bool, str | None], real_hook_repo: Path +) -> None: + """The real hook warns above the review threshold and blocks only above the combined diff cap.""" + total, staged_lines, lands, verdict = case + (real_hook_repo / "harness.real").write_text("preflight\n", encoding="utf-8") + stage(real_hook_repo, "notes.txt", "staged line\n" * staged_lines) + unstaged_lines = total - staged_lines + if unstaged_lines: + (real_hook_repo / "README.md").write_text( + "seed\n" + "unstaged line\n" * unstaged_lines, encoding="utf-8" + ) + before = gate.run_git(["rev-parse", "HEAD"], real_hook_repo).strip() + + result = git_process(real_hook_repo, "commit", "-q", "-m", f"{total} line iteration") + output = result.stdout + result.stderr + after = gate.run_git(["rev-parse", "HEAD"], real_hook_repo).strip() + + assert result.returncode == (0 if lands else 1) + assert (after != before) is lands + assert f"{total} lines modified" in output + assert get_logged_calls_and_clear(real_hook_repo) == [{"arguments": ["preflight"], "RALPH_LOOP": "1"}] + if verdict is None: + assert "WARNED" not in output + assert "FAILED" not in output + else: + assert verdict in output @pytest.mark.parametrize("real_hook_repo", [("pre-commit", "pre-push")], indirect=True) @@ -175,7 +209,7 @@ def test_prepare_commit_msg_hook_rejects_empty_agent_then_accepts_staged_work(re {"arguments": ["prepare-commit-msg", "COMMIT_EDITMSG", "message"], "RALPH_LOOP": "1"} ] assert gate.run_git(["rev-parse", "HEAD"], real_hook_repo).strip() == before - assert "Empty-tree commit detected" in agent_empty.stdout + agent_empty.stderr + assert "Empty commit detected" in agent_empty.stdout + agent_empty.stderr stage(real_hook_repo, "feature.py", "value = 1\n") agent_work = git_process(real_hook_repo, "commit", "-q", "-m", "agent work") @@ -187,19 +221,31 @@ def test_prepare_commit_msg_hook_rejects_empty_agent_then_accepts_staged_work(re "feature.py" ] + # githooks(5): the hook's first parameter is always the message file; a plain + # `git commit` (no -m/-t/merge/squash/amend) passes no source argument at all. + stage(real_hook_repo, "plain.py", "plain = 1\n") + git_process(real_hook_repo, "-c", "core.editor=true", "commit", "-q") + assert get_logged_calls_and_clear(real_hook_repo) == [ + {"arguments": ["prepare-commit-msg", "COMMIT_EDITMSG"], "RALPH_LOOP": "1"} + ] + @pytest.mark.parametrize("real_hook_repo", [("prepare-commit-msg",)], indirect=True) def test_prepare_commit_msg_hook_allows_human_empty_commit(real_hook_repo: Path) -> None: - """The hook does not apply agent containment to a human's empty commit.""" + """A human bypasses the harness executable entirely and can create an empty commit.""" + before = gate.run_git(["rev-parse", "HEAD"], real_hook_repo).strip() human_empty = git_process( real_hook_repo, "commit", "--allow-empty", "--no-verify", "-q", "-m", "human empty", loop=False ) - assert human_empty.returncode == 0 - assert get_logged_calls_and_clear(real_hook_repo) == [ - {"arguments": ["prepare-commit-msg", "COMMIT_EDITMSG", "message"], "RALPH_LOOP": None} - ] - assert gate.run_git(["show", "--name-only", "--format=", "HEAD"], real_hook_repo).splitlines() == [] + assert ( + human_empty.returncode, + human_empty.stdout, + human_empty.stderr, + gate.run_git(["rev-parse", "HEAD"], real_hook_repo).strip() != before, + gate.run_git(["show", "--name-only", "--format=", "HEAD"], real_hook_repo).splitlines(), + (real_hook_repo / "harness.calls").exists(), + ) == (0, "", "", True, [], False) @pytest.mark.parametrize("real_hook_repo", [("pre-commit", "prepare-commit-msg")], indirect=True) @@ -222,9 +268,12 @@ def test_agent_iteration_is_contained_and_rejected( get_logged_calls_and_clear(real_hook_repo), ) == (True, True, initial_head, [prepare]) message_file = real_hook_repo / ".git" / "COMMIT_EDITMSG" - message_file.write_text("# generated comment only\n", encoding="utf-8") - assert gate.prepare_commit_msg(["prepare-commit-msg", str(message_file), "message"]) == 1 - assert "Commit message is blank" in capsys.readouterr().out + message_file.write_text("\n\n# generated comment only\n", encoding="utf-8") + assert gates.prepare_commit_msg(["prepare-commit-msg", str(message_file), "message"]) == 1 + assert capsys.readouterr().out == ( + "PHASE: PRE COMMIT MESSAGE\n" + "Commit message is blank. Provide an informative message with your agent ID.\n\n" + ) bad = git_process(real_hook_repo, "commit", "-q", "-m", "bad and forbidden work") assert ( @@ -232,17 +281,23 @@ def test_agent_iteration_is_contained_and_rejected( gate.run_git(["rev-parse", "HEAD"], real_hook_repo).strip(), gate.run_git(["diff", "--cached", "--name-only"], real_hook_repo).splitlines(), get_logged_calls_and_clear(real_hook_repo), - [value in bad.stdout + bad.stderr for value in ("# noqa", "--no-verify", "_bad")], + [ + value in bad.stdout + bad.stderr + for value in ("Run with `# noqa`", "# noqa", "--no-verify", "_bad") + ], ) == ( True, initial_head, - ["release.sh", "src/clean.py", "src/feature.py", "src/named.py", "src/sloppy.py"], + ["docs/notes.md", "release.sh", "src/clean.py", "src/feature.py", "src/named.py", "src/sloppy.py"], [preflight], - [True, True, True], + [True, True, True, True], ) assert (real_hook_repo / "harness" / "gate.py").exists() - gate.run_git(["reset", "-q", "HEAD", "--", "release.sh", "src/named.py", "src/sloppy.py"], real_hook_repo) + gate.run_git( + ["reset", "-q", "HEAD", "--", "docs/notes.md", "release.sh", "src/named.py", "src/sloppy.py"], + real_hook_repo, + ) good = git_process(real_hook_repo, "commit", "-q", "-m", "good work") good_head = gate.run_git(["rev-parse", "HEAD"], real_hook_repo).strip() assert ( @@ -257,7 +312,7 @@ def test_agent_iteration_is_contained_and_rejected( forbidden = git_process(real_hook_repo, "commit", "-q", "-m", "forbidden only") assert ( forbidden.returncode != 0, - "Empty-tree commit detected" in forbidden.stdout + forbidden.stderr, + "Empty commit detected" in forbidden.stdout + forbidden.stderr, gate.run_git(["rev-parse", "HEAD"], real_hook_repo).strip(), gate.run_git(["diff", "--cached", "--name-only"], real_hook_repo).splitlines(), get_logged_calls_and_clear(real_hook_repo), @@ -266,7 +321,7 @@ def test_agent_iteration_is_contained_and_rejected( empty = git_process(real_hook_repo, "commit", "-q", "--allow-empty", "--no-verify", "-m", "empty work") assert ( empty.returncode != 0, - "Empty-tree commit detected" in empty.stdout + empty.stderr, + "Empty commit detected" in empty.stdout + empty.stderr, gate.run_git(["rev-parse", "HEAD"], real_hook_repo).strip(), get_logged_calls_and_clear(real_hook_repo), ) == (True, True, good_head, [prepare]) @@ -279,28 +334,38 @@ def test_agent_iteration_that_does_the_work_lands( monkeypatch.setenv("RALPH_LOOP", "1") monkeypatch.chdir(git_repo) stage(git_repo, "src/feature.py", "value = 2\n") - stage(git_repo, "docs/notes.md", "Run with `# noqa` to silence the linter.\n") (git_repo / ".git" / "COMMIT_EDITMSG").write_text("add the feature\n", encoding="utf-8") - assert gate.prepare_commit_msg(["prepare-commit-msg", ".git/COMMIT_EDITMSG", "message"]) == 0 + assert gates.prepare_commit_msg(["prepare-commit-msg", ".git/COMMIT_EDITMSG", "message"]) == 0 (git_repo / ".git" / "COMMIT_EDITMSG").write_text("add the feature\n", encoding="utf-8") - assert gate.prepare_commit_msg(["prepare-commit-msg", ".git/COMMIT_EDITMSG", "commit"]) == 0 - assert gate.run_non_human_checks() == [] - assert gate.run_git(["diff", "--cached", "--name-only"]).splitlines() == [ - "docs/notes.md", - "src/feature.py", - ] + assert gates.prepare_commit_msg(["prepare-commit-msg", ".git/COMMIT_EDITMSG"]) == 0 + with pytest.raises((IsADirectoryError, PermissionError)): + gates.prepare_commit_msg(["prepare-commit-msg"]) + monkeypatch.setattr(gates, "commit_checks", {}) + assert (gates.run_preflight(), gate.run_git(["diff", "--cached", "--name-only"]).splitlines()) == ( + {"pass": [], "fail": [], "warn": []}, + ["src/feature.py"], + ) gate.run_git(["commit", "-q", "-m", "add the feature"], git_repo) wipe_history(git_repo) stage(git_repo, "first.py", "x = 1\n") (git_repo / ".git" / "COMMIT_EDITMSG").write_text("first commit\n", encoding="utf-8") - assert gate.prepare_commit_msg(["prepare-commit-msg", ".git/COMMIT_EDITMSG", "message"]) == 0 + assert gates.prepare_commit_msg(["prepare-commit-msg", ".git/COMMIT_EDITMSG", "message"]) == 0 assert "[COMMIT BLOCKED]" not in capsys.readouterr().out - calls = fake_popen(monkeypatch) - assert gate.run_gate()["fail"] == [] - assert [launch[0] for launch in calls] == list(gate.FULL_CHECKS.values()) + git_calls: list[tuple[list[str], bool | None]] = [] + + def record(args: list[str], check: bool = True) -> str: + git_calls.append((args, check)) + return "abc123\n" if args[0] == "rev-parse" else "first.py\n" + + monkeypatch.setattr(gate, "run_git", record) + assert gates.prepare_commit_msg(["prepare-commit-msg", ".git/COMMIT_EDITMSG"]) == 0 + assert git_calls == [ + (["rev-parse", "--verify", "HEAD"], False), + (["diff-index", "--cached", "--name-only", "HEAD"], True), + ] @pytest.mark.parametrize( @@ -325,17 +390,21 @@ def test_agent_cannot_commit_an_empty_iteration( """Nothing staged is nothing done, and rewriting history is not a way to produce work.""" monkeypatch.setenv("RALPH_LOOP", "1") monkeypatch.chdir(git_repo) - empty = "Empty-tree commit detected. Stage real work and don't use --allow-empty. Lazy.\n" + empty = "Empty commit detected. Stage real work, Don't use --allow-empty. Or say if you're blocked\n" (git_repo / ".git" / "COMMIT_EDITMSG").write_text("did nothing\n", encoding="utf-8") - assert gate.prepare_commit_msg(["prepare-commit-msg", ".git/COMMIT_EDITMSG", source]) == 1 - assert capsys.readouterr().out == f"\n[COMMIT BLOCKED]:\n{refusal}{empty}\n" + assert gates.prepare_commit_msg(["prepare-commit-msg", ".git/COMMIT_EDITMSG", source]) == 1 + assert capsys.readouterr().out == f"PHASE: PRE COMMIT MESSAGE\n{refusal}{empty}\n" wipe_history(git_repo) (git_repo / ".git" / "COMMIT_EDITMSG").write_text("did nothing\n", encoding="utf-8") - assert gate.prepare_commit_msg(["prepare-commit-msg", ".git/COMMIT_EDITMSG", "message"]) == 1 - assert capsys.readouterr().out == f"\n[COMMIT BLOCKED]:\n{empty}\n" - assert gate.run_non_human_checks() == [] + assert gates.prepare_commit_msg(["prepare-commit-msg", ".git/COMMIT_EDITMSG", "message"]) == 1 + assert capsys.readouterr().out == f"PHASE: PRE COMMIT MESSAGE\n{empty}\n" + + blank = "Commit message is blank. Provide an informative message with your agent ID.\n" + (git_repo / ".git" / "COMMIT_EDITMSG").write_text("", encoding="utf-8") + assert gates.prepare_commit_msg(["prepare-commit-msg", ".git/COMMIT_EDITMSG", "message"]) == 1 + assert capsys.readouterr().out == f"PHASE: PRE COMMIT MESSAGE\n{empty}{blank}\n" def test_human_running_the_same_commands_is_not_policed( @@ -350,15 +419,19 @@ def test_human_running_the_same_commands_is_not_policed( before = gate.run_git(["diff", "--cached", "--name-only"]).splitlines() (git_repo / ".git" / "COMMIT_EDITMSG").write_text("", encoding="utf-8") - assert gate.prepare_commit_msg(["prepare-commit-msg", ".git/COMMIT_EDITMSG", "message"]) == 0 + assert gates.prepare_commit_msg(["prepare-commit-msg", ".git/COMMIT_EDITMSG", "message"]) == 0 assert not capsys.readouterr().out calls = fake_popen(monkeypatch) - assert gate.run_preflight()["fail"] == [] + assert gates.run_preflight()["fail"] == [] recorder.assert_not_called() assert gate.run_git(["diff", "--cached", "--name-only"]).splitlines() == before assert "harness/gate.py" in before - assert all(env["FORCE_COLOR"] == "1" for _, _, env in calls) + assert all( + (env["FORCE_COLOR"], env["CLICOLOR_FORCE"], env["SEMGREP_FORCE_COLOR"]) == ("1", "1", "1") + for _, _, env in calls + ) + assert all(cwd == git_repo for _, cwd, _ in calls) assert not [key for _, _, env in calls for key in env if key.startswith("GIT_")] @@ -367,10 +440,10 @@ def test_human_running_the_same_commands_is_not_policed( [ *( pytest.param(f"{directory}blocked.txt", id=f"dir-{directory}") - for directory in gate.FORBIDDEN_DIRS + for directory in gates.forbidden_dirs if directory != ".git/" ), - *(pytest.param(path, id=f"file-{path}") for path in gate.FORBIDDEN_FILES), + *(pytest.param(path, id=f"file-{path}") for path in gates.forbidden_files), ], ) def test_every_configured_forbidden_path_is_ejected_except_dot_git( @@ -382,34 +455,50 @@ def test_every_configured_forbidden_path_is_ejected_except_dot_git( stage(git_repo, forbidden_path, "blocked\n") assert gate.run_git(["diff", "--cached", "--name-only"]).splitlines() == [forbidden_path] - assert gate.run_non_human_checks() == [] - assert gate.run_git(["diff", "--cached", "--name-only"]).splitlines() == [] - - assert ".git/" in gate.FORBIDDEN_DIRS - - -def test_every_configured_check_can_block_the_gate(monkeypatch: pytest.MonkeyPatch, git_repo: Path) -> None: - """Each configured check takes its turn failing; all of them run and the failing one blocks.""" - monkeypatch.setenv("RALPH_LOOP", "1") - stage(git_repo, "src/mod.py", "value = 1\n") - fake_popen(monkeypatch, fails=list(gate.FULL_CHECKS.values())) - - assert gate.run_gate() == { - "pass": [], - "fail": ["lint", "pylint", "complexipy", "security", "types", "pytest"], - "warn": ["format"], - } + monkeypatch.setattr(gates, "commit_checks", {}) + assert ( + gates.run_preflight(), + gate.run_git(["diff", "--cached", "--name-only"]).splitlines(), + ".git/" in gates.forbidden_dirs, + ) == ({"pass": [], "fail": [], "warn": []}, [], True) def test_gate_runs_exactly_what_pyproject_configures( monkeypatch: pytest.MonkeyPatch, capfd: pytest.CaptureFixture[str], git_repo: Path ) -> None: - """The gate dispatches whatever is configured, in order, and says so when nothing is.""" + """A root owns its complete configuration, Git target, command dispatch, and containment results.""" raw_toml = tomllib.loads((REPO_ROOT / "pyproject.toml").read_bytes().decode())["tool"]["harness"] - assert raw_toml["preflight"] == gate.COMMIT_CHECKS - assert raw_toml["preflight"] | raw_toml["gate"] == gate.FULL_CHECKS - - live = gate.run_checks({ + configured = Gate(REPO_ROOT) + assert vars(configured) == { + "repo_root": REPO_ROOT, + "forbidden": raw_toml["FORBIDDEN"], + "languages": raw_toml["languages"], + "agents": raw_toml["agents"], + "commit_checks": raw_toml["preflight"], + "full_checks": raw_toml["preflight"] | raw_toml["gate"], + "forbidden_files": tuple(raw_toml["FORBIDDEN"]["FILES"]), + "forbidden_dirs": tuple(raw_toml["FORBIDDEN"]["DIRS"]), + "forbidden_patterns": tuple(raw_toml["FORBIDDEN"]["PATTERNS"]), + "error_diff_lines": raw_toml["error_diff_lines"], + } + assert vars(gates) == {**vars(configured), "repo_root": git_repo} + (git_repo / "pyproject.toml").write_text("[project]\nname = 'x'\n", encoding="utf-8") + with pytest.raises(KeyError): + Gate(git_repo) + assert ( + Path(gate.run_git(["rev-parse", "--show-toplevel"]).strip()), + Path(gate.run_git(["rev-parse", "--show-toplevel"], REPO_ROOT).strip()), + ) == (git_repo, REPO_ROOT) + monkeypatch.setenv("GIT_DIR", str(git_repo / "no-such-dir")) + monkeypatch.delenv("RALPH_LOOP", raising=False) + assert Path(gate.run_git(["rev-parse", "--show-toplevel"]).strip()) == git_repo + monkeypatch.delenv("GIT_DIR") + absent = ["rev-parse", "--verify", "refs/heads/absent"] + assert not gate.run_git(absent, git_repo, check=False) + with pytest.raises(subprocess.CalledProcessError): + gate.run_git(absent, git_repo) + + live = gates.run_checks({ "ruff lint": [sys.executable, "-c", "print('hello from the check')"], "pyright types": [sys.executable, "-c", "raise SystemExit(7)"], "ruff format": [sys.executable, "-c", "raise SystemExit(1)"], @@ -418,68 +507,202 @@ def test_gate_runs_exactly_what_pyproject_configures( printed = capfd.readouterr().out assert "hello from the check" in printed assert "PHASE: RUFF LINT" in printed + assert "\x1b[5;36;48;5;235m" in printed # rule drawn in the blink-cyan-on-grey15 style + assert " \x1b[2;3" in printed # command line centered (leading spaces), in dim italic - calls = fake_popen(monkeypatch) - preflight = gate.run_preflight() - assert [launch[0] for launch in calls] == list(gate.COMMIT_CHECKS.values()) - assert all(cwd == gate.REPO_ROOT for _, cwd, _ in calls) - assert preflight == {"pass": ["lint", "pylint", "format", "complexipy"], "fail": [], "warn": []} - - preflight_output = capfd.readouterr().out - for name in gate.COMMIT_CHECKS: - assert preflight_output.count(f"PHASE: {name.upper()}") == 1 - assert "PHASE: COMMAND" not in preflight_output - - calls.clear() - full = gate.run_gate() - assert [launch[0] for launch in calls] == list(gate.FULL_CHECKS.values()) - assert full == { - "pass": ["lint", "pylint", "format", "complexipy", "security", "types", "pytest"], - "fail": [], - "warn": [], - } - gate_output = capfd.readouterr().out - for name in gate.FULL_CHECKS: - assert gate_output.count(f"PHASE: {name.upper()}") == 1 - assert "PHASE: COMMAND" not in gate_output - - without_format = {name: cmd for name, cmd in gate.COMMIT_CHECKS.items() if name != "format"} - monkeypatch.setattr(gate, "COMMIT_CHECKS", without_format) - assert gate.run_preflight() == {"pass": list(without_format), "fail": [], "warn": []} - - js_checks = {"lint": ["npm", "run", "lint"], "format": ["npm", "run", "format:check"]} - monkeypatch.setattr(gate, "FULL_CHECKS", js_checks) monkeypatch.setenv("RALPH_LOOP", "1") - stage(git_repo, "src/app.js", "console.log('pass');\n") - assert gate.run_gate() == {"pass": ["lint", "format"], "fail": [], "warn": []} - - monkeypatch.setattr(gate, "FULL_CHECKS", {}) - assert gate.run_gate() == {"pass": [], "fail": [], "warn": []} - stage(git_repo, "src/mod.py", "_bad = 1\nf = lambda: 0\n") - assert gate.run_gate() == { + monkeypatch.setattr(gates, "full_checks", {}) + stage( + git_repo, + "src/mod.py", + "_bad = 1\n" + "f = lambda: 0\n" + "for item in []:\n" + " for inner in []:\n" + " continue\n" + "for item in []:\n" + " if item:\n" + " continue\n" + "while flag:\n" + " if flag:\n" + " continue\n" + "class Pointless:\n" + " def only(self):\n" + " pass\n" + "class Based(dict):\n" + " pass\n" + "class Keyed(metaclass=type):\n" + " pass\n" + "class TwoMethods:\n" + " def one(self):\n" + " pass\n" + " def two(self):\n" + " pass\n" + "assert True\n" + "globals()\n" + "locals()\n" + "print(*[1, 2])\n" + "pairs = [x for x in [] for y in [] if x]\n", + ) + assert gates.run_gate() == { "pass": [], "fail": [ ( - "problems:\nsrc/mod.py:1: Name '_bad' starts with underscore\nsrc/mod.py:2: Lambda found " - "hurting readability and adding complexity." + "src/mod.py:9: 'continue' inside a while loop banned to prevent infinite freezes\n" + "src/mod.py:12: 'Pointless': no base, decorator, or behavior: use function or Pydantic\n" + "src/mod.py:24: Lazy test assertion detected\n" + "src/mod.py:1: Name '_bad' starts with underscore\n" + "src/mod.py:2: Lambda found hurting readability and adding complexity.\n" + "src/mod.py:25: Dynamic injection of memory registry spotted\n" + "src/mod.py:26: Dynamic injection of memory registry spotted\n" + "src/mod.py:28: Overly complex comprehension, use a loop or type Set math\n" + "src/mod.py:5: Overly-nested 'continue' detected inside multiple if/for blocks" ) ], "warn": [], } + printed = capfd.readouterr().out + assert printed == ( + "PHASE: AGENT CHECKS" + "\nrunning non-human agent checks" + "\nPHASE: BANNED PATTERNS CHECK" + "\nchecking for banned patterns in staged files" + "\nPHASE: USER PREFERENCES" + "\nchecking that user's preferences are respected\n" + ) + assert "\x1b" not in printed # agents in the loop get plain text, never ANSI + assert gate.run_git(["diff-index", "--cached", "--name-only", "HEAD"]) == "src/mod.py\n" + + +def test_diff_size_counts_only_relevant_changed_lines( + monkeypatch: pytest.MonkeyPatch, capfd: pytest.CaptureFixture[str], git_repo: Path +) -> None: + """Count additions, deletions, and docs while excluding generated and binary files.""" + monkeypatch.setenv("RALPH_LOOP", "1") + monkeypatch.setattr(gates, "commit_checks", {}) + stage(git_repo, "src/mod.py", "".join(f"old_{line} = {line}\n" for line in range(WARNING_THRESHOLD))) + gate.run_git(["commit", "-q", "-m", "seed rewrite"], git_repo) + stage(git_repo, "src/mod.py", "new = 1\n") + + rewritten = gates.run_preflight() + rewrite_output = capfd.readouterr().out + + gate.run_git(["commit", "-q", "-m", "rewrite module"], git_repo) + filtered_lines = WARNING_THRESHOLD + 1 + stage(git_repo, "notes.md", "note\n" * filtered_lines) + stage(git_repo, "src/tiny.py", "tiny_one = 1\ntiny_two = 2\n") + stage(git_repo, "uv.lock", "generated\n" * 500) + (git_repo / "logo.png").write_bytes(b"\0binary") + gate.run_git(["add", "logo.png"], git_repo) + + filtered = gates.run_preflight() + filtered_output = capfd.readouterr().out + + rewritten_lines = WARNING_THRESHOLD + 1 + assert (rewritten["pass"], rewritten["fail"], len(rewritten["warn"])) == ([], [], 1) + assert f"{rewritten_lines} lines modified" in rewrite_output + assert (filtered["pass"], filtered["fail"], len(filtered["warn"])) == ([], [], 1) + assert f"{filtered_lines + 2} lines modified" in filtered_output + + +@pytest.mark.parametrize( + ("lines", "verdict"), + [ + pytest.param(WARNING_THRESHOLD, "quiet", id="at-warn"), + pytest.param(WARNING_THRESHOLD + 1, "advised", id="over-warn"), + pytest.param(gates.error_diff_lines, "advised", id="at-cap"), + pytest.param(gates.error_diff_lines + 1, "blocked", id="over-cap"), + ], +) +def test_diff_size_warns_then_blocks_as_the_change_grows( + lines: int, + verdict: str, + monkeypatch: pytest.MonkeyPatch, + capfd: pytest.CaptureFixture[str], + git_repo: Path, +) -> None: + """The configured thresholds advise first and only block once the change reaches the cap.""" + monkeypatch.setenv("RALPH_LOOP", "1") + monkeypatch.setattr(gates, "commit_checks", {}) + stage(git_repo, "src/big.py", "value = 1\n" * lines) + + results = gates.run_preflight() + output = capfd.readouterr().out + + message = ( + f"{lines} lines modified. WARN at 75% {WARNING_THRESHOLD} lines, " + f"ERROR at {gates.error_diff_lines}.\nSuggestion: Refactor bloat, inline helpers, " + "reduce mis-direction, re-use fixtures, cut duplication, slim down if-elif-else blocks." + ) + expected: dict[str, list[str]] = {"pass": [], "fail": [], "warn": []} + if verdict != "quiet": + expected["fail" if verdict == "blocked" else "warn"].append(message) + + assert (results, message in output, "PHASE: DIFF SIZE" in output) == (expected, True, True) + + +def test_diff_size_measures_the_very_first_commit_of_a_repository( + monkeypatch: pytest.MonkeyPatch, capfd: pytest.CaptureFixture[str], git_repo: Path +) -> None: + """With no HEAD to compare against, the empty tree is the baseline, so nothing escapes unmeasured.""" + monkeypatch.setenv("RALPH_LOOP", "1") + wipe_history(git_repo) + filepath = "src/first.py" + lines = WARNING_THRESHOLD + 1 + stage(git_repo, filepath, "value = 1\n" * lines) + + assert not gate.run_git(["rev-parse", "--verify", "HEAD"], git_repo, check=False) + monkeypatch.setattr(gates, "commit_checks", {}) + results = gates.run_preflight() + output = capfd.readouterr().out + + assert (results["pass"], results["fail"], len(results["warn"])) == ([], [], 1) + assert f"{lines} lines modified" in output + + +def test_diff_size_still_measures_a_commit_with_nothing_staged( + monkeypatch: pytest.MonkeyPatch, capfd: pytest.CaptureFixture[str], git_repo: Path +) -> None: + """Sprawl left entirely unstaged is still sprawl: the size check runs before the empty-commit exit.""" + monkeypatch.setenv("RALPH_LOOP", "1") + filepaths = ["src/mod.py", "mod.py"] + stage(git_repo, filepaths[0], "value = 1\n") + gate.run_git(["commit", "-q", "-m", "seed the file"], git_repo) + # One line already exists, so the rewrite has to be a line longer to add MAX_DIFF_LINES of its own. + (git_repo / "src" / filepaths[1]).write_text( + "value = 1\n" * (gates.error_diff_lines + 2), encoding="utf-8" + ) + + monkeypatch.setattr(gates, "commit_checks", {}) + results = gates.run_preflight() + output = capfd.readouterr().out + + assert ("PHASE: EMPTY COMMIT" in output, results) == ( + True, + { + "pass": [], + "fail": [ + ( + f"{gates.error_diff_lines + 1} lines modified. WARN at 75% " + f"{WARNING_THRESHOLD} lines, ERROR at {gates.error_diff_lines}.\n" + "Suggestion: Refactor bloat, inline helpers, reduce mis-direction, re-use " + "fixtures, cut duplication, slim down if-elif-else blocks." + ) + ], + "warn": [], + }, + ) def test_lint_command_keeps_required_flags() -> None: """The fast lint command remains Ruff's fixing-aware repository-wide check.""" - command = gate.COMMIT_CHECKS["lint"] - - assert command[:2] == ["ruff", "check"] - assert "--show-fixes" in command - assert command[-1] == "." + command = gates.commit_checks["lint"] + assert command == ["ruff", "check", "--no-cache", "--show-fixes", "."] def test_type_check_keeps_machine_readable_output() -> None: """Pyright retains stable JSON output for callers that parse its diagnostics.""" - command = gate.FULL_CHECKS["types"] + command = gates.full_checks["types"] assert command[0] == "pyright" assert "--outputjson" in command @@ -487,7 +710,7 @@ def test_type_check_keeps_machine_readable_output() -> None: def test_security_scan_keeps_blocking_rules() -> None: """Semgrep stays blocking, scans the repository, and includes code and secret rules.""" - command = gate.FULL_CHECKS["security"] + command = gates.full_checks["security"] configs = [command[index + 1] for index, item in enumerate(command[:-1]) if item == "--config"] assert command[:2] == ["semgrep", "scan"] @@ -499,7 +722,7 @@ def test_security_scan_keeps_blocking_rules() -> None: def test_pytest_gate_keeps_full_coverage_threshold() -> None: """The configured test gate continues to require complete measured coverage.""" - command = gate.FULL_CHECKS["pytest"] + command = gates.full_checks["pytest"] assert {"--cov", "--cov-report=term-missing", "--cov-fail-under=100"} <= set(command) @@ -515,9 +738,9 @@ def test_preflight_flags_preferences_break_under_loop( monkeypatch.setattr(gate, "prefs", recorder) source = "def _bad(*args):\n transform = lambda item: item\n return transform(*args)\n" stage(git_repo, "src/mod.py", source) - fake_popen(monkeypatch, fails=[gate.COMMIT_CHECKS["lint"]]) + fake_popen(monkeypatch, fails=[gates.commit_checks["lint"]]) - result = gate.run_preflight() + result = gates.run_preflight() assert { "preferences": recorder.call_args_list, @@ -531,7 +754,6 @@ def test_preflight_flags_preferences_break_under_loop( "fail": [ "lint", ( - "problems:\n" "src/mod.py:1: Name '_bad' starts with underscore\n" "src/mod.py:1: '*args'/'**kwargs' hide the function signature, use explicit parameters\n" "src/mod.py:2: Lambda found hurting readability and adding complexity.\n" @@ -545,67 +767,62 @@ def test_preflight_flags_preferences_break_under_loop( } -def test_preflight_preferences_read_one_file_at_a_time( +def test_preferences_read_each_staged_blob_not_the_worktree( monkeypatch: pytest.MonkeyPatch, git_repo: Path ) -> None: - """Preferences receive each staged Python file and its index content separately.""" + """Preferences parse each staged Python blob and ignore later working-tree edits.""" monkeypatch.setenv("RALPH_LOOP", "1") monkeypatch.chdir(git_repo) - recorder = Mock(side_effect=["src/a.py:1: first violation", ""]) - monkeypatch.setattr(gate, "prefs", recorder) - stage(git_repo, "src/a.py", "a = 1\n") + monkeypatch.setattr(gates, "commit_checks", {}) + stage(git_repo, "src/a.py", "_staged_only = 1\n") stage(git_repo, "src/b.py", "b = 2\n") - (git_repo / "src/a.py").write_text("_working_tree_only = 3\n", encoding="utf-8") + (git_repo / "src/a.py").write_text("working_tree_clean = 3\n", encoding="utf-8") (git_repo / "src/b.py").write_text("_also_not_staged = 4\n", encoding="utf-8") - fake_popen(monkeypatch) - result = gate.run_preflight() - - assert { - "preferences": recorder.call_args_list, - "result": result, - "working_sources": [ + assert ( + gates.run_preflight(), + [ (git_repo / "src/a.py").read_text(encoding="utf-8"), (git_repo / "src/b.py").read_text(encoding="utf-8"), ], - } == { - "preferences": [call("src/a.py", "a = 1\n"), call("src/b.py", "b = 2\n")], - "result": { - "pass": list(gate.COMMIT_CHECKS), - "fail": ["problems:\nsrc/a.py:1: first violation"], - "warn": [], - }, - "working_sources": ["_working_tree_only = 3\n", "_also_not_staged = 4\n"], - } + ) == ( + {"pass": [], "fail": ["src/a.py:1: Name '_staged_only' starts with underscore"], "warn": []}, + ["working_tree_clean = 3\n", "_also_not_staged = 4\n"], + ) -def test_check_for_bad_patterns_appends_a_preference_violation( +def test_bad_patterns_and_preferences_report_separate_violations( monkeypatch: pytest.MonkeyPatch, git_repo: Path ) -> None: - """A staged Python preference violation is included in the returned problems.""" + """The scanners accept diff lines and file paths while preserving both violations.""" monkeypatch.setenv("RALPH_LOOP", "1") monkeypatch.chdir(git_repo) + monkeypatch.setattr(gates, "commit_checks", {}) assert gate.prefs is not None recorder = Mock(wraps=gate.prefs) monkeypatch.setattr(gate, "prefs", recorder) source = "def _bad(*args):\n return 1 # noqa\n" stage(git_repo, "src/mod.py", source) - problems = gate.check_for_bad_patterns() + results = gates.run_preflight() assert { "preferences": recorder.call_args_list, - "problems": problems, + "results": results, "staged_paths": gate.run_git(["diff", "--cached", "--name-only"], git_repo).splitlines(), } == { "preferences": [call("src/mod.py", source)], - "problems": [ - "'# noqa' line: return 1 # noqa", - ( - "src/mod.py:1: Name '_bad' starts with underscore\n" - "src/mod.py:1: '*args'/'**kwargs' hide the function signature, use explicit parameters" - ), - ], + "results": { + "pass": [], + "fail": [ + "'# noqa' line: return 1 # noqa", + ( + "src/mod.py:1: Name '_bad' starts with underscore\n" + "src/mod.py:1: '*args'/'**kwargs' hide the function signature, use explicit parameters" + ), + ], + "warn": [], + }, "staged_paths": ["src/mod.py"], } @@ -623,52 +840,47 @@ def test_preferences_only_ever_read_python( ) -> None: """Non-Python is never parsed as Python, whether by suffix, by deletion, or by project language.""" monkeypatch.setenv("RALPH_LOOP", "1") + monkeypatch.setattr(gates, "commit_checks", {}) recorder = Mock(return_value="unexpected preference call") monkeypatch.setattr(gate, "prefs", recorder) stage(git_repo, name, content) - assert gate.run_non_human_checks() == [] + assert gates.run_preflight() == {"pass": [], "fail": [], "warn": []} recorder.assert_not_called() - monkeypatch.setattr(gate, "languages", ["rb"]) + monkeypatch.setattr(gates, "languages", ("rb",)) stage(git_repo, "app.rb", "def foo; end\n") - assert gate.check_for_bad_patterns() == [] + assert gates.run_preflight() == {"pass": [], "fail": [], "warn": []} recorder.assert_not_called() - monkeypatch.setattr(gate, "languages", ["py"]) + monkeypatch.setattr(gates, "languages", ("py",)) stage(git_repo, "src/gone.py", "value = 1\n") gate.run_git(["commit", "-q", "-m", "add gone"], git_repo) gate.run_git(["rm", "-q", "src/gone.py"], git_repo) - assert gate.run_non_human_checks() == [] + assert gates.run_preflight() == {"pass": [], "fail": [], "warn": []} recorder.assert_not_called() def test_deleting_preferences_disables_the_check_not_the_gate( monkeypatch: pytest.MonkeyPatch, git_repo: Path ) -> None: - """preferences.py is meant to be deletable, so the gate keeps running without it.""" + """A missing preferences module imports cleanly and disables only that optional check.""" monkeypatch.setenv("RALPH_LOOP", "1") + monkeypatch.setattr(gates, "commit_checks", {}) stage(git_repo, "src/mod.py", "_bad = 1\n") - fake_popen(monkeypatch) - - assert gate.run_preflight() == { - "pass": list(gate.COMMIT_CHECKS), - "fail": ["problems:\nsrc/mod.py:1: Name '_bad' starts with underscore"], - "warn": [], - } - assert gate.run_gate() == { - "pass": list(gate.FULL_CHECKS), - "fail": ["problems:\nsrc/mod.py:1: Name '_bad' starts with underscore"], + assert gates.run_preflight() == { + "pass": [], + "fail": ["src/mod.py:1: Name '_bad' starts with underscore"], "warn": [], } + + with monkeypatch.context() as missing_preferences: + missing_preferences.setitem(sys.modules, "preferences.preferences", None) + imported = runpy.run_path(str(REPO_ROOT / "harness" / "gate.py")) + assert imported["prefs"] is None + monkeypatch.setattr(gate, "prefs", None) - monkeypatch.setitem(sys.modules, "preferences.preferences", None) - importlib.reload(gate) - monkeypatch.setattr(gate, "REPO_ROOT", git_repo) + monkeypatch.setattr(gates, "full_checks", {}) assert gate.prefs is None - assert gate.run_preflight() == {"pass": list(gate.COMMIT_CHECKS), "fail": [], "warn": []} - assert gate.run_gate() == {"pass": list(gate.FULL_CHECKS), "fail": [], "warn": []} - - monkeypatch.undo() - importlib.reload(gate) - assert gate.prefs is not None + assert gates.run_preflight() == {"pass": [], "fail": [], "warn": []} + assert gates.run_gate() == {"pass": [], "fail": [], "warn": []} diff --git a/harness/tests/test_properties.py b/harness/tests/test_properties.py index 9dc8d1a..a905e9d 100644 --- a/harness/tests/test_properties.py +++ b/harness/tests/test_properties.py @@ -20,40 +20,20 @@ from __future__ import annotations -from collections.abc import Iterator from pathlib import Path import pytest from hypothesis import example, given, settings, strategies from harness import gate - - -def seed_repo(directory: Path) -> Path: - """Create a temp git repository with one commit and point gate's git calls at it.""" - gate.run_git(["init", "-q"], directory) - gate.run_git(["config", "user.email", "harness@test.local"], directory) - gate.run_git(["config", "user.name", "harness-test"], directory) - (directory / "README.md").write_text("seed\n", encoding="utf-8") - gate.run_git(["add", "README.md"], directory) - gate.run_git(["commit", "-q", "-m", "seed"], directory) - return directory - - -@pytest.fixture(scope="module") -def scan_repo(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Path]: - """A temp repo shared by the generated examples, since @given cannot take a per-test fixture.""" - repo = seed_repo(tmp_path_factory.mktemp("banned-patterns")) - with pytest.MonkeyPatch.context() as patch: - patch.setattr(gate, "REPO_ROOT", repo) - yield repo +from harness.gate import gates def scan_staged(repo: Path, source: str) -> list[str]: """Stage one file, run the real banned-pattern scan over the real index, then clear the index.""" (repo / "x.py").write_text(source, encoding="utf-8") gate.run_git(["add", "x.py"], repo) - problems = gate.check_for_bad_patterns() + problems = gates.run_preflight()["fail"] gate.run_git(["reset", "-q"], repo) return problems @@ -68,7 +48,7 @@ def recased_pattern(draw: strategies.DrawFn) -> tuple[str, str]: Returns: (pattern, recased) where recased differs from pattern only in the case of its letters. """ - pattern = draw(strategies.sampled_from(gate.FORBIDDEN_PATTERNS)) + pattern = draw(strategies.sampled_from(gates.forbidden_patterns)) # Recase each alphabetic character independently; symbols (e.g. in '--no-verify') pass through. recased = "".join( draw(strategies.sampled_from([char.lower(), char.upper()])) if char.isalpha() else char @@ -77,7 +57,7 @@ def recased_pattern(draw: strategies.DrawFn) -> tuple[str, str]: return pattern, recased -@settings(max_examples=50) +@settings(max_examples=50, deadline=None) @given(case=recased_pattern()) @example(case=("# noqa", "# noqa")) # lowercase-alpha pattern @example(case=("--no-verify", "--NO-verify")) # symbol-heavy pattern @@ -98,37 +78,49 @@ def test_mixed_case_forbidden_entry_still_matches( lowercase. A mixed-case entry has to match too, which it only does because the scan casefolds the pattern as well as the line. """ - monkeypatch.setattr(gate, "FORBIDDEN_PATTERNS", [pattern_in_toml]) + monkeypatch.setenv("RALPH_LOOP", "1") + monkeypatch.setattr(gates, "commit_checks", {}) + monkeypatch.setattr(gates, "forbidden_patterns", (pattern_in_toml,)) problems = scan_staged(git_repo, "value = 1 # hookspath\n") assert any(problem.startswith(f"'{pattern_in_toml}' line:") for problem in problems) -def test_banned_pattern_ignores_the_diff_file_header(git_repo: Path) -> None: +def test_banned_pattern_ignores_the_diff_file_header(monkeypatch: pytest.MonkeyPatch, git_repo: Path) -> None: """A file whose name contains a forbidden pattern puts that pattern in the diff's '+++ b/...' header. The header is not code an agent added, so it must not be reported. """ + monkeypatch.setenv("RALPH_LOOP", "1") + monkeypatch.setattr(gates, "commit_checks", {}) (git_repo / "noqa_helpers.py").write_text("value = 1\n", encoding="utf-8") gate.run_git(["add", "noqa_helpers.py"], git_repo) - assert gate.check_for_bad_patterns() == [] + assert gates.run_preflight() == {"pass": [], "fail": [], "warn": []} -def test_banned_pattern_ignores_a_removed_line(git_repo: Path) -> None: +def test_banned_pattern_ignores_a_removed_line(monkeypatch: pytest.MonkeyPatch, git_repo: Path) -> None: """Deleting a line that carries an escape hatch is the fix, not the offense, so a removed '-' line is never reported. """ + monkeypatch.setenv("RALPH_LOOP", "1") + monkeypatch.setattr(gates, "commit_checks", {}) scan_staged(git_repo, "value = 1 # noqa\n") gate.run_git(["add", "x.py"], git_repo) gate.run_git(["commit", "-q", "-m", "seed noqa"], git_repo) assert scan_staged(git_repo, "value = 1\n") == [] -def test_casefold_colliding_forbidden_paths_are_both_ejected(git_repo: Path) -> None: +def test_casefold_colliding_forbidden_paths_are_both_ejected( + monkeypatch: pytest.MonkeyPatch, git_repo: Path +) -> None: """Two forbidden paths differing only in case must both be unstaged. A case-insensitive filesystem cannot hold both as files, so they go into the index directly; neither may slip through. """ colliding = ["harness/Gate.py", "harness/gate.py"] + monkeypatch.setenv("RALPH_LOOP", "1") blob = gate.run_git(["hash-object", "-w", "README.md"], git_repo).strip() for path in colliding: gate.run_git(["update-index", "--add", "--cacheinfo", f"100644,{blob},{path}"], git_repo) - gate.run_non_human_checks() - assert gate.run_git(["diff", "--cached", "--name-only"]).splitlines() == [] + monkeypatch.setattr(gates, "commit_checks", {}) + assert ( + gates.run_preflight(), + gate.run_git(["diff", "--cached", "--name-only"]).splitlines(), + ) == ({"pass": [], "fail": [], "warn": []}, []) diff --git a/harness/tests/test_ralph.py b/harness/tests/test_ralph.py index cc5923d..cae7e8a 100644 --- a/harness/tests/test_ralph.py +++ b/harness/tests/test_ralph.py @@ -8,6 +8,7 @@ from __future__ import annotations +import json import os import shutil import subprocess @@ -78,7 +79,8 @@ def test_loop_passes_prompt_and_completes(tmp_path: Path) -> None: assert (tmp_path / "received-prompt.txt").read_text(encoding="utf-8") == ( "do the most important thing\n\nRALPH_ITERATION=1/1\n" ) - assert "completed 1 iteration(s)" in result.stderr + events = [json.loads(line) for line in result.stdout.splitlines()] + assert (events[0]["iteration"], events[-1]["completed"]) == (1, 1) def test_default_iterations_are_two_when_omitted(tmp_path: Path) -> None: @@ -87,9 +89,9 @@ def test_default_iterations_are_two_when_omitted(tmp_path: Path) -> None: write_executable(worker, "#!/bin/sh\nexit 0\n") result = run_ralph(tmp_path, worker, []) assert result.returncode == 0 - assert "iteration 1/2" in result.stderr - assert "iteration 2/2" in result.stderr - assert "completed 2 iteration(s)" in result.stderr + events = [json.loads(line) for line in result.stdout.splitlines()] + assert [event.get("iteration") for event in events] == [1, 2, None] + assert events[-1]["completed"] == 2 # The fake timeout recorded its duration arg, so we can prove the 20-min default without waiting. assert (tmp_path / "timeout-secs").read_text(encoding="utf-8") == "1200\n1200\n" @@ -100,9 +102,9 @@ def test_nonzero_worker_exit_propagates_and_stops(tmp_path: Path) -> None: write_executable(worker, "#!/bin/sh\nexit 7\n") result = run_ralph(tmp_path, worker, ["2", "1"]) assert result.returncode == 7 - assert "iteration 1/2" in result.stderr - assert "iteration 2/2" not in result.stderr - assert "completed" not in result.stderr + events = [json.loads(line) for line in result.stdout.splitlines()] + assert [event["iteration"] for event in events] == [1] + assert all("completed" not in event for event in events) def test_timeout_propagates_and_stops(tmp_path: Path) -> None: diff --git a/harness/tests/test_ralph_ps1.py b/harness/tests/test_ralph_ps1.py index 2e16da1..4dc2976 100644 --- a/harness/tests/test_ralph_ps1.py +++ b/harness/tests/test_ralph_ps1.py @@ -83,6 +83,11 @@ def test_defaults_run_twice_and_pass_prompt_marker_and_environment(tmp_path: Pat ) assert (tmp_path / f"loop-{iteration}.txt").read_text(encoding="utf-8") == "1" assert "completed 2 iteration(s)" in result.stderr + # stdout is the run receipt `harness run` saves as .jsonl, so it must match ralph.sh's contract + events = [json.loads(line) for line in result.stdout.splitlines()] + assert [event["type"] for event in events] == ["ralph", "ralph", "ralph"] + assert [event.get("iteration") for event in events] == [1, 2, None] + assert events[-1]["completed"] == 2 def test_explicit_one_iteration_completes(tmp_path: Path) -> None: @@ -133,6 +138,7 @@ def test_nonzero_worker_exit_propagates_and_stops(tmp_path: Path) -> None: assert "iteration 1/2" in result.stderr assert "iteration 2/2" not in result.stderr assert "completed" not in result.stderr + assert "completed" not in result.stdout # a failed loop never writes a completion receipt def test_fractional_timeout_returns_124_and_stops_process_tree(tmp_path: Path) -> None: @@ -146,3 +152,4 @@ def test_fractional_timeout_returns_124_and_stops_process_tree(tmp_path: Path) - assert "iteration 1/2" in result.stderr assert "iteration 2/2" not in result.stderr assert "completed" not in result.stderr + assert "completed" not in result.stdout # a timed-out loop never writes a completion receipt diff --git a/mutation-score.json b/mutation-score.json new file mode 100644 index 0000000..7d79926 --- /dev/null +++ b/mutation-score.json @@ -0,0 +1,6 @@ +{ + "schemaVersion": 1, + "label": "mutation", + "message": "83.3%", + "color": "#55ff00" +} diff --git a/mutation/check_mutmut.py b/mutation/check_mutmut.py index 534d1ff..b90041b 100644 --- a/mutation/check_mutmut.py +++ b/mutation/check_mutmut.py @@ -48,6 +48,7 @@ from __future__ import annotations import json +import os from pathlib import Path import typer @@ -55,7 +56,9 @@ from rich.console import Console from rich.table import Table -console = Console(force_terminal=True) +console = Console(force_terminal=True, color_system=None if os.environ.get("RALPH_LOOP") else "256") + +MINIMUM_MUTATION_SCORE = 62.0 JsonDocument = dict[str, object] | list[object] | str | int | float | bool | None @@ -71,7 +74,7 @@ def analyze_mutmut_report(file_path: str = "mutants/mutmut-cicd-stats.json") -> Raises: JSONDecodeError: If the report does not contain valid JSON. - Exit: If the report file does not exist. + Exit: If the report file does not exist or the mutation score is below the minimum. """ if not Path(file_path).exists(): rprint(rf"[red]Error: Mutmut JSON report not found at [\]'{file_path}'") @@ -96,8 +99,11 @@ def analyze_mutmut_report(file_path: str = "mutants/mutmut-cicd-stats.json") -> table = Table(title="\n[cyan2]MUTMUT MUTATION RESULTS[/]\n", box=None, padding=(0, 2)) for stat, result in data.items(): table.add_row(f"[turquoise] {stat}[/]", f"[blue] {result}[/]") - table.add_row(f"[bold italic turquoise] MUTATION SCORE: [/][bold italic blue]{mutation_score}[/]") + table.add_row(f"[bold italic cyan2] MUTATION SCORE: [/][bold italic yellow2]{mutation_score}[/]") console.print(table, justify="center") + if mutation_score < MINIMUM_MUTATION_SCORE: + raise typer.Exit(code=1) + return mutation_score diff --git a/pyproject.toml b/pyproject.toml index 27d863e..5a2c3ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,8 +1,9 @@ [project] name = "harness" version = "0.1.0" -requires-python = ">=3.11" +requires-python = ">=3.10,<4" dependencies = [ + "mutmut>=3.7.0", "packaging", "tomlkit", "typer", @@ -41,12 +42,16 @@ packages = [{ include = "harness" }] # CI runs `harness gate`, so it uses the same commands. # Humans own this file (it is agent-forbidden below). # ============================================================================== +[tool.harness] +languages = ["py"] +error_diff_lines = 400 # ~90th percentile of pull requests, 200 LOC generally ok + # argv preset per agent for `harness run ` [tool.harness.agents] claude = [ "claude", "--model", "opus", - "--permission-mode", "acceptEdits", + "--permission-mode", "auto", # "--bare" # for one-shot minimal run: skips MCP, hooks, plugins, CLAUDE.md, reduced startup, and sets # CLAUDE_CODE_SIMPLE + uses CLAUDE_CODE_OAUTH_TOKEN (ANTHROPIC_API_KEY billed), no .claude log "--no-session-persistence", # no-save session data good for disposable auto tasks @@ -67,7 +72,7 @@ agy = [ "--model", "gemini-3.5-flash-low", "--dangerously-skip-permissions", "--add-dir", ".", - "--log-file", "{log_file}", + "--log-file", "{log_path}", "--prompt", "-" ] copilot = [ @@ -76,10 +81,6 @@ copilot = [ "--output-format", "json", "--stream", "on", "--allow-all", "--no-ask-user", ] - -[tool.harness] -languages = ["py"] - # EXAMPLE OF WHAT Javascript checks might be # # [tool.harness.preflight] @@ -101,6 +102,7 @@ format = ["ruff", "format", "--no-cache", "--check"] complexipy = ["complexipy", "."] [tool.harness.gate] +audit = ["pip-audit", ".", "--strict"] security = [ "semgrep", "scan", "--error", # exit nonzero on findings so the gate blocks the commit, not just reports @@ -119,10 +121,8 @@ pytest = [ DIRS = ["harness/", ".githooks/", ".github/", "preferences/", "tests/preferences/", ".git/"] FILES = [ "agents.md", - "pyproject.toml", "docs/prompt.md", - "docs/plan.md", # delete or comment out if you want agents to manage the vision - "uv.lock", + "docs/plan.md", # delete/comment line if you want agents to manage the core plan # tooling/config files that would weaken checks in this file "pytest.ini", "tox.ini", @@ -134,6 +134,11 @@ FILES = [ "pyrightconfig.json", ".pylintrc", ".gitmodules", + ".gitattributes", + "pyproject.toml", # agents will claim inability to add dependencies with this forbidden + "requirements.txt", + "uv.lock", # if *.lock remains, agents will leave it uncommitted if file changes + "poetry.lock" ] PATTERNS = [ "# noqa", @@ -207,12 +212,9 @@ skip_covered = false # Same behavior for older coverage config rea # ============================================================================== [tool.mutmut] max-children = 2 -source_paths = ["preferences/preferences.py", "harness/gate.py"] # what to mutate -also_copy = ["harness", ".githooks"] # not mutated paths, copied for imports -do_not_mutate = ["harness/tests/*"] # mutmut can't tell what's a test file -pytest_add_cli_args_test_selection = [ - "harness/tests/test_properties.py", "harness/tests/test_gate.py", -] +source_paths = ["preferences", "src", "harness"] # what to mutate +also_copy = ["mutation", ".githooks"] # not mutated paths, copied for imports +do_not_mutate = ["harness/tests/*"] # ============================================================================== # Complexipy Configuration @@ -253,6 +255,7 @@ parameter_documentation.accept-no-param-doc = false # Require Args docs when Py reports.reports = "yes" # Print detailed Pylint reports so humans see why score changed reports.score = true # Keep Pylint's score visible as a coarse trend signal "messages control".enable = ["F", "I", "R0022", "missing-param-doc"] # Fatal/info, stale options, param docs +design.max-attributes = 10 # ============================================================================== # Unified Ruff Configuration diff --git a/requirements.txt b/requirements.txt index 4268e2f..e013df8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,11 @@ complexipy hypothesis +mutmut packaging +pip-audit pylint pyright +pytest-xdist pytest pytest-cov ruff diff --git a/tests/mutation/test_check_mutmut.py b/tests/mutation/test_check_mutmut.py index 9b1b79a..026b058 100644 --- a/tests/mutation/test_check_mutmut.py +++ b/tests/mutation/test_check_mutmut.py @@ -11,7 +11,7 @@ import typer from click import unstyle -from mutation.check_mutmut import analyze_mutmut_report +from mutation.check_mutmut import MINIMUM_MUTATION_SCORE, analyze_mutmut_report def test_report_with_timeout_passes_and_renders(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: @@ -32,21 +32,36 @@ def test_report_with_timeout_passes_and_renders(tmp_path: Path, capsys: pytest.C assert "MUTATION SCORE: 100.0" in output -def test_actionable_result_fails(tmp_path: Path) -> None: - """A survived mutant is included in the report analysis.""" +def test_report_enforces_threshold(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + """Mutation scores at or above the minimum pass, while lower scores fail.""" data = json.loads(Path(__file__).with_name("mutmut-cicd-stats.json").read_text(encoding="utf-8")) data["survived"] = 1 data["total"] += 1 report = tmp_path / "mutmut-cicd-stats.json" report.write_text(json.dumps(data), encoding="utf-8") - mutation_score = analyze_mutmut_report(report) - assert mutation_score >= 60.0 + mutation_score = analyze_mutmut_report(str(report)) + assert MINIMUM_MUTATION_SCORE <= mutation_score < 100.0 + + passing_report = {"killed": MINIMUM_MUTATION_SCORE, "timeout": 0, "total": 100, "skipped": 0} + report.write_text(json.dumps(passing_report), encoding="utf-8") + assert analyze_mutmut_report(str(report)) == pytest.approx(MINIMUM_MUTATION_SCORE) + + failing_score = MINIMUM_MUTATION_SCORE - 1 + report.write_text( + json.dumps({"killed": failing_score, "timeout": 0, "total": 100, "skipped": 0}), encoding="utf-8" + ) + with pytest.raises(typer.Exit) as exc_info: + analyze_mutmut_report(str(report)) + + assert exc_info.value.exit_code == 1 + output = " ".join(unstyle(capsys.readouterr().out).split()) + assert f"MUTATION SCORE: {failing_score}" in output def test_missing_report_fails(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: """A missing export errors CI.""" with pytest.raises(typer.Exit) as exc_info: - analyze_mutmut_report(tmp_path / "missing.json") + analyze_mutmut_report(str(tmp_path / "missing.json")) assert exc_info.value.exit_code == 1 assert "Error: Mutmut JSON report not found" in capsys.readouterr().out @@ -58,4 +73,4 @@ def test_malformed_report_fails(tmp_path: Path) -> None: report.write_text("{", encoding="utf-8") with pytest.raises(json.JSONDecodeError): - analyze_mutmut_report(report) + analyze_mutmut_report(str(report)) diff --git a/tests/preferences/test_preferences.py b/tests/preferences/test_preferences.py index 1121f07..8b4c7dc 100644 --- a/tests/preferences/test_preferences.py +++ b/tests/preferences/test_preferences.py @@ -48,13 +48,14 @@ def test_every_check_shaped_function_is_registered() -> None: """Every check-shaped function in preferences.py (one param, returns `str | None`) must be in CHECKS. Catches a check that is defined but never wired up -- e.g. dropping `function_argument_assignment_underscore_lead` from the registry would silently stop enforcing it. - Helpers like `starless_literal` (returns bool) and `preferences_violations` (two args) are excluded. + Helpers like `preferences_violations` (two args) and mutmut's generated clones are excluded. """ registered = set(CHECKS.values()) unregistered = [ name for name, fn in inspect.getmembers(preferences, inspect.isfunction) if fn.__module__ == preferences.__name__ + and "mutmut" not in name # mutmut clones every check; only the trampoline keeps the name and list(inspect.signature(fn).parameters) == ["node"] and fn.__annotations__.get("return") == "str | None" and fn not in registered