diff --git a/docs/CLI.md b/docs/CLI.md index 06e2d61..2160ee3 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -20,6 +20,7 @@ Day-to-day posture, status, and review commands. | `doberman doctor` | Read-only health self-check; exits non-zero if a critical check fails. | `--path`/`-p`, `--json` | | `doberman policy-history` | Append-only policy-change ledger, newest first. | `--last`/`-n`, `--path`/`-p`, `--json` | | `doberman log` | Recent redacted decision log, newest first. | `--last`/`-n`, `--path`/`-p`, `--jsonl` | +| `doberman decision-log-prune` | Delete resolved decisions by age and/or retained-row budget. Never touches pending AUTH rows or the policy-change ledger. | `--older-than-days`, `--max-rows`, `--path`/`-p` | | `doberman tui` | Interactive decision log with a plain-language "why" panel. Needs the `tui` extra. | `--path`/`-p` | | `doberman dash` | Localhost-only dashboard: live decision feed, stats, and an AUTH approve/deny queue. Needs the `dash` extra. | `--port`, `--path`/`-p` | | `doberman demo` | Scripted attack reel through the real decision engine. Nothing runs against a real tool or downstream server. | `--path`/`-p`, `--mode`, `--fast` | @@ -173,6 +174,8 @@ Code `2` is reserved for input-validation failures that could be caught before a | `demo` | `1` | Invalid mode name, or a scenario did not match its expected outcome. | | `memory reset` | `1` | No possession factor enrolled, gate denied, or the DB reset failed. | | `memory prune` | `1` | The DB prune operation failed. | +| `decision-log-prune` | `2` | Neither `--older-than-days` nor `--max-rows` was provided. | +| `decision-log-prune` | `1` | The DB prune operation failed. | | `uninstall` | `1` | No possession factor enrolled, confirmation declined, name mismatch, gate denied, or some items were not removed. | Commands not listed (`scan`, `review`, `status`, `log`, `policy-history`, `install-hooks`, `uninstall-hooks`, `session-summary`, `version`, `memory`, `setup`, `hook pre`/`post`/`openclaw`/`codex-pre`) exit `0` on success and rely on Typer's default handler to return `1` on an unhandled exception; they have no `typer.Exit(code=...)` call sites of their own. diff --git a/src/doberman/cli/main.py b/src/doberman/cli/main.py index 2cd2a2d..34a7d23 100644 --- a/src/doberman/cli/main.py +++ b/src/doberman/cli/main.py @@ -56,7 +56,7 @@ from doberman.policy.preferences import DIMENSIONS, preset_name from doberman.render import verdict_label, verdict_label_str from doberman.storage.db import active_elevations, grant_elevation, revoke_elevation -from doberman.storage.log import memory_summary, read_decisions +from doberman.storage.log import memory_summary, prune_decisions, read_decisions from doberman.storage.memory import prune_stale_entities, reset_memory from doberman.storage.taint import clear_taint, entity_scope, read_taint from doberman.storage.tool_pins import approve_pin @@ -1217,7 +1217,14 @@ def tune( report = build_friction_report(rows) if json_out: - typer.echo(json.dumps({**report, "proposals": proposals}, sort_keys=True, default=str)) + typer.echo( + json.dumps( + {**report, "proposals": proposals}, + sort_keys=True, + separators=(",", ":"), + default=str, + ) + ) return if not rows: @@ -1635,6 +1642,42 @@ def policy_history( ) +@app.command("decision-log-prune", rich_help_panel="Daily") +def decision_log_prune( + older_than_days: int | None = typer.Option( + None, + "--older-than-days", + min=1, + help="Delete resolved decisions whose timestamp is this many days old or older.", + ), + max_rows: int | None = typer.Option( + None, + "--max-rows", + min=0, + help="Retain at most this many newest resolved decisions; delete the rest.", + ), + path: str = typer.Option(".", "--path", "-p", help="Repository root."), +) -> None: + """Prune resolved decision rows by age and/or retained-row budget. + + A maintenance operation outside the decision path. It never touches pending + AUTH rows and never modifies the append-only policy-change ledger. + """ + if older_than_days is None and max_rows is None: + typer.echo("error: specify --older-than-days and/or --max-rows", err=True) + raise typer.Exit(code=2) + try: + result = asyncio.run( + prune_decisions(path, older_than_days=older_than_days, max_rows=max_rows) + ) + except Exception as exc: # noqa: BLE001 — never report a failed prune as success + typer.echo(f"error: decision-log prune failed: {exc}", err=True) + raise typer.Exit(code=1) from exc + + deleted = result["age_deleted"] + result["overflow_deleted"] + typer.echo(f"Decision log pruned: {deleted} row(s).") + + @app.command("install-hooks", rich_help_panel="Getting started") def install_hooks( global_: bool = typer.Option( diff --git a/src/doberman/storage/log.py b/src/doberman/storage/log.py index f82c537..4fb26eb 100644 --- a/src/doberman/storage/log.py +++ b/src/doberman/storage/log.py @@ -21,7 +21,7 @@ import json import logging -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from pathlib import PurePosixPath from doberman.models import ActionType, Decision, SecurityObject @@ -78,6 +78,14 @@ "final_verdict FROM decisions WHERE session_id = ? ORDER BY id DESC LIMIT ?" ) +# Maintenance deletes only fully-resolved decision rows. Every verdict except +# AUTH is final; an AUTH row remains eligible only when its challenge already +# produced an explicit outcome. A missing auth_result is deliberately kept. +_RESOLVED_DECISIONS_PREDICATE = ( + "(final_verdict <> 'AUTH' AND (auth_result IS NULL " + "OR auth_result IN ('approved', 'denied', 'executed')))" +) +_DELETE_RESOLVED_DECISIONS = "DELETE FROM decisions WHERE " + _RESOLVED_DECISIONS_PREDICATE # noqa: S608 — fixed clause, params bound _DECISION_COLUMNS = [ "id", "ts", @@ -278,6 +286,62 @@ async def record_shadow( logger.warning("shadow log write failed for action %s; continuing", decision.action_id) +async def prune_decisions( + repo_root: str, + *, + older_than_days: int | None = None, + max_rows: int | None = None, + now: datetime | None = None, +) -> dict[str, int]: + """Prune resolved decision-log rows by age and/or a retained row budget. + + This is an operator-initiated maintenance operation, not part of the hot + decision path. It only removes rows whose verdict is final (or whose AUTH + outcome is explicitly denied/approved/executed); unresolved AUTH rows are + never deleted. The append-only ``policy_changes`` ledger is not touched. + + ``older_than_days`` uses the same ISO-8601 timestamp convention as the rest + of storage: exact cutoff stays, one second older goes. With ``max_rows``, + the newest matching rows are retained first, so an old-but-unresolved row + cannot displace a newer resolved row from the budget. + + Returns counts only and raises on storage errors rather than reporting a + partial delete as successful. + """ + if older_than_days is None and max_rows is None: + raise ValueError("specify --older-than-days and/or --max-rows") + if older_than_days is not None and older_than_days < 1: + raise ValueError("--older-than-days must be at least 1") + if max_rows is not None and max_rows < 0: + raise ValueError("--max-rows cannot be negative") + + age_deleted = 0 + overflow_deleted = 0 + async with open_db(repo_root) as conn: + if older_than_days is not None: + when = now or datetime.now(timezone.utc) + cutoff = (when - timedelta(days=older_than_days)).isoformat() + cur = await conn.execute(_DELETE_RESOLVED_DECISIONS + " AND ts < ?", (cutoff,)) + age_deleted = cur.rowcount + + if max_rows is not None: + query = ( + "DELETE FROM decisions WHERE " # noqa: S608 + + _RESOLVED_DECISIONS_PREDICATE + + " AND id NOT IN (SELECT id FROM decisions WHERE " + + _RESOLVED_DECISIONS_PREDICATE + + " ORDER BY id DESC LIMIT ?)" + ) # noqa: S608 — fixed clauses, params bound + cur = await conn.execute( + query, + (max_rows,), + ) + overflow_deleted = cur.rowcount + + await conn.commit() + return {"age_deleted": age_deleted, "overflow_deleted": overflow_deleted} + + async def read_decisions(repo_root: str, *, limit: int | None = None) -> list[dict]: """Read decision rows, newest first (for ``doberman log``). Fails closed to [].""" from doberman.storage.db import db_path diff --git a/tests/integration/test_cli_views.py b/tests/integration/test_cli_views.py index e2bf12e..649e506 100644 --- a/tests/integration/test_cli_views.py +++ b/tests/integration/test_cli_views.py @@ -67,6 +67,27 @@ def test_log_shows_rows_and_reasons(tmp_path): assert _SECRET not in result.stdout +def test_decision_log_prune_requires_a_policy(tmp_path): + result = runner.invoke(app, ["decision-log-prune", "--path", str(tmp_path)]) + assert result.exit_code == 2 + assert "specify --older-than-days and/or --max-rows" in result.output + + +def test_decision_log_prune_reports_count_only(tmp_path): + root = str(tmp_path) + _seed_auth_secret_read(root) + + result = runner.invoke( + app, + ["decision-log-prune", "--older-than-days", "90", "--max-rows", "1", "--path", root], + ) + + assert result.exit_code == 0, result.output + assert "Decision log pruned: 0 row(s)." in result.output + assert "hmac:abc123" not in result.output # no fingerprint value + assert _SECRET not in result.stdout + + def test_memory_shows_classes_and_counts_only(tmp_path): root = str(tmp_path) _seed_auth_secret_read(root) diff --git a/tests/integration/test_decision_log.py b/tests/integration/test_decision_log.py index 2e3bc6b..d198c64 100644 --- a/tests/integration/test_decision_log.py +++ b/tests/integration/test_decision_log.py @@ -1,7 +1,7 @@ """Slice 8.2 — append-only, redacted decision-log writer (wired into the proxy).""" import inspect -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from doberman.auth.challenge import AuthResult, AuthTier from doberman.models import ( @@ -15,7 +15,12 @@ ) from doberman.proxy import executor from doberman.storage.db import open_db -from doberman.storage.log import read_decisions, recent_session_decisions, record_decision +from doberman.storage.log import ( + prune_decisions, + read_decisions, + recent_session_decisions, + record_decision, +) from .test_proxy_passthrough import proxied_session @@ -80,9 +85,10 @@ def test_writer_has_no_update_or_delete_path_for_decisions(): # decision row (it only INSERTs, and upserts last_seen on fingerprints). import doberman.storage.log as log_module - src = inspect.getsource(log_module) - assert "UPDATE decisions" not in src - assert "DELETE FROM decisions" not in src + for name in ("build_record", "record_decision", "record_shadow"): + source = inspect.getsource(getattr(log_module, name)) + assert "UPDATE decisions" not in source + assert "DELETE FROM decisions" not in source def _decision_and_action(verdict: Verdict, action_id: str) -> tuple[Decision, SecurityObject]: @@ -137,3 +143,78 @@ async def test_recent_session_decisions_fails_closed(): assert await recent_session_decisions(executor.REPO_ROOT, "no-such-session", 10) == [] # No DB has been created at all in this repo root yet. assert await recent_session_decisions(str(executor.REPO_ROOT) + "-missing", "s1", 10) == [] + + +async def _seed_decision(root: str, action_id: str, ts: datetime) -> None: + decision, action = _decision_and_action(Verdict.PASS, action_id) + await record_decision(decision, action, repo_root=root, now=ts) + + +async def _decision_verdicts_and_count(root: str) -> tuple[int, set[str]]: + rows = await read_decisions(root) + return len(rows), {row["final_verdict"] for row in rows} + + +async def test_prune_by_age_keeps_new_resolved_and_never_touches_a_verdict(tmp_path): + root = str(tmp_path) + now = datetime.now(timezone.utc) + await _seed_decision(root, "old", now - timedelta(days=90, seconds=1)) + await _seed_decision(root, "boundary", now - timedelta(days=90)) + await _seed_decision(root, "fresh", now - timedelta(days=1)) + + before_count, before_verdicts = await _decision_verdicts_and_count(root) + assert before_count == 3 + assert before_verdicts == {"PASS"} + + result = await prune_decisions(root, older_than_days=90, now=now) + + rows = await read_decisions(root) + count, verdicts = await _decision_verdicts_and_count(root) + assert result == {"age_deleted": 1, "overflow_deleted": 0} + assert count == 2 + assert {row["action_id"] for row in rows} == {"boundary", "fresh"} + # A decision's persisted verdict is the same whether pruning is due or not. + assert verdicts == {"PASS"} + assert verdicts == before_verdicts + + +async def test_prune_by_max_rows_keeps_newest_resolved_only(tmp_path): + root = str(tmp_path) + now = datetime.now(timezone.utc) + await _seed_decision(root, "old", now - timedelta(days=3)) + await _seed_decision(root, "newer", now - timedelta(days=2)) + await _seed_decision(root, "newest", now - timedelta(days=1)) + + result = await prune_decisions(root, max_rows=2, now=now) + + actions = [row["action_id"] for row in await read_decisions(root)] + assert result == {"age_deleted": 0, "overflow_deleted": 1} + assert set(actions) == {"newer", "newest"} + + +async def test_prune_never_deletes_unresolved_auth(tmp_path): + root = str(tmp_path) + now = datetime.now(timezone.utc) + decision, action = _decision_and_action(Verdict.AUTH, "pending-auth") + await record_decision( + decision, + action, + repo_root=root, + now=now - timedelta(days=365), + ) + + result = await prune_decisions(root, older_than_days=1, max_rows=0, now=now) + + rows = await read_decisions(root) + assert result == {"age_deleted": 0, "overflow_deleted": 0} + assert len(rows) == 1 + assert rows[0]["final_verdict"] == "AUTH" + + +async def test_prune_requires_at_least_one_policy(tmp_path): + raised = False + try: + await prune_decisions(str(tmp_path)) + except ValueError: + raised = True + assert raised diff --git a/tests/unit/test_cli_help.py b/tests/unit/test_cli_help.py index e6329f5..8492619 100644 --- a/tests/unit/test_cli_help.py +++ b/tests/unit/test_cli_help.py @@ -28,6 +28,7 @@ ("doctor",), ("revoke",), ("log",), + ("decision-log-prune",), ("tui",), ("dash",), ("demo",), diff --git a/tests/unit/test_cli_tune_json.py b/tests/unit/test_cli_tune_json.py new file mode 100644 index 0000000..1166ad6 --- /dev/null +++ b/tests/unit/test_cli_tune_json.py @@ -0,0 +1,25 @@ +"""`doberman tune --json` uses the CLI's compact machine-readable contract.""" + +from __future__ import annotations + +import json + +from typer.testing import CliRunner + +from doberman.cli.main import app +from tests.unit.test_friction_tune import _seed_five_approved_migrations + +runner = CliRunner() + + +def test_tune_json_is_parseable_and_compact(tmp_path): + root = str(tmp_path) + _seed_five_approved_migrations(root) + + result = runner.invoke(app, ["tune", "--path", root, "--json"]) + + assert result.exit_code == 0, result.output + payload = json.loads(result.stdout) + assert "proposals" in payload + assert '", "' not in result.stdout + assert '": "' not in result.stdout