From 4d5cbed7fb72a41e36355bbe30c0cf6ca29101ac Mon Sep 17 00:00:00 2001 From: Sebastian Legarraga <64795732+slegarraga@users.noreply.github.com> Date: Tue, 25 Aug 2026 17:49:46 -0400 Subject: [PATCH 1/2] feat(storage): add configurable decision log pruning --- docs/CLI.md | 3 + src/doberman/cli/main.py | 38 ++++++++++- src/doberman/storage/log.py | 66 ++++++++++++++++++- tests/integration/test_cli_views.py | 21 ++++++ tests/integration/test_decision_log.py | 91 ++++++++++++++++++++++++-- tests/unit/test_cli_help.py | 1 + 6 files changed, 213 insertions(+), 7 deletions(-) diff --git a/docs/CLI.md b/docs/CLI.md index 26b4bda9..32ec3874 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -20,6 +20,7 @@ Day-to-day posture, status, and review commands. | `doberman doctor` | Read-only health self-check; exits non-zero if a critical check fails. | `--path`/`-p`, `--json` | | `doberman policy-history` | Append-only policy-change ledger, newest first. | `--last`/`-n`, `--path`/`-p`, `--json` | | `doberman log` | Recent redacted decision log, newest first. | `--last`/`-n`, `--path`/`-p`, `--jsonl` | +| `doberman decision-log-prune` | Delete resolved decisions by age and/or retained-row budget. Never touches pending AUTH rows or the policy-change ledger. | `--older-than-days`, `--max-rows`, `--path`/`-p` | | `doberman tui` | Interactive decision log with a plain-language "why" panel. Needs the `tui` extra. | `--path`/`-p` | | `doberman dash` | Localhost-only dashboard: live decision feed, stats, and an AUTH approve/deny queue. Needs the `dash` extra. | `--port`, `--path`/`-p` | | `doberman demo` | Scripted attack reel through the real decision engine. Nothing runs against a real tool or downstream server. | `--path`/`-p`, `--mode`, `--fast`, `--quiet`/`-q` | @@ -182,6 +183,8 @@ Code `2` is reserved for input-validation failures that could be caught before a | `demo` | `1` | Invalid mode name, or a scenario did not match its expected outcome. | | `memory reset` | `1` | No possession factor enrolled, gate denied, or the DB reset failed. | | `memory prune` | `1` | The DB prune operation failed. | +| `decision-log-prune` | `2` | Neither `--older-than-days` nor `--max-rows` was provided. | +| `decision-log-prune` | `1` | The DB prune operation failed. | | `uninstall` | `1` | No possession factor enrolled, confirmation declined, name mismatch, gate denied, or some items were not removed. | Commands not listed (`scan`, `review`, `status`, `log`, `policy-history`, `install-hooks`, `uninstall-hooks`, `session-summary`, `version`, `memory`, `setup`, `hook pre`/`post`/`openclaw`/`codex-pre`) exit `0` on success and rely on Typer's default handler to return `1` on an unhandled exception; they have no `typer.Exit(code=...)` call sites of their own. diff --git a/src/doberman/cli/main.py b/src/doberman/cli/main.py index c99ab25b..ee724ba7 100644 --- a/src/doberman/cli/main.py +++ b/src/doberman/cli/main.py @@ -65,7 +65,7 @@ from doberman.storage.approval_memory import count_live as count_live_approval_memory from doberman.storage.db import active_elevations, grant_elevation, revoke_elevation from doberman.storage.exclusions import add_exclusion, is_excluded, remove_exclusion -from doberman.storage.log import memory_summary, read_decisions +from doberman.storage.log import memory_summary, prune_decisions, read_decisions from doberman.storage.memory import prune_stale_entities, reset_memory from doberman.storage.taint import clear_taint, entity_scope, read_taint from doberman.storage.tool_pins import approve_pin @@ -1797,6 +1797,42 @@ def policy_history( ) +@app.command("decision-log-prune", rich_help_panel="Daily") +def decision_log_prune( + older_than_days: int | None = typer.Option( + None, + "--older-than-days", + min=1, + help="Delete resolved decisions whose timestamp is this many days old or older.", + ), + max_rows: int | None = typer.Option( + None, + "--max-rows", + min=0, + help="Retain at most this many newest resolved decisions; delete the rest.", + ), + path: str = typer.Option(".", "--path", "-p", help="Repository root."), +) -> None: + """Prune resolved decision rows by age and/or retained-row budget. + + A maintenance operation outside the decision path. It never touches pending + AUTH rows and never modifies the append-only policy-change ledger. + """ + if older_than_days is None and max_rows is None: + typer.echo("error: specify --older-than-days and/or --max-rows", err=True) + raise typer.Exit(code=2) + try: + result = asyncio.run( + prune_decisions(path, older_than_days=older_than_days, max_rows=max_rows) + ) + except Exception as exc: # noqa: BLE001 — never report a failed prune as success + typer.echo(f"error: decision-log prune failed: {exc}", err=True) + raise typer.Exit(code=1) from exc + + deleted = result["age_deleted"] + result["overflow_deleted"] + typer.echo(f"Decision log pruned: {deleted} row(s).") + + @app.command("install-hooks", rich_help_panel="Getting started") def install_hooks( global_: bool = typer.Option( diff --git a/src/doberman/storage/log.py b/src/doberman/storage/log.py index f82c537d..4fb26eb2 100644 --- a/src/doberman/storage/log.py +++ b/src/doberman/storage/log.py @@ -21,7 +21,7 @@ import json import logging -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from pathlib import PurePosixPath from doberman.models import ActionType, Decision, SecurityObject @@ -78,6 +78,14 @@ "final_verdict FROM decisions WHERE session_id = ? ORDER BY id DESC LIMIT ?" ) +# Maintenance deletes only fully-resolved decision rows. Every verdict except +# AUTH is final; an AUTH row remains eligible only when its challenge already +# produced an explicit outcome. A missing auth_result is deliberately kept. +_RESOLVED_DECISIONS_PREDICATE = ( + "(final_verdict <> 'AUTH' AND (auth_result IS NULL " + "OR auth_result IN ('approved', 'denied', 'executed')))" +) +_DELETE_RESOLVED_DECISIONS = "DELETE FROM decisions WHERE " + _RESOLVED_DECISIONS_PREDICATE # noqa: S608 — fixed clause, params bound _DECISION_COLUMNS = [ "id", "ts", @@ -278,6 +286,62 @@ async def record_shadow( logger.warning("shadow log write failed for action %s; continuing", decision.action_id) +async def prune_decisions( + repo_root: str, + *, + older_than_days: int | None = None, + max_rows: int | None = None, + now: datetime | None = None, +) -> dict[str, int]: + """Prune resolved decision-log rows by age and/or a retained row budget. + + This is an operator-initiated maintenance operation, not part of the hot + decision path. It only removes rows whose verdict is final (or whose AUTH + outcome is explicitly denied/approved/executed); unresolved AUTH rows are + never deleted. The append-only ``policy_changes`` ledger is not touched. + + ``older_than_days`` uses the same ISO-8601 timestamp convention as the rest + of storage: exact cutoff stays, one second older goes. With ``max_rows``, + the newest matching rows are retained first, so an old-but-unresolved row + cannot displace a newer resolved row from the budget. + + Returns counts only and raises on storage errors rather than reporting a + partial delete as successful. + """ + if older_than_days is None and max_rows is None: + raise ValueError("specify --older-than-days and/or --max-rows") + if older_than_days is not None and older_than_days < 1: + raise ValueError("--older-than-days must be at least 1") + if max_rows is not None and max_rows < 0: + raise ValueError("--max-rows cannot be negative") + + age_deleted = 0 + overflow_deleted = 0 + async with open_db(repo_root) as conn: + if older_than_days is not None: + when = now or datetime.now(timezone.utc) + cutoff = (when - timedelta(days=older_than_days)).isoformat() + cur = await conn.execute(_DELETE_RESOLVED_DECISIONS + " AND ts < ?", (cutoff,)) + age_deleted = cur.rowcount + + if max_rows is not None: + query = ( + "DELETE FROM decisions WHERE " # noqa: S608 + + _RESOLVED_DECISIONS_PREDICATE + + " AND id NOT IN (SELECT id FROM decisions WHERE " + + _RESOLVED_DECISIONS_PREDICATE + + " ORDER BY id DESC LIMIT ?)" + ) # noqa: S608 — fixed clauses, params bound + cur = await conn.execute( + query, + (max_rows,), + ) + overflow_deleted = cur.rowcount + + await conn.commit() + return {"age_deleted": age_deleted, "overflow_deleted": overflow_deleted} + + async def read_decisions(repo_root: str, *, limit: int | None = None) -> list[dict]: """Read decision rows, newest first (for ``doberman log``). Fails closed to [].""" from doberman.storage.db import db_path diff --git a/tests/integration/test_cli_views.py b/tests/integration/test_cli_views.py index e2bf12e6..649e5069 100644 --- a/tests/integration/test_cli_views.py +++ b/tests/integration/test_cli_views.py @@ -67,6 +67,27 @@ def test_log_shows_rows_and_reasons(tmp_path): assert _SECRET not in result.stdout +def test_decision_log_prune_requires_a_policy(tmp_path): + result = runner.invoke(app, ["decision-log-prune", "--path", str(tmp_path)]) + assert result.exit_code == 2 + assert "specify --older-than-days and/or --max-rows" in result.output + + +def test_decision_log_prune_reports_count_only(tmp_path): + root = str(tmp_path) + _seed_auth_secret_read(root) + + result = runner.invoke( + app, + ["decision-log-prune", "--older-than-days", "90", "--max-rows", "1", "--path", root], + ) + + assert result.exit_code == 0, result.output + assert "Decision log pruned: 0 row(s)." in result.output + assert "hmac:abc123" not in result.output # no fingerprint value + assert _SECRET not in result.stdout + + def test_memory_shows_classes_and_counts_only(tmp_path): root = str(tmp_path) _seed_auth_secret_read(root) diff --git a/tests/integration/test_decision_log.py b/tests/integration/test_decision_log.py index 2e3bc6b6..d198c642 100644 --- a/tests/integration/test_decision_log.py +++ b/tests/integration/test_decision_log.py @@ -1,7 +1,7 @@ """Slice 8.2 — append-only, redacted decision-log writer (wired into the proxy).""" import inspect -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from doberman.auth.challenge import AuthResult, AuthTier from doberman.models import ( @@ -15,7 +15,12 @@ ) from doberman.proxy import executor from doberman.storage.db import open_db -from doberman.storage.log import read_decisions, recent_session_decisions, record_decision +from doberman.storage.log import ( + prune_decisions, + read_decisions, + recent_session_decisions, + record_decision, +) from .test_proxy_passthrough import proxied_session @@ -80,9 +85,10 @@ def test_writer_has_no_update_or_delete_path_for_decisions(): # decision row (it only INSERTs, and upserts last_seen on fingerprints). import doberman.storage.log as log_module - src = inspect.getsource(log_module) - assert "UPDATE decisions" not in src - assert "DELETE FROM decisions" not in src + for name in ("build_record", "record_decision", "record_shadow"): + source = inspect.getsource(getattr(log_module, name)) + assert "UPDATE decisions" not in source + assert "DELETE FROM decisions" not in source def _decision_and_action(verdict: Verdict, action_id: str) -> tuple[Decision, SecurityObject]: @@ -137,3 +143,78 @@ async def test_recent_session_decisions_fails_closed(): assert await recent_session_decisions(executor.REPO_ROOT, "no-such-session", 10) == [] # No DB has been created at all in this repo root yet. assert await recent_session_decisions(str(executor.REPO_ROOT) + "-missing", "s1", 10) == [] + + +async def _seed_decision(root: str, action_id: str, ts: datetime) -> None: + decision, action = _decision_and_action(Verdict.PASS, action_id) + await record_decision(decision, action, repo_root=root, now=ts) + + +async def _decision_verdicts_and_count(root: str) -> tuple[int, set[str]]: + rows = await read_decisions(root) + return len(rows), {row["final_verdict"] for row in rows} + + +async def test_prune_by_age_keeps_new_resolved_and_never_touches_a_verdict(tmp_path): + root = str(tmp_path) + now = datetime.now(timezone.utc) + await _seed_decision(root, "old", now - timedelta(days=90, seconds=1)) + await _seed_decision(root, "boundary", now - timedelta(days=90)) + await _seed_decision(root, "fresh", now - timedelta(days=1)) + + before_count, before_verdicts = await _decision_verdicts_and_count(root) + assert before_count == 3 + assert before_verdicts == {"PASS"} + + result = await prune_decisions(root, older_than_days=90, now=now) + + rows = await read_decisions(root) + count, verdicts = await _decision_verdicts_and_count(root) + assert result == {"age_deleted": 1, "overflow_deleted": 0} + assert count == 2 + assert {row["action_id"] for row in rows} == {"boundary", "fresh"} + # A decision's persisted verdict is the same whether pruning is due or not. + assert verdicts == {"PASS"} + assert verdicts == before_verdicts + + +async def test_prune_by_max_rows_keeps_newest_resolved_only(tmp_path): + root = str(tmp_path) + now = datetime.now(timezone.utc) + await _seed_decision(root, "old", now - timedelta(days=3)) + await _seed_decision(root, "newer", now - timedelta(days=2)) + await _seed_decision(root, "newest", now - timedelta(days=1)) + + result = await prune_decisions(root, max_rows=2, now=now) + + actions = [row["action_id"] for row in await read_decisions(root)] + assert result == {"age_deleted": 0, "overflow_deleted": 1} + assert set(actions) == {"newer", "newest"} + + +async def test_prune_never_deletes_unresolved_auth(tmp_path): + root = str(tmp_path) + now = datetime.now(timezone.utc) + decision, action = _decision_and_action(Verdict.AUTH, "pending-auth") + await record_decision( + decision, + action, + repo_root=root, + now=now - timedelta(days=365), + ) + + result = await prune_decisions(root, older_than_days=1, max_rows=0, now=now) + + rows = await read_decisions(root) + assert result == {"age_deleted": 0, "overflow_deleted": 0} + assert len(rows) == 1 + assert rows[0]["final_verdict"] == "AUTH" + + +async def test_prune_requires_at_least_one_policy(tmp_path): + raised = False + try: + await prune_decisions(str(tmp_path)) + except ValueError: + raised = True + assert raised diff --git a/tests/unit/test_cli_help.py b/tests/unit/test_cli_help.py index c361a594..bd1a551f 100644 --- a/tests/unit/test_cli_help.py +++ b/tests/unit/test_cli_help.py @@ -28,6 +28,7 @@ ("doctor",), ("revoke",), ("log",), + ("decision-log-prune",), ("tui",), ("dash",), ("demo",), From f1516314c1d63c38681f79c861fbff308cc4c1ff Mon Sep 17 00:00:00 2001 From: Sebastian Legarraga <64795732+slegarraga@users.noreply.github.com> Date: Thu, 27 Aug 2026 22:30:33 -0400 Subject: [PATCH 2/2] fix(storage): address decision log pruning review --- CHANGELOG.md | 7 +++++++ README.md | 3 +++ src/doberman/engine/rules/commands.py | 1 + src/doberman/storage/log.py | 3 +-- tests/integration/test_decision_log.py | 18 ++++++++++++++++++ tests/unit/test_rule_commands_control_plane.py | 1 + 6 files changed, 31 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87f9addd..855a2fc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ Shipped history for Doberman. Planned work lives on the [roadmap](README.md#road [git log](https://github.com/DobermanCore/Doberman-Core/commits/main) and [releases](https://github.com/DobermanCore/Doberman-Core/releases) (latest: **v0.18.4**, the third friction-reduction patch — a five-minute approval memory for exact repeats, `uninstall --global`, a `doctor` check for dangling hooks, and usage telemetry on by default — atop **v0.18.3**'s tap-to-approve 2FA (a Windows Hello / Touch ID biometric can stand in for the TOTP code) — atop **v0.18.2**'s friction-reduction patch — spurious secret-detector prompts on ordinary ids/paths/UUIDs fixed, and the detector now self-checks and fails closed — atop **v0.18.1**'s README-image docs patch and **v0.18.0**'s security-audit wave — the proxy output-secret gate closed over error and structured/embedded channels, per-user auth state and Windows-separator paths brought under control-plane protection — plus the RAND-aligned guardrail rehaul and the UX/contributor work since 0.17.1). +## Unreleased + +- **Decision-log retention is explicit and fail-safe.** `doberman decision-log-prune` lets an + operator delete resolved rows by age and/or retained-row budget. Pending AUTH challenges and the + append-only policy-change ledger are preserved, and mediated agents cannot invoke the mutating + command through the shell. + ## v0.18.4 — 2026-08-26 > **Friction reduction, part three, and the lights come on.** A repeat of an approved action now diff --git a/README.md b/README.md index e083aac5..79846309 100644 --- a/README.md +++ b/README.md @@ -166,6 +166,9 @@ stays empty. That last assertion is the chokepoint property the whole project ha Doberman's proxy speaks MCP as pinned in `pyproject.toml` (`mcp>=1.27,<2`). Its cross-call protections (taint ledger, read-vs-send fingerprints, decision log) key off repo-local identity, never the protocol session, and are regression-tested stateless. +Operators can bound retained decision rows with `doberman decision-log-prune`; it deletes only +resolved decisions and never pending AUTH rows or the append-only policy-change ledger. See the +[CLI reference](docs/CLI.md) for the age and row-budget options. --- diff --git a/src/doberman/engine/rules/commands.py b/src/doberman/engine/rules/commands.py index a9a3c649..eb9e96d7 100644 --- a/src/doberman/engine/rules/commands.py +++ b/src/doberman/engine/rules/commands.py @@ -72,6 +72,7 @@ "password", "revoke", "memory", + "decision-log-prune", "tools", "approvals", # `tune --accept` grants a standing elevation (a weakening) through the diff --git a/src/doberman/storage/log.py b/src/doberman/storage/log.py index 4fb26eb2..d7d16034 100644 --- a/src/doberman/storage/log.py +++ b/src/doberman/storage/log.py @@ -82,8 +82,7 @@ # AUTH is final; an AUTH row remains eligible only when its challenge already # produced an explicit outcome. A missing auth_result is deliberately kept. _RESOLVED_DECISIONS_PREDICATE = ( - "(final_verdict <> 'AUTH' AND (auth_result IS NULL " - "OR auth_result IN ('approved', 'denied', 'executed')))" + "(final_verdict <> 'AUTH' OR auth_result IN ('approved', 'denied', 'executed'))" ) _DELETE_RESOLVED_DECISIONS = "DELETE FROM decisions WHERE " + _RESOLVED_DECISIONS_PREDICATE # noqa: S608 — fixed clause, params bound _DECISION_COLUMNS = [ diff --git a/tests/integration/test_decision_log.py b/tests/integration/test_decision_log.py index d198c642..018bcbd2 100644 --- a/tests/integration/test_decision_log.py +++ b/tests/integration/test_decision_log.py @@ -211,6 +211,24 @@ async def test_prune_never_deletes_unresolved_auth(tmp_path): assert rows[0]["final_verdict"] == "AUTH" +async def test_prune_deletes_resolved_auth(tmp_path): + root = str(tmp_path) + now = datetime.now(timezone.utc) + decision, action = _decision_and_action(Verdict.AUTH, "approved-auth") + await record_decision( + decision, + action, + repo_root=root, + auth_result="approved", + now=now - timedelta(days=365), + ) + + result = await prune_decisions(root, older_than_days=1, max_rows=0, now=now) + + assert result == {"age_deleted": 1, "overflow_deleted": 0} + assert await read_decisions(root) == [] + + async def test_prune_requires_at_least_one_policy(tmp_path): raised = False try: diff --git a/tests/unit/test_rule_commands_control_plane.py b/tests/unit/test_rule_commands_control_plane.py index 56cf7f5a..8e64ecfe 100644 --- a/tests/unit/test_rule_commands_control_plane.py +++ b/tests/unit/test_rule_commands_control_plane.py @@ -189,6 +189,7 @@ def test_doberman_setup_is_blocked(): "doberman tune --accept abc123", "doberman memory reset", "doberman memory prune --older-than-days 30", + "doberman decision-log-prune --max-rows 0", "doberman tools approve fs_read", "doberman approvals status", "doberman approvals clear",