diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e994a8..ebde91e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,37 @@ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). --- +## [0.38.1] — 2026-08-19 + +### Fixed +- **A stack verification that checked nothing reported `ALL OK`.** The verdict + line was `"ALL OK" if n_ok == len(results)`, which on an empty run is + `0 == 0`: `stack/verify_all.py` printed `=== verdict: ALL OK (0/0) ===` and + **exited 0** for a config declaring no ledger. This is a *vacuous pass* — the + verdict was true only because nothing existed to falsify it, the failure mode + `vacuity detection` has named in formal verification since 1997 ("every + request is eventually granted" is true of a model where no request is ever + made). + + It matters because this verdict is a speech gate: it is the check run before + stating a measured number, and callers chain on its exit code. A partial + honesty signal did exist (the skipped L2 layer printed `⚠️`), but the two + things automation and readers actually act on — the verdict line and the exit + status — both said pass. + + Both verifiers now separate "everything passed" from "nothing was measured", + and exit `2` in the latter case. `stack/verify_self.py` was not reachable this + way from its CLI (its linkage check always reports), but carried the identical + shape and is now guarded too. Relatedly, `mm verify_chain: seals valid` now + prints the number of entries it checked: a green line without its denominator + cannot be told apart from a green line with nothing behind it. + + Found by a sibling lane while running an unrelated gate, reported with a + reproduction. The fix is a port — the same `and total` guard already existed + in an internal tool; what was missing was not the mechanism but its reach. + +--- + ## [0.38.0] — 2026-08-15 ### Fixed diff --git a/measure_mirror/__init__.py b/measure_mirror/__init__.py index d148232..173c525 100644 --- a/measure_mirror/__init__.py +++ b/measure_mirror/__init__.py @@ -48,4 +48,4 @@ "catch_history", "report", "Finding", "recover_resolution", "declared_pre_seal_checks", ] -__version__ = "0.38.0" +__version__ = "0.38.1" diff --git a/pyproject.toml b/pyproject.toml index 4a034ee..fa552e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "measure-mirror" -version = "0.38.0" +version = "0.38.1" description = "Catch AI evaluation illusions — false positives and false negatives — automatically." readme = "README.md" requires-python = ">=3.10" diff --git a/stack/verify_all.py b/stack/verify_all.py index abc0960..4a724a2 100644 --- a/stack/verify_all.py +++ b/stack/verify_all.py @@ -81,10 +81,19 @@ def report(level, layer, name, msg): print(f"{WARN} [L2 witness] (skipped) — no witness ledger configured; stack degrades to " "measure-mirror self-verify (case-study witness ledger is private, see honesty box)") - n_ok = sum(results) - verdict = "ALL OK" if n_ok == len(results) else "FAILURES PRESENT" - print(f"=== verdict: {verdict} ({n_ok}/{len(results)}) ===") - raise SystemExit(0 if n_ok == len(results) else 1) + n_ok, total = sum(results), len(results) + if not total: + # A config that declares nothing must NOT read as a pass. `n_ok == len(results)` + # is 0 == 0 on an empty run, so the old line printed ALL OK (0/0) and exited 0 — + # a *vacuous pass*: true only because there was nothing to falsify it. Callers + # chaining on `&&`, and humans reading the verdict line, could not see it. + print(f"=== verdict: NOTHING VERIFIED (0/0) — no ledger was checked ===") + print(f" the config declares no ledger this orchestrator can read; " + f"a green verdict here would certify nothing.") + raise SystemExit(2) + verdict = "ALL OK" if n_ok == total else "FAILURES PRESENT" + print(f"=== verdict: {verdict} ({n_ok}/{total}) ===") + raise SystemExit(0 if n_ok == total else 1) if __name__ == "__main__": diff --git a/stack/verify_self.py b/stack/verify_self.py index 1b197ef..32892a5 100644 --- a/stack/verify_self.py +++ b/stack/verify_self.py @@ -49,7 +49,10 @@ def mm_self_verify(path, name, report): if bad: report(FAIL, "L1 chain", name, f"mm verify_chain: {[str(f) for f in bad]}") else: - report(OK, "L1 chain", name, "mm verify_chain: seals valid") + # Report the DENOMINATOR, not just the colour: "seals valid" over zero entries + # is the same vacuous pass the verdict-line guard blocks one level up. + n = len(load_jsonl(path)) + report(OK, "L1 chain", name, f"mm verify_chain: seals valid ({n} entries checked)") except Exception as e: report(WARN, "L1 chain", name, f"mm lib unavailable, linkage-only ({e})") @@ -96,10 +99,16 @@ def report(level, layer, name, msg): print("=== verify-self (measure-mirror: L1 chain + L3 anchors) ===") verify_self(ledger, anchor_dir, report) - n_ok = sum(results) - verdict = "ALL OK" if n_ok == len(results) else "FAILURES PRESENT" - print(f"=== verdict: {verdict} ({n_ok}/{len(results)}) ===") - sys.exit(0 if n_ok == len(results) else 1) + n_ok, total = sum(results), len(results) + if not total: + # Same guard as verify_all.py. Not reachable from this CLI today (generic_linkage + # always reports), but the bare `n_ok == len(results)` is the shape that let a + # vacuous pass through one layer up — pin it here so a refactor cannot reopen it. + print(f"=== verdict: NOTHING VERIFIED (0/0) — no check ran ===") + sys.exit(2) + verdict = "ALL OK" if n_ok == total else "FAILURES PRESENT" + print(f"=== verdict: {verdict} ({n_ok}/{total}) ===") + sys.exit(0 if n_ok == total else 1) if __name__ == "__main__": diff --git a/tests/test_vacuous_pass.py b/tests/test_vacuous_pass.py new file mode 100644 index 0000000..18a1b45 --- /dev/null +++ b/tests/test_vacuous_pass.py @@ -0,0 +1,52 @@ +"""A verifier that checked nothing must not report a pass. + +`verdict = "ALL OK" if n_ok == len(results)` is `0 == 0` when no check ran, so an empty +config printed `ALL OK (0/0)` and exited 0 — true only because nothing could falsify it +(a *vacuous pass*, the failure mode `vacuity detection` names in formal verification). +The verdict line and the exit code are what automation and readers act on, so both must +distinguish "everything passed" from "nothing was measured". +""" +import json +import subprocess +import sys +from pathlib import Path + +STACK = Path(__file__).resolve().parent.parent / "stack" + + +def _run(script, *args): + return subprocess.run([sys.executable, str(STACK / script), *args], + capture_output=True, text=True) + + +def test_empty_config_is_not_a_pass(tmp_path): + cfg = tmp_path / "empty.json" + cfg.write_text(json.dumps({"mm_ledgers": {}}), encoding="utf-8") + r = _run("verify_all.py", "--config", str(cfg)) + assert "ALL OK" not in r.stdout, "an empty declaration must never read as a pass" + assert "NOTHING VERIFIED" in r.stdout + assert r.returncode != 0, "exit 0 would let `verify && publish` proceed on no evidence" + + +def test_real_config_still_passes(tmp_path): + """The guard must not break the normal green path (regression on the fix itself).""" + led = tmp_path / "l.jsonl" + led.write_text('{"prev_seal":"genesis","seal":"a"}\n', encoding="utf-8") + anchors = tmp_path / "anchors" + anchors.mkdir() + cfg = tmp_path / "c.json" + cfg.write_text(json.dumps({"mm_ledgers": {"x": str(led)}, "anchor_dir": str(anchors)}), + encoding="utf-8") + r = _run("verify_all.py", "--config", str(cfg)) + assert "NOTHING VERIFIED" not in r.stdout + assert "verdict:" in r.stdout + + +def test_seal_check_reports_its_denominator(tmp_path): + """"seals valid" without a count cannot be told apart from "no seals to check".""" + led = STACK / "evidence" / "compute_governor.jsonl" # real seals, so the OK path runs + anchors = tmp_path / "anchors" + anchors.mkdir() + r = _run("verify_self.py", str(led), str(anchors)) + assert "seals valid" in r.stdout + assert "entries checked" in r.stdout, "a green line must carry the number it verified"