Fix evidence checks for unknown clock states and equivalent metadata wording

misc/mcp/README.md

@@ -243,6 +243,13 @@ it would, so it does not consult `robots.txt` unless asked to with
 `--respect-robots` (`CPU_PERF_RESPECT_ROBOTS=1`); sources a site then
 disallows are reported as blocked.
 
+`check_evidence` is a lexical metadata check, not validation of a measurement.
+Explicitly unknown or unmeasured clock, turbo/DVFS and SMT states remain incomplete;
+a nominal or advertised clock alone is not a runtime frequency. Conflicting known
+and unknown declarations are treated conservatively. Arbitrary prose still needs
+human review. Its deterministic claim evals can be run with
+`python misc/mcp/tests/eval_evidence.py` from a source checkout.
+
 ## Tools
 
 | Tool | What it answers |

misc/mcp/src/cpu_perf/evidence.py

@@ -28,7 +28,7 @@
 )
 CORES = re.compile(
     r"\b((?:\d+|one|two|four|six|eight|ten|twelve|sixteen|thirty-two|sixty-four|single)[- ]?"
-    r"(?:p-|e-|performance |efficiency )?(?:cores?|threads?|vcpus?|hardware threads|sockets?|copies|processors?)|"
+    r"(?:pinned )?(?:p-|e-|performance |efficiency )?(?:cores?|threads?|vcpus?|hardware threads|sockets?|copies|processors?)|"
     r"single[- ]threaded|cores? used|taskset|numactl|\d+\s*t\b)",
     I,
 )
@@ -37,7 +37,7 @@
 SMT = re.compile(r"\b(smt|hyper-?threading|ht (?:on|off)|siblings?|no smt|smt: none)\b", I)
 COMPILER = re.compile(
     r"\b(gcc|g\+\+|clang|llvm|icx|icpx|icc|msvc|rustc|go\s*1\.\d+|javac|openjdk|nvcc|xlc|armclang|aocc|"
-    r"apple clang|cl\.exe)\b(?:\s*(?:version\s*)?\d+(?:\.\d+)*)?",
+    r"apple clang|cl\.exe)(?=\b|\d)(?:\s*(?:version\s*)?\d+(?:\.\d+)*)?",
     I,
 )
 FLAGS = re.compile(
@@ -61,7 +61,8 @@
     I,
 )
 RUNS = re.compile(
-    r"\b((?:\d+|three|five|seven|ten|eleven|twenty)\s*(?:timed\s+)?(?:runs?|repetitions?|reps|trials|iterations|samples|rounds|passes)|"
+    r"\b((?:\d+|one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve|sixteen|twenty)\s*"
+    r"(?:timed\s+)?(?:(?:ABBA|A/A)\s+blocks?|runs?|repetitions?|reps|trials|iterations|samples|rounds|passes)|"
     r"perf stat|rdtsc|clock_gettime|clock_monotonic\w*|now_ns|hyperfine|google benchmark|criterion|"
     r"warm-?up|llama-bench|likwid)\b",
     I,
@@ -72,6 +73,39 @@
     I,
 )
 
+# A mention of an unknown state is not evidence that the state is known. Keep
+# this deliberately lexical: recognise explicit declarations, not arbitrary NLP.
+_CLOCK_SUBJECT = r"(?:frequency|clock|turbo|boost|dvfs|smt|hyper-?threading)(?:\s+states?)?"
+_CLOCK_SUBJECTS = rf"{_CLOCK_SUBJECT}(?:\s*(?:,\s*(?:and\s+)?|and\s+|/\s*){_CLOCK_SUBJECT})*"
+_UNCERTAIN = r"(?:unknown|unmeasured|unrecorded|unreported|unspecified|unavailable|not\s+(?:known|measured|recorded|reported|specified|available))"
+_UNCERTAIN_CLOCK = re.compile(
+    rf"\b(?P<after>{_CLOCK_SUBJECTS})(?:\s+(?:during the run|at runtime|for the run|is|was|were|are|remains?))*"
+    rf"\s*[:=]?\s*{_UNCERTAIN}\b|\b{_UNCERTAIN}\s+(?P<before>{_CLOCK_SUBJECTS})\b", I,
+)
+_NOMINAL_FREQ = re.compile(
+    r"\b(?:nominal|advertised)\s+(?:(?:frequency|clock)\s*[:=]?\s*)?"
+    r"\d+(?:\.\d+)?\s*(?:ghz|mhz)\b|"
+    r"\b\d+(?:\.\d+)?\s*(?:ghz|mhz)\s+(?:nominal|advertised)\b", I,
+)
+
+
+def _unknown_clock_fields(text: str) -> set[str]:
+    unknown: set[str] = set()
+    for match in _UNCERTAIN_CLOCK.finditer(text):
+        subjects = match.group("before") or match.group("after")
+        if re.search(r"\b(?:frequency|clock)\b", subjects, I):
+            unknown.add("frequency")
+        if TURBO.search(subjects):
+            unknown.add("turbo/DVFS state")
+        if SMT.search(subjects):
+            unknown.add("SMT state")
+    return unknown
+
+
+def _runtime_frequencies(text: str) -> list[str]:
+    return _find(FREQ, _NOMINAL_FREQ.sub(" ", text))
+
+
 FIELDS = (
     (1, "CPU model and microarchitecture", "Which CPU model, and which microarchitecture?"),
     (2, "core count used", "How many cores or threads did the run use?"),
@@ -134,10 +168,21 @@ def add(num: int, status: str, found: list[str], note: str = "") -> None:
     cores = _find(CORES, text)
     add(2, "present" if cores else "missing", cores)
     freq, turbo, smt = _find(FREQ, text), _find(TURBO, text), _find(SMT, text)
-    have = [bool(freq), bool(turbo), bool(smt)]
-    missing_parts = [n for n, h in zip(("frequency", "turbo/DVFS state", "SMT state"), have) if not h]
-    add(3, "present" if all(have) else "partial" if any(have) else "missing", freq + turbo + smt,
-        ("missing " + ", ".join(missing_parts)) if missing_parts and any(have) else "")
+    unknown = _unknown_clock_fields(text)
+    runtime_freq = _runtime_frequencies(text)
+    names = ("frequency", "turbo/DVFS state", "SMT state")
+    have = [bool(values) and name not in unknown
+            for name, values in zip(names, (runtime_freq, turbo, smt))]
+    missing_parts = [name for name, present in zip(names, have) if not present]
+    notes = []
+    if missing_parts:
+        notes.append("missing or uncertain " + ", ".join(missing_parts))
+    if unknown:
+        notes.append("explicitly unknown/unmeasured: " + ", ".join(n for n in names if n in unknown))
+    if freq and not runtime_freq:
+        notes.append("nominal/advertised frequency only")
+    add(3, "present" if all(have) else "partial" if freq or turbo or smt else "missing",
+        freq + turbo + smt, "; ".join(notes))
     comp, flags = _find(COMPILER, text), _find(FLAGS, text)
     add(4, "present" if comp and flags else "partial" if comp or flags else "missing", comp + flags,
         "" if comp and flags else ("compiler without flags" if comp else "flags without compiler" if flags else ""))

misc/mcp/tests/eval_evidence.py

@@ -0,0 +1,86 @@
+"""Deterministic evidence-checker evals, including failures from an ARM investigation.
+
+Run ``python tests/eval_evidence.py`` for per-case outcomes and aggregate errors.
+Numbers below are synthetic test inputs, not performance measurements.
+"""
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+CPU = "AWS Graviton4, Arm Neoverse V2; one core pinned to CPU2. "
+CLOCK = "Measured frequency 2.6 GHz, turbo off, SMT off. "
+BUILD = "Compiler Clang 18.1.3 -O3. "
+WORK = "Workload: B+ tree iterator benchmark, 100000 members. "
+END = "Compared with baseline revision A. 6 runs with median elapsed time, 7% faster."
+
+
+def claim(clock=CLOCK, cpu=CPU, build=BUILD, end=END):
+    return cpu + clock + build + WORK + end
+
+
+# id, text, expected verdict, fields that must be present, fields that must not be present
+CASES = [
+    ("complete", claim(), "quotable", set(range(1, 8)), set()),
+    ("unknown-input-is-not-unknown-clock", claim() + " Input key distribution unknown.", "quotable", {3}, set()),
+    ("disabled-smt", claim(clock="2.6 GHz, turbo disabled, SMT disabled. "), "quotable", {3}, set()),
+    ("no-smt", claim(clock="2.6 GHz, turbo off, no SMT on this part. "), "quotable", {3}, set()),
+    ("estimated-clock-dvfs", claim(clock="Estimated clock 2.6 GHz. DVFS is on and cannot be disabled. No SMT. "), "quotable", {3}, set()),
+    ("six-word", claim(end=END.replace("6 runs", "Six runs")), "quotable", {7}, set()),
+    ("six-abba-blocks", claim(end=END.replace("6 runs", "Six ABBA blocks")), "quotable", {7}, set()),
+    ("three-aa-blocks", claim(end=END.replace("6 runs", "Three A/A blocks")), "quotable", {7}, set()),
+    ("numeric-abba-blocks", claim(end=END.replace("6 runs", "6 ABBA blocks")), "quotable", {7}, set()),
+    ("compact-clang-version", claim(build=BUILD.replace("Clang 18", "Clang18")), "quotable", {4}, set()),
+    ("compact-gcc-version", claim(build="Compiler GCC14.2 -O3. "), "quotable", {4}, set()),
+    ("pinned-core", claim(cpu=CPU.replace("one core pinned to", "one pinned core on")), "quotable", {2}, set()),
+    ("single-pinned-thread", claim(cpu=CPU.replace("one core", "single pinned thread")), "quotable", {2}, set()),
+    ("nominal-plus-measured", claim(clock="Nominal 2.5 GHz; measured frequency 2.6 GHz, turbo off, SMT off. "), "quotable", {3}, set()),
+]
+
+for name, clock in [
+    ("turbo-unknown", "2.6 GHz, turbo state unknown, SMT off. "),
+    ("smt-unknown", "2.6 GHz, turbo off, SMT state unknown. "),
+    ("unknown-before-smt", "2.6 GHz, turbo off; unknown SMT state. "),
+    ("smt-not-recorded", "2.6 GHz, turbo off, SMT was not recorded. "),
+    ("turbo-not-known", "2.6 GHz, turbo state is not known, SMT off. "),
+    ("turbo-unspecified", "2.6 GHz, turbo unspecified, SMT off. "),
+    ("smt-unreported", "2.6 GHz, turbo off, SMT unreported. "),
+    ("dvfs-unknown", "2.6 GHz, DVFS unknown, no SMT. "),
+    ("coordinated-unknown", "2.6 GHz; turbo and SMT states unknown. "),
+    ("coordinated-unknown-prefix", "2.6 GHz; unknown turbo and SMT states. "),
+    ("frequency-unmeasured", "Nominal 2.6 GHz, turbo off, SMT off. Frequency during the run was not measured. "),
+    ("frequency-unknown-with-number", "2.6 GHz advertised, turbo off, SMT off; actual frequency unknown. "),
+    ("clock-unmeasured", "2.6 GHz, turbo off, SMT off. Runtime clock unmeasured. "),
+    ("all-unknown", "Nominal 2.6 GHz; frequency, turbo and SMT state unknown. "),
+    ("nominal-only-prefix", "Nominal frequency 2.6 GHz, turbo off, SMT off. "),
+    ("nominal-only-suffix", "Frequency 2.6 GHz nominal, turbo off, SMT off. "),
+    ("contradictory-smt", "2.6 GHz, turbo off, SMT off; SMT state unknown. "),
+    ("original-false-positive", "Frequency 2.6 GHz nominal, turbo state unknown, SMT state unknown. Frequency during the run was not measured. "),
+]:
+    CASES.append((name, claim(clock=clock), "drop_number", {1, 2, 4, 5, 6, 7}, {3}))
+
+CASES.extend([
+    ("missing-clock", claim(clock=""), "drop_number", {1, 2, 4, 5, 6, 7}, {3}),
+    ("abba-without-count", claim(end=END.replace("6 runs", "ABBA blocks")), "drop_number", set(), {7}),
+    ("unrelated-compiler-word", claim(build="Compiler clangorous -O3. "), "drop_number", set(), {4}),
+    ("no-number", "This paper defines the roofline model.", "no_number", set(), set()),
+])
+
+
+def evaluate(audit):
+    rows = []
+    for ident, text, verdict, present, absent in CASES:
+        r = audit(text)
+        states = {f.number: f.status for f in r.fields}
+        ok = r.verdict == verdict and all(states[n] == "present" for n in present) and all(states[n] != "present" for n in absent)
+        rows.append(dict(id=ident, ok=ok, expected=verdict, actual=r.verdict, fields=states))
+    return dict(cases=len(rows), passed=sum(r["ok"] for r in rows),
+                false_quotable=sum(r["actual"] == "quotable" and r["expected"] == "drop_number" for r in rows),
+                false_rejection=sum(r["actual"] == "drop_number" and r["expected"] == "quotable" for r in rows), rows=rows)
+
+
+if __name__ == "__main__":
+    sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
+    from cpu_perf.evidence import audit
+    print(json.dumps(evaluate(audit), indent=2))

misc/mcp/tests/test_evidence_eval.py

@@ -0,0 +1,13 @@
+"""Golden claim evals: false confidence and equivalent wording both matter."""
+import pytest
+from cpu_perf.evidence import audit
+from eval_evidence import CASES
+
+
+@pytest.mark.parametrize("ident,text,verdict,present,absent", CASES, ids=[c[0] for c in CASES])
+def test_evidence_cases(ident, text, verdict, present, absent):
+    result = audit(text)
+    states = {f.number: f.status for f in result.fields}
+    assert result.verdict == verdict, result
+    assert all(states[n] == "present" for n in present), result
+    assert all(states[n] != "present" for n in absent), result

misc/mcp/tests/test_server.py

@@ -17,6 +17,7 @@
 from cpu_perf.server import create_server
 
 from fixture_site import Site
+from eval_evidence import CASES as EVIDENCE_CASES
 
 TOOLS = {
     "ask", "lookup", "search", "fetch", "get_section", "get_entry", "reading_path", "get_benchmark", "editorial_record",
@@ -433,3 +434,15 @@ def test_out_of_range_arguments_get_a_clear_error(server):
     r = call(server, "lookup", {"query": "roofline", "limit": 500})
     assert r.is_error
     assert "limit" in r.content[0].text
+
+
+@pytest.mark.parametrize("ident,text,verdict,present,absent", EVIDENCE_CASES, ids=[c[0] for c in EVIDENCE_CASES])
+def test_evidence_eval_through_mcp(server, ident, text, verdict, present, absent):
+    result = call(server, "check_evidence", {"claim": text})
+    assert not result.is_error
+    data = result.structured_content
+    assert data["verdict"] == verdict
+    states = {f["number"]: f["status"] for f in data["fields"]}
+    assert all(states[n] == "present" for n in present)
+    assert all(states[n] != "present" for n in absent)
+    assert verdict in result.content[0].text