diff --git a/misc/mcp/README.md b/misc/mcp/README.md index 91be81f..a82f03b 100644 --- a/misc/mcp/README.md +++ b/misc/mcp/README.md @@ -243,6 +243,13 @@ it would, so it does not consult `robots.txt` unless asked to with `--respect-robots` (`CPU_PERF_RESPECT_ROBOTS=1`); sources a site then disallows are reported as blocked. +`check_evidence` is a lexical metadata check, not validation of a measurement. +Explicitly unknown or unmeasured clock, turbo/DVFS and SMT states remain incomplete; +a nominal or advertised clock alone is not a runtime frequency. Conflicting known +and unknown declarations are treated conservatively. Arbitrary prose still needs +human review. Its deterministic claim evals can be run with +`python misc/mcp/tests/eval_evidence.py` from a source checkout. + ## Tools | Tool | What it answers | diff --git a/misc/mcp/src/cpu_perf/evidence.py b/misc/mcp/src/cpu_perf/evidence.py index 0289518..47c31ce 100644 --- a/misc/mcp/src/cpu_perf/evidence.py +++ b/misc/mcp/src/cpu_perf/evidence.py @@ -28,7 +28,7 @@ ) CORES = re.compile( r"\b((?:\d+|one|two|four|six|eight|ten|twelve|sixteen|thirty-two|sixty-four|single)[- ]?" - r"(?:p-|e-|performance |efficiency )?(?:cores?|threads?|vcpus?|hardware threads|sockets?|copies|processors?)|" + r"(?:pinned )?(?:p-|e-|performance |efficiency )?(?:cores?|threads?|vcpus?|hardware threads|sockets?|copies|processors?)|" r"single[- ]threaded|cores? used|taskset|numactl|\d+\s*t\b)", I, ) @@ -37,7 +37,7 @@ SMT = re.compile(r"\b(smt|hyper-?threading|ht (?:on|off)|siblings?|no smt|smt: none)\b", I) COMPILER = re.compile( r"\b(gcc|g\+\+|clang|llvm|icx|icpx|icc|msvc|rustc|go\s*1\.\d+|javac|openjdk|nvcc|xlc|armclang|aocc|" - r"apple clang|cl\.exe)\b(?:\s*(?:version\s*)?\d+(?:\.\d+)*)?", + r"apple clang|cl\.exe)(?=\b|\d)(?:\s*(?:version\s*)?\d+(?:\.\d+)*)?", I, ) FLAGS = re.compile( @@ -61,7 +61,8 @@ I, ) RUNS = re.compile( - r"\b((?:\d+|three|five|seven|ten|eleven|twenty)\s*(?:timed\s+)?(?:runs?|repetitions?|reps|trials|iterations|samples|rounds|passes)|" + r"\b((?:\d+|one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve|sixteen|twenty)\s*" + r"(?:timed\s+)?(?:(?:ABBA|A/A)\s+blocks?|runs?|repetitions?|reps|trials|iterations|samples|rounds|passes)|" r"perf stat|rdtsc|clock_gettime|clock_monotonic\w*|now_ns|hyperfine|google benchmark|criterion|" r"warm-?up|llama-bench|likwid)\b", I, @@ -72,6 +73,39 @@ I, ) +# A mention of an unknown state is not evidence that the state is known. Keep +# this deliberately lexical: recognise explicit declarations, not arbitrary NLP. +_CLOCK_SUBJECT = r"(?:frequency|clock|turbo|boost|dvfs|smt|hyper-?threading)(?:\s+states?)?" +_CLOCK_SUBJECTS = rf"{_CLOCK_SUBJECT}(?:\s*(?:,\s*(?:and\s+)?|and\s+|/\s*){_CLOCK_SUBJECT})*" +_UNCERTAIN = r"(?:unknown|unmeasured|unrecorded|unreported|unspecified|unavailable|not\s+(?:known|measured|recorded|reported|specified|available))" +_UNCERTAIN_CLOCK = re.compile( + rf"\b(?P{_CLOCK_SUBJECTS})(?:\s+(?:during the run|at runtime|for the run|is|was|were|are|remains?))*" + rf"\s*[:=]?\s*{_UNCERTAIN}\b|\b{_UNCERTAIN}\s+(?P{_CLOCK_SUBJECTS})\b", I, +) +_NOMINAL_FREQ = re.compile( + r"\b(?:nominal|advertised)\s+(?:(?:frequency|clock)\s*[:=]?\s*)?" + r"\d+(?:\.\d+)?\s*(?:ghz|mhz)\b|" + r"\b\d+(?:\.\d+)?\s*(?:ghz|mhz)\s+(?:nominal|advertised)\b", I, +) + + +def _unknown_clock_fields(text: str) -> set[str]: + unknown: set[str] = set() + for match in _UNCERTAIN_CLOCK.finditer(text): + subjects = match.group("before") or match.group("after") + if re.search(r"\b(?:frequency|clock)\b", subjects, I): + unknown.add("frequency") + if TURBO.search(subjects): + unknown.add("turbo/DVFS state") + if SMT.search(subjects): + unknown.add("SMT state") + return unknown + + +def _runtime_frequencies(text: str) -> list[str]: + return _find(FREQ, _NOMINAL_FREQ.sub(" ", text)) + + FIELDS = ( (1, "CPU model and microarchitecture", "Which CPU model, and which microarchitecture?"), (2, "core count used", "How many cores or threads did the run use?"), @@ -134,10 +168,21 @@ def add(num: int, status: str, found: list[str], note: str = "") -> None: cores = _find(CORES, text) add(2, "present" if cores else "missing", cores) freq, turbo, smt = _find(FREQ, text), _find(TURBO, text), _find(SMT, text) - have = [bool(freq), bool(turbo), bool(smt)] - missing_parts = [n for n, h in zip(("frequency", "turbo/DVFS state", "SMT state"), have) if not h] - add(3, "present" if all(have) else "partial" if any(have) else "missing", freq + turbo + smt, - ("missing " + ", ".join(missing_parts)) if missing_parts and any(have) else "") + unknown = _unknown_clock_fields(text) + runtime_freq = _runtime_frequencies(text) + names = ("frequency", "turbo/DVFS state", "SMT state") + have = [bool(values) and name not in unknown + for name, values in zip(names, (runtime_freq, turbo, smt))] + missing_parts = [name for name, present in zip(names, have) if not present] + notes = [] + if missing_parts: + notes.append("missing or uncertain " + ", ".join(missing_parts)) + if unknown: + notes.append("explicitly unknown/unmeasured: " + ", ".join(n for n in names if n in unknown)) + if freq and not runtime_freq: + notes.append("nominal/advertised frequency only") + add(3, "present" if all(have) else "partial" if freq or turbo or smt else "missing", + freq + turbo + smt, "; ".join(notes)) comp, flags = _find(COMPILER, text), _find(FLAGS, text) add(4, "present" if comp and flags else "partial" if comp or flags else "missing", comp + flags, "" if comp and flags else ("compiler without flags" if comp else "flags without compiler" if flags else "")) diff --git a/misc/mcp/tests/eval_evidence.py b/misc/mcp/tests/eval_evidence.py new file mode 100644 index 0000000..b5f9d43 --- /dev/null +++ b/misc/mcp/tests/eval_evidence.py @@ -0,0 +1,86 @@ +"""Deterministic evidence-checker evals, including failures from an ARM investigation. + +Run ``python tests/eval_evidence.py`` for per-case outcomes and aggregate errors. +Numbers below are synthetic test inputs, not performance measurements. +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + +CPU = "AWS Graviton4, Arm Neoverse V2; one core pinned to CPU2. " +CLOCK = "Measured frequency 2.6 GHz, turbo off, SMT off. " +BUILD = "Compiler Clang 18.1.3 -O3. " +WORK = "Workload: B+ tree iterator benchmark, 100000 members. " +END = "Compared with baseline revision A. 6 runs with median elapsed time, 7% faster." + + +def claim(clock=CLOCK, cpu=CPU, build=BUILD, end=END): + return cpu + clock + build + WORK + end + + +# id, text, expected verdict, fields that must be present, fields that must not be present +CASES = [ + ("complete", claim(), "quotable", set(range(1, 8)), set()), + ("unknown-input-is-not-unknown-clock", claim() + " Input key distribution unknown.", "quotable", {3}, set()), + ("disabled-smt", claim(clock="2.6 GHz, turbo disabled, SMT disabled. "), "quotable", {3}, set()), + ("no-smt", claim(clock="2.6 GHz, turbo off, no SMT on this part. "), "quotable", {3}, set()), + ("estimated-clock-dvfs", claim(clock="Estimated clock 2.6 GHz. DVFS is on and cannot be disabled. No SMT. "), "quotable", {3}, set()), + ("six-word", claim(end=END.replace("6 runs", "Six runs")), "quotable", {7}, set()), + ("six-abba-blocks", claim(end=END.replace("6 runs", "Six ABBA blocks")), "quotable", {7}, set()), + ("three-aa-blocks", claim(end=END.replace("6 runs", "Three A/A blocks")), "quotable", {7}, set()), + ("numeric-abba-blocks", claim(end=END.replace("6 runs", "6 ABBA blocks")), "quotable", {7}, set()), + ("compact-clang-version", claim(build=BUILD.replace("Clang 18", "Clang18")), "quotable", {4}, set()), + ("compact-gcc-version", claim(build="Compiler GCC14.2 -O3. "), "quotable", {4}, set()), + ("pinned-core", claim(cpu=CPU.replace("one core pinned to", "one pinned core on")), "quotable", {2}, set()), + ("single-pinned-thread", claim(cpu=CPU.replace("one core", "single pinned thread")), "quotable", {2}, set()), + ("nominal-plus-measured", claim(clock="Nominal 2.5 GHz; measured frequency 2.6 GHz, turbo off, SMT off. "), "quotable", {3}, set()), +] + +for name, clock in [ + ("turbo-unknown", "2.6 GHz, turbo state unknown, SMT off. "), + ("smt-unknown", "2.6 GHz, turbo off, SMT state unknown. "), + ("unknown-before-smt", "2.6 GHz, turbo off; unknown SMT state. "), + ("smt-not-recorded", "2.6 GHz, turbo off, SMT was not recorded. "), + ("turbo-not-known", "2.6 GHz, turbo state is not known, SMT off. "), + ("turbo-unspecified", "2.6 GHz, turbo unspecified, SMT off. "), + ("smt-unreported", "2.6 GHz, turbo off, SMT unreported. "), + ("dvfs-unknown", "2.6 GHz, DVFS unknown, no SMT. "), + ("coordinated-unknown", "2.6 GHz; turbo and SMT states unknown. "), + ("coordinated-unknown-prefix", "2.6 GHz; unknown turbo and SMT states. "), + ("frequency-unmeasured", "Nominal 2.6 GHz, turbo off, SMT off. Frequency during the run was not measured. "), + ("frequency-unknown-with-number", "2.6 GHz advertised, turbo off, SMT off; actual frequency unknown. "), + ("clock-unmeasured", "2.6 GHz, turbo off, SMT off. Runtime clock unmeasured. "), + ("all-unknown", "Nominal 2.6 GHz; frequency, turbo and SMT state unknown. "), + ("nominal-only-prefix", "Nominal frequency 2.6 GHz, turbo off, SMT off. "), + ("nominal-only-suffix", "Frequency 2.6 GHz nominal, turbo off, SMT off. "), + ("contradictory-smt", "2.6 GHz, turbo off, SMT off; SMT state unknown. "), + ("original-false-positive", "Frequency 2.6 GHz nominal, turbo state unknown, SMT state unknown. Frequency during the run was not measured. "), +]: + CASES.append((name, claim(clock=clock), "drop_number", {1, 2, 4, 5, 6, 7}, {3})) + +CASES.extend([ + ("missing-clock", claim(clock=""), "drop_number", {1, 2, 4, 5, 6, 7}, {3}), + ("abba-without-count", claim(end=END.replace("6 runs", "ABBA blocks")), "drop_number", set(), {7}), + ("unrelated-compiler-word", claim(build="Compiler clangorous -O3. "), "drop_number", set(), {4}), + ("no-number", "This paper defines the roofline model.", "no_number", set(), set()), +]) + + +def evaluate(audit): + rows = [] + for ident, text, verdict, present, absent in CASES: + r = audit(text) + states = {f.number: f.status for f in r.fields} + ok = r.verdict == verdict and all(states[n] == "present" for n in present) and all(states[n] != "present" for n in absent) + rows.append(dict(id=ident, ok=ok, expected=verdict, actual=r.verdict, fields=states)) + return dict(cases=len(rows), passed=sum(r["ok"] for r in rows), + false_quotable=sum(r["actual"] == "quotable" and r["expected"] == "drop_number" for r in rows), + false_rejection=sum(r["actual"] == "drop_number" and r["expected"] == "quotable" for r in rows), rows=rows) + + +if __name__ == "__main__": + sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + from cpu_perf.evidence import audit + print(json.dumps(evaluate(audit), indent=2)) diff --git a/misc/mcp/tests/test_evidence_eval.py b/misc/mcp/tests/test_evidence_eval.py new file mode 100644 index 0000000..3e31c94 --- /dev/null +++ b/misc/mcp/tests/test_evidence_eval.py @@ -0,0 +1,13 @@ +"""Golden claim evals: false confidence and equivalent wording both matter.""" +import pytest +from cpu_perf.evidence import audit +from eval_evidence import CASES + + +@pytest.mark.parametrize("ident,text,verdict,present,absent", CASES, ids=[c[0] for c in CASES]) +def test_evidence_cases(ident, text, verdict, present, absent): + result = audit(text) + states = {f.number: f.status for f in result.fields} + assert result.verdict == verdict, result + assert all(states[n] == "present" for n in present), result + assert all(states[n] != "present" for n in absent), result diff --git a/misc/mcp/tests/test_server.py b/misc/mcp/tests/test_server.py index 020446d..f154204 100644 --- a/misc/mcp/tests/test_server.py +++ b/misc/mcp/tests/test_server.py @@ -17,6 +17,7 @@ from cpu_perf.server import create_server from fixture_site import Site +from eval_evidence import CASES as EVIDENCE_CASES TOOLS = { "ask", "lookup", "search", "fetch", "get_section", "get_entry", "reading_path", "get_benchmark", "editorial_record", @@ -433,3 +434,15 @@ def test_out_of_range_arguments_get_a_clear_error(server): r = call(server, "lookup", {"query": "roofline", "limit": 500}) assert r.is_error assert "limit" in r.content[0].text + + +@pytest.mark.parametrize("ident,text,verdict,present,absent", EVIDENCE_CASES, ids=[c[0] for c in EVIDENCE_CASES]) +def test_evidence_eval_through_mcp(server, ident, text, verdict, present, absent): + result = call(server, "check_evidence", {"claim": text}) + assert not result.is_error + data = result.structured_content + assert data["verdict"] == verdict + states = {f["number"]: f["status"] for f in data["fields"]} + assert all(states[n] == "present" for n in present) + assert all(states[n] != "present" for n in absent) + assert verdict in result.content[0].text