feat(dataset-quality): count skills in trajectory coverage

datasets/dataset_quality/example_run_config.yaml

@@ -1,10 +1,8 @@
 ############################################################
 ### Dataset Quality — example run config
 ############################################################
-# Grades the intrinsic quality of a CUJ dataset (no eval / agent run). Runs on the
-# standard agent pipeline: a no-op generator does nothing, and the `dataset_quality`
-# scorer grades the whole dataset holistically (each sub-scorer computes a 0-100
-# sub-score; grading rolls them into a weighted global score + letter grade).
+# Grades the dataset's CUJs, not a model: no agent runs and nothing is executed.
+# The product only supplies the tools and skills those CUJs are scored against.
 
 # Forced by dataset_format below; listed here only to document what runs.
 orchestrator: agent
@@ -26,12 +24,9 @@ simulated_user_model_config: datasets/model_configs/noop_agent_model.yaml
 ############################################################
 ### Scorers
 ############################################################
-# The `dataset_quality` scorer fetches the product's tool catalog live from its own
-# model_config (each setup.mcp_servers entry with an httpUrl/url is queried via MCP
-# tools/list; names are namespaced <server>__<tool> to match expected_trajectory).
-# google_credentials servers use ADC, so this needs credentials + network (e.g.
-# `gcloud auth application-default login`). Run the LLM judges with
-# EVAL_GCP_PROJECT_REGION=global.
+# Reads the product's capability surface from the model_config below:
+# setup.mcp_servers for tools (named <server>__<tool>, matching
+# expected_trajectory) and setup.skills for skills.
 scorers:
   dataset_quality:
     model_config: datasets/model_configs/gemini_cli_model.yaml
@@ -41,6 +36,8 @@ scorers:
     # required only by the judge sub-scorers; trajectory_coverage and
     # naming_distribution are static.
     sub_scorers:
+      # Counts MCP tools and skill scripts (matched against a CUJ's
+      # expected_trajectory) plus each skill's own name (expected_skills).
       trajectory_coverage:
         # weight: 20  (default)
       naming_distribution:
@@ -71,8 +68,6 @@ scorers:
 ############################################################
 ### Reporting
 ############################################################
-# The grade lands as one summary scoring row plus one row per category; uncomment
-# bigquery to also append to the table.
 reporting:
   csv:
     output_directory: 'results'

evalbench/generators/models/skills_catalog.py

@@ -0,0 +1,255 @@
+"""Reads a product's skill catalog from a model config's setup block.
+
+The skills counterpart of mcp_client. An entry that won't resolve raises instead
+of being skipped: a half-resolved catalog is a shrunken denominator, which
+inflates any coverage measured against it.
+"""
+
+from dataclasses import dataclass
+import logging
+import os
+import re
+import shutil
+import subprocess
+import tempfile
+
+import yaml
+
+
+_GIT_URL_PATTERN = re.compile(r"^(https?|git|ssh)://|^git@|\.git(#.*)?$")
+
+_CLONE_TIMEOUT_S = 120
+
+# Leading `---` delimited YAML block at the top of a SKILL.md.
+_FRONTMATTER_PATTERN = re.compile(r"\A---\s*\n(.*?)\n---\s*(?:\n|\Z)", re.DOTALL)
+
+# Languages a skill's scripts are written in.
+_SCRIPT_SUFFIXES = frozenset({".js", ".cjs", ".mjs", ".ts", ".py", ".sh", ".bash"})
+
+
+class SkillCatalogError(Exception):
+    """A declared skills entry could not be resolved."""
+
+
+@dataclass(frozen=True)
+class Skill:
+    """One skill in a product's catalog.
+
+    `scripts` holds a skill's adjacent scripts/ directory — the layout MCP-toolbox
+    generated skills use, and what a CUJ trajectory names. A skill without one is
+    graded on its name alone.
+    """
+
+    name: str
+    description: str = ""
+    scripts: tuple[str, ...] = ()
+
+
+def resolve_skills(setup: dict) -> list[Skill]:
+    """Skills declared by a model config's setup block.
+
+    Reads setup.skills and setup.skills_dir, returning an empty list when neither
+    is present. Each declared entry must yield at least one skill or it raises
+    SkillCatalogError, so "no skills" stays distinguishable from "skills failed
+    to resolve".
+    """
+    setup = setup or {}
+    skills: list[Skill] = []
+
+    for entry in setup.get("skills") or []:
+        _collect_entry(entry, skills)
+
+    skills_dir = setup.get("skills_dir")
+    if skills_dir:
+        _extend(skills, _scan_dir(skills_dir))
+
+    return skills
+
+
+def _collect_entry(entry, into: list[Skill]) -> None:
+    """Resolves one setup.skills entry and appends it to the list."""
+    if isinstance(entry, str):
+        _extend(into, _resolve_target(entry, wanted=None))
+        return
+    if not isinstance(entry, dict):
+        raise SkillCatalogError(f"unusable skills entry {entry!r}")
+
+    target = entry.get("url") or entry.get("path")
+    if not target:
+        raise SkillCatalogError(f"skills entry {entry!r} names no url or path")
+
+    _extend(into, _resolve_target(target, wanted=_wanted_names(entry)))
+
+
+def _wanted_names(entry: dict) -> set[str] | None:
+    """The subset of a source's skills an entry installs, if it narrows them.
+
+    An entry may install only some of a repo's skills, in which case the rest are
+    not part of the product's surface and must not land in the denominator.
+    """
+    names = entry.get("skills") or entry.get("skill_names")
+    if not names:
+        single = entry.get("skill") or entry.get("name")
+        names = [single] if single else None
+    return {str(name) for name in names} if names else None
+
+
+def _resolve_target(target: str, wanted: set[str] | None) -> list[Skill]:
+    """Scan a local path, or clone a git URL to a temp dir and scan that."""
+    if not _GIT_URL_PATTERN.search(target):
+        return _scan_dir(target, wanted)
+
+    clone_dir = _clone(target)
+    try:
+        return _scan_dir(clone_dir, wanted)
+    finally:
+        shutil.rmtree(clone_dir, ignore_errors=True)
+
+
+def _clone(url: str) -> str:
+    """Clones a skills repository into a temporary directory.
+
+    A '<url>#<ref>' ref is passed to --branch, so it must be a branch or tag; a
+    commit SHA fails rather than resolving. Retrying such a ref unpinned would
+    clone the default branch, grading the dataset against the wrong catalog.
+    """
+    clone_url, _, ref = url.partition("#")
+    dest = tempfile.mkdtemp(prefix="skills_catalog_")
+    cmd = ["git", "clone", "--depth", "1"]
+    if ref:
+        cmd += ["--branch", ref]
+    try:
+        subprocess.run(
+            cmd + [clone_url, dest], check=True, capture_output=True, text=True,
+            timeout=_CLONE_TIMEOUT_S,
+        )
+    except (subprocess.CalledProcessError, subprocess.TimeoutExpired,
+            OSError) as e:
+        shutil.rmtree(dest, ignore_errors=True)
+        raise SkillCatalogError(f"clone failed for {url}: {e}") from e
+    return dest
+
+
+def _scan_dir(root: str, wanted: set[str] | None = None) -> list[Skill]:
+    """Reads every skill under a root, optionally narrowed to the wanted names."""
+    if not os.path.isdir(root):
+        raise SkillCatalogError(f"skills path not found: {root}")
+    skills = []
+    by_identity = {}
+    for path in _find_skill_dirs(root):
+        skill = _read_skill(path)
+        skills.append(skill)
+        # An entry may name a skill by its frontmatter name or by its directory.
+        by_identity[skill.name.lower()] = skill
+        by_identity.setdefault(os.path.basename(path).lower(), skill)
+
+    if not skills:
+        raise SkillCatalogError(f"no SKILL.md found under {root}")
+    if not wanted:
+        return skills
+
+    # Every wanted name must match. One that doesn't would go missing from the
+    # denominator and inflate coverage.
+    missing = sorted(name for name in wanted if name.lower() not in by_identity)
+    if missing:
+        raise SkillCatalogError(f"{root} holds no skill named: {', '.join(missing)}")
+    wanted_skills = {by_identity[name.lower()] for name in wanted}
+    return [skill for skill in skills if skill in wanted_skills]
+
+
+def _find_skill_dirs(root: str) -> list[str]:
+    """Skill directories under a root, matching the generators' three layouts."""
+    if os.path.exists(os.path.join(root, "SKILL.md")):
+        return [root]
+
+    skills_root = os.path.join(root, "skills")
+    if os.path.isdir(skills_root):
+        return _child_skill_dirs(skills_root)
+
+    return _child_skill_dirs(root)
+
+
+def _child_skill_dirs(parent: str) -> list[str]:
+    try:
+        entries = sorted(os.listdir(parent))
+    except OSError as e:
+        raise SkillCatalogError(f"cannot list {parent}: {e}") from e
+    return [
+        os.path.join(parent, entry)
+        for entry in entries
+        if os.path.exists(os.path.join(parent, entry, "SKILL.md"))
+    ]
+
+
+def _read_skill(skill_dir: str) -> Skill:
+    """One skill's name and description, from SKILL.md frontmatter.
+
+    An agent activates a skill by its frontmatter name, so that wins over the
+    directory name, which is the fallback when the frontmatter is missing or
+    malformed.
+    """
+    dir_name = os.path.basename(os.path.normpath(skill_dir))
+    meta = _read_frontmatter(os.path.join(skill_dir, "SKILL.md"))
+    name = meta.get("name") or dir_name
+    description = meta.get("description") or ""
+    return Skill(
+        name=str(name).strip(),
+        description=str(description).strip(),
+        scripts=_read_scripts(skill_dir),
+    )
+
+
+def _read_scripts(skill_dir: str) -> tuple[str, ...]:
+    """The invocable scripts in a skill's scripts/ directory.
+
+    Kept as filenames rather than stems because that is how a trajectory names
+    them ("list_instances.js").
+    """
+    scripts_dir = os.path.join(skill_dir, "scripts")
+    try:
+        entries = sorted(os.listdir(scripts_dir))
+    except OSError:
+        return ()
+    return tuple(
+        entry for entry in entries
+        if _is_script(entry) and os.path.isfile(os.path.join(scripts_dir, entry))
+    )
+
+
+def _is_script(filename: str) -> bool:
+    """Whether a scripts/ entry is an invocable script a CUJ could name."""
+    stem, suffix = os.path.splitext(filename)
+    if suffix.lower() not in _SCRIPT_SUFFIXES or stem == "__init__":
+        return False
+    return not (stem.startswith("test_") or stem.endswith("_test"))
+
+
+def _read_frontmatter(skill_md: str) -> dict:
+    try:
+        with open(skill_md, "r", encoding="utf-8") as f:
+            text = f.read()
+    except OSError as e:
+        logging.warning("skills_catalog: cannot read %s: %s", skill_md, e)
+        return {}
+
+    match = _FRONTMATTER_PATTERN.match(text)
+    if not match:
+        return {}
+    try:
+        meta = yaml.safe_load(match.group(1))
+    except yaml.YAMLError as e:
+        logging.warning(
+            "skills_catalog: malformed frontmatter in %s: %s", skill_md, e
+        )
+        return {}
+    return meta if isinstance(meta, dict) else {}
+
+
+def _extend(skills: list[Skill], found: list[Skill]) -> None:
+    """Appends the found skills, dropping names already present (ignoring case)."""
+    seen = {skill.name.lower() for skill in skills}
+    for skill in found:
+        key = skill.name.lower()
+        if key and key not in seen:
+            seen.add(key)
+            skills.append(skill)

evalbench/scorers/dataset_quality/composition.py

@@ -91,10 +91,10 @@ def run(self, context: DatasetQualityContext) -> SubScoreContribution:
         return SubScoreContribution(
             score=score,
             metrics={
-                "dq_multitool_count": n_multi,
-                "dq_sequence_count": n_seq,
-                "dq_multitool_score": multitool_score,
-                "dq_sequencing_score": sequencing_score,
+                "multitool_cuj_count": n_multi,
+                "sequence_cuj_count": n_seq,
+                "multitool_score": multitool_score,
+                "sequencing_score": sequencing_score,
             },
             suggestions=suggestions,
             evidence={KEY_MULTI_TOOL: multi_ids, KEY_SEQUENCE_DEPENDENCY: seq_ids},

evalbench/scorers/dataset_quality/context.py

@@ -45,13 +45,22 @@ def expected_trajectory(scenario: dict) -> list:
     return trajectory if isinstance(trajectory, list) else []
 
 
+def expected_skills(scenario: dict) -> list:
+    """A scenario's expected_skills, or empty when it isn't a list."""
+    skills = scenario.get("expected_skills")
+    return skills if isinstance(skills, list) else []
+
+
 @dataclass
 class DatasetQualityContext:
     """Everything a scorer needs to grade one product's CUJ dataset."""
 
     product_name: str
     scenarios: list[dict]
     tools: list  # list[mcp.types.Tool]
+    skills: list = field(default_factory=list)  # list[skills_catalog.Skill]
+    # Why the declared skills could not be resolved, when they couldn't.
+    skills_error: str | None = None
 
     @property
     def n(self) -> int:
@@ -66,6 +75,18 @@ def tool_names(self) -> list[str]:
         names = (self._tool_field(tool, "name") for tool in self.tools)
         return [name for name in names if name]
 
+    @property
+    def skill_names(self) -> list[str]:
+        return list(dict.fromkeys(skill.name for skill in self.skills if skill.name))
+
+    @property
+    def script_names(self) -> list[str]:
+        """Every script shipped by a catalog skill, de-duplicated across skills."""
+        scripts = []
+        for skill in self.skills:
+            scripts.extend(skill.scripts)
+        return list(dict.fromkeys(scripts))
+
     @property
     def tool_names_str(self) -> str:
         return ", ".join(self.tool_names) or "(none provided)"

evalbench/scorers/dataset_quality/cuj_diversity.py

@@ -77,8 +77,8 @@ def run(self, context: DatasetQualityContext) -> SubScoreContribution:
         return SubScoreContribution(
             score=score,
             metrics={
-                "dq_paths_covered": len(covered),
-                "dq_paths_total": len(CUJ_PATHS),
+                "paths_covered": len(covered),
+                "paths_total": len(CUJ_PATHS),
             },
             suggestions=suggestions,
             example_prompts=(

evalbench/scorers/dataset_quality/error_recovery.py

@@ -76,8 +76,8 @@ def run(self, context: DatasetQualityContext) -> SubScoreContribution:
         return SubScoreContribution(
             score=score,
             metrics={
-                "dq_error_modes_covered": len(covered),
-                "dq_error_modes_total": len(ERROR_RECOVERY_MODES),
+                "error_modes_covered": len(covered),
+                "error_modes_total": len(ERROR_RECOVERY_MODES),
             },
             suggestions=suggestions,
             example_prompts=(

evalbench/scorers/dataset_quality/naming_distribution.py

@@ -80,7 +80,7 @@ def run(self, context: DatasetQualityContext) -> SubScoreContribution:
         )
         return SubScoreContribution(
             score=score,
-            metrics={"dq_tool_named_count": n_named},
+            metrics={"tool_named_cuj_count": n_named},
             suggestions=suggestions,
             evidence={"names_tool_ids": named_ids, "intent_based_ids": intent_ids},
         )