feat: add MoreHopQA multi-hop reasoning benchmark

examples/README.md

@@ -108,3 +108,4 @@ All necessary details are provided in the comments at the top of each script.
 | [Agent Benchmark: HotpotQA](/examples/evaluate/agent-benchmarking/run_hotpot.py) | [ragbits-evaluate](/packages/ragbits-evaluate) | Example of how to benchmark on HotpotQA comparing `Agent` and `TodoAgent`. |
 | [Agent Benchmark: GAIA](/examples/evaluate/agent-benchmarking/run_gaia.py)       | [ragbits-evaluate](/packages/ragbits-evaluate) | Example of how to benchmark on GAIA with tools comparing `Agent` and `TodoAgent`. |
 | [Agent Benchmark: HumanEval](/examples/evaluate/agent-benchmarking/run_humaneval.py) | [ragbits-evaluate](/packages/ragbits-evaluate) | Example of how to benchmark on HumanEval comparing `Agent` and `TodoAgent`. |
+| [Agent Benchmark: MoreHopQA](/examples/evaluate/agent-benchmarking/run_morehopqa.py) | [ragbits-evaluate](/packages/ragbits-evaluate) | Example of how to benchmark on MoreHopQA comparing `Agent` and `TodoAgent`. |

examples/evaluate/agent-benchmarking/example_agents.py

@@ -243,6 +243,59 @@ def parse_final_answer(text: str) -> str:
         return candidate
 
 
+class AgentMoreHopQA:
+    """Factory for the MoreHopQA base agent for multi-hop reasoning."""
+
+    @staticmethod
+    def add(a: int, b: int) -> int:
+        """Add two integers."""
+        return a + b
+
+    @staticmethod
+    def subtract(a: int, b: int) -> int:
+        """Subtract two integers."""
+        return a - b
+
+    @staticmethod
+    def multiply(a: int, b: int) -> int:
+        """Multiply two integers."""
+        return a * b
+
+    @staticmethod
+    def divide(a: int, b: int) -> float:
+        """Divide two integers as float."""
+        if b == 0:
+            raise ValueError("Cannot divide by zero.")
+        return a / b
+
+    @staticmethod
+    def build_system_prompt() -> str:
+        """Return the MoreHopQA system prompt."""
+        return "You are a helpful AI assistant. Answer the question concisely and accurately.\n"
+
+    @classmethod
+    def build_tools(cls) -> list[Callable[..., Any]]:
+        """Return the callable toolset used by the MoreHopQA agent."""
+        tools: list[Callable[..., Any]] = [
+            cls.add,
+            cls.subtract,
+            cls.multiply,
+            cls.divide,
+        ]
+        return tools
+
+    @classmethod
+    def build(cls) -> Agent:
+        """Build the MoreHopQA agent."""
+        agent: Agent = Agent(
+            llm=LiteLLM("gpt-4.1-mini"),
+            prompt=cls.build_system_prompt(),
+            tools=cls.build_tools(),
+            default_options=AgentOptions(max_turns=30),
+        )
+        return agent
+
+
 # Export agent instances for import
 humaneval_agent: Agent = AgentHumanEval.build()
 humaneval_planning_agent: Agent = build_planning_agent(
@@ -262,3 +315,10 @@ def parse_final_answer(text: str) -> str:
     tools=AgentGAIA.build_tools(),
     max_turns=50,
 )
+
+morehopqa_agent: Agent = AgentMoreHopQA.build()
+morehopqa_planning_agent: Agent = build_planning_agent(
+    base_prompt=AgentMoreHopQA.build_system_prompt(),
+    tools=AgentMoreHopQA.build_tools(),
+    max_turns=50,
+)

examples/evaluate/agent-benchmarking/run_morehopqa.py

@@ -0,0 +1,120 @@
+"""
+Ragbits Evaluation Example: MoreHopQA
+
+This example evaluates MoreHopQA using any agent provided via a path compatible with
+the CLI's `import_agent_from_path` utility, e.g.:
+
+uv run python examples/evaluate/agent-benchmarking/run_morehopqa.py \
+    --agent_path examples/evaluate/agent-benchmarking/example_agents.py:morehopqa_agent
+
+You can also specify a range of examples:
+
+uv run python examples/evaluate/agent-benchmarking/run_morehopqa.py \
+    --agent_path examples/evaluate/agent-benchmarking/example_agents.py:morehopqa_agent \
+    --start_idx 0 \
+    --end_idx 10
+
+To use the planning-enabled variant, pass `morehopqa_planning_agent`. The script will
+conditionally print planning stats when the agent is planning-based and extended logs
+are enabled.
+
+MoreHopQA is a multi-hop reasoning benchmark that tests an agent's ability to follow
+chains of reasoning over several steps.
+"""
+
+# /// script
+# requires-python = ">=3.10"
+# dependencies = [
+#     "ragbits-agents",
+#     "ragbits-core",
+#     "ragbits-evaluate",
+# ]
+# ///
+
+import argparse
+import asyncio
+import logging
+from pathlib import Path
+
+from ragbits.agents import Agent
+from ragbits.agents.cli import import_agent_from_path
+from ragbits.core.sources.git import GitSource
+from ragbits.evaluate.dataloaders.morehopqa import MoreHopQADataLoader
+from ragbits.evaluate.evaluator import Evaluator
+from ragbits.evaluate.metrics.base import MetricSet
+from ragbits.evaluate.metrics.morehopqa import MoreHopQAEfficiency, MoreHopQAOutcome, MoreHopQATooling
+from ragbits.evaluate.pipelines.morehopqa import MoreHopQAPipeline
+
+
+async def main(
+    agent_path: str | None,
+    start_idx: int | None,
+    end_idx: int | None,
+) -> None:
+    """Run MoreHopQA evaluation with a pluggable agent."""
+    logging.getLogger("LiteLLM").setLevel(logging.ERROR)
+
+    default_agent_path = "examples/evaluate/agent-benchmarking/example_agents.py:morehopqa_agent"
+    evaluation_target: Agent = import_agent_from_path(agent_path or default_agent_path)
+
+    source = GitSource(
+        repo_url="https://github.com/Alab-NII/morehopqa.git", file_path="datasets/files/morehopqa_final_150samples.json"
+    )
+
+    dataloader = MoreHopQADataLoader(
+        source=source,
+        split="data",
+        start_idx=start_idx,
+        end_idx=end_idx,
+    )
+
+    # Pipeline
+    use_planning = "planning" in (agent_path or default_agent_path)
+    log_file = "morehopqa_planning_examples.ndjson" if use_planning else "morehopqa_examples.ndjson"
+    log_path = Path(__file__).with_name(log_file)
+
+    pipeline = MoreHopQAPipeline(
+        evaluation_target=evaluation_target,
+        per_example_log_file=log_path,
+        extended_logs=use_planning,
+    )
+
+    # Metrics
+    metrics = MetricSet(MoreHopQAOutcome(), MoreHopQATooling(), MoreHopQAEfficiency())
+
+    evaluator = Evaluator(batch_size=5, parallelize_batches=True)
+    results = await evaluator.compute(pipeline=pipeline, dataloader=dataloader, metricset=metrics)
+
+    print("\nMetrics:")
+    for key, value in results.metrics.items():
+        print(f"  {key}: {value:.4f}")
+
+    if use_planning:
+        from utils import print_planning_stats
+
+        await print_planning_stats(log_path, dataloader)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="MoreHopQA evaluation")
+    parser.add_argument(
+        "--agent_path",
+        type=str,
+        default=None,
+        help=("Path to agent 'path/to/file.py:var'. " "Defaults to exported morehopqa_agent in example_agents."),
+    )
+    parser.add_argument(
+        "--start_idx",
+        type=int,
+        default=None,
+        help="Starting index for examples to evaluate. Defaults to 0.",
+    )
+    parser.add_argument(
+        "--end_idx",
+        type=int,
+        default=None,
+        help="Ending index for examples to evaluate. If not specified, all examples are used.",
+    )
+    args = parser.parse_args()
+
+    asyncio.run(main(args.agent_path, args.start_idx, args.end_idx))

packages/ragbits-evaluate/CHANGELOG.md

@@ -14,6 +14,8 @@
 
 - Feat: add support for response adapters from `ragbits.chat.adapters` in agent simulation, enabling production chat interfaces to be used directly without wrapper classes
 
+- Feat: add MoreHopQA multi-hop reasoning benchmark for agent evaluation with dataloader, metrics, and evaluation pipeline
+
 ## 1.6.2 (2026-03-26)
 
 - ragbits-document-search updated to version v1.6.2

packages/ragbits-evaluate/src/ragbits/evaluate/dataloaders/morehopqa.py

@@ -0,0 +1,93 @@
+import json
+from collections.abc import Iterable
+
+from ragbits.core.sources.base import Source
+from ragbits.evaluate.dataloaders.base import DataLoader
+from ragbits.evaluate.pipelines.morehopqa import MoreHopQAData
+
+
+class MoreHopQADataLoader(DataLoader[MoreHopQAData]):
+    """
+    MoreHopQA benchmark evaluation data loader.
+
+    The source should point to a local/remote file exported from the
+    GitHub repository. Rows are expected to contain at least:
+    - "question" (str) - the question
+    - "answer" (str) - the correct answer
+    """
+
+    def __init__(
+        self,
+        source: Source,
+        *,
+        split: str = "data",
+        question_key: str = "question",
+        answer_key: str = "answer",
+        start_idx: int | None = None,
+        end_idx: int | None = None,
+    ) -> None:
+        """
+        Initialize the MoreHopQA data loader.
+
+        Args:
+            source: The source to load the data from.
+            split: The split to load the data from (file name generated by the source helper).
+            question_key: Column name for the question.
+            answer_key: Column name for the answer.
+            start_idx: The starting index for the examples to load. If None, starts from 0.
+            end_idx: The ending index for the examples to load. If None, loads all examples.
+        """
+        required = {question_key, answer_key}
+        super().__init__(source=source, split=split, required_keys=required)
+        self.question_key = question_key
+        self.answer_key = answer_key
+        self.start_idx = start_idx
+        self.end_idx = end_idx
+
+    async def load(self) -> Iterable[MoreHopQAData]:
+        """
+        Load the data from a JSON array file.
+
+        Returns:
+            The loaded evaluation data.
+        """
+        data_path = await self.source.fetch()
+        with open(data_path, encoding="utf-8") as f:
+            dataset = json.load(f)
+
+        if not isinstance(dataset, list):
+            raise ValueError(f"Expected JSON array, got {type(dataset).__name__}")
+
+        return await self.map(dataset)
+
+    async def map(self, dataset: Iterable[dict]) -> Iterable[MoreHopQAData]:
+        """
+        Map the dataset to the MoreHopQA evaluation data schema.
+
+        Args:
+            dataset: The dataset to map.
+
+        Returns:
+            The MoreHopQA evaluation data rows.
+        """
+        # Apply slicing if start_idx or end_idx is specified
+        start = self.start_idx if self.start_idx is not None else 0
+        dataset_list = list(dataset)
+        end = self.end_idx if self.end_idx is not None else len(dataset_list)
+        dataset = dataset_list[start:end]
+
+        result = []
+        for idx, row in enumerate(dataset):
+            task_id = str(idx)
+            question = str(row.get(self.question_key, ""))
+            answer = str(row.get(self.answer_key, ""))
+
+            result.append(
+                MoreHopQAData(
+                    task_id=task_id,
+                    question=question,
+                    answer=answer,
+                )
+            )
+
+        return result

packages/ragbits-evaluate/src/ragbits/evaluate/metrics/morehopqa.py

@@ -0,0 +1,84 @@
+from statistics import mean
+
+from ragbits.evaluate.metrics.base import Metric
+from ragbits.evaluate.pipelines.morehopqa import MoreHopQAResult
+
+
+class MoreHopQAOutcome(Metric[MoreHopQAResult]):
+    """
+    Computes task success rate over MoreHopQA tasks.
+    Measures the fraction of tasks that were successfully solved.
+    """
+
+    @staticmethod
+    async def compute(results: list[MoreHopQAResult]) -> dict:
+        """Compute task success rate.
+
+        Returns:
+            Dictionary with morehopqa_task_success_rate: fraction of successfully solved tasks.
+        """
+        success_count = sum(1 for r in results if r.task_success)
+        success_rate = (success_count / len(results)) if results else 0.0
+
+        return {"morehopqa_task_success_rate": float(success_rate)}
+
+
+class MoreHopQATooling(Metric[MoreHopQAResult]):
+    """
+    Tool utilization and performance metrics:
+    - morehopqa_tool_trigger_rate: fraction of tasks where tools were used
+    - morehopqa_avg_num_tool_calls: average number of tool calls per task
+    - morehopqa_avg_tool_error_count: average number of tool errors per task
+    - averaged_freq: average tool usage/calls per task
+    """
+
+    @staticmethod
+    async def compute(results: list[MoreHopQAResult]) -> dict:
+        """Compute tool utilization and performance metrics.
+
+        Returns:
+            Dictionary with tool trigger rate, average tool calls, average errors,
+            and flattened tool frequency usage as numeric metrics.
+        """
+        tool_triggered_count = sum(1 for r in results if r.tool_triggered)
+        tool_trigger_rate = (tool_triggered_count / len(results)) if results else 0.0
+        avg_tool_calls = float(mean(r.num_tool_calls for r in results)) if results else 0.0
+        avg_tool_errors = float(mean(r.tool_error_count for r in results)) if results else 0.0
+
+        # tool frequency as average per task (mean calls per task per tool)
+        total_tasks = len(results) if results else 1
+        aggregated_counts: dict[str, int] = {}
+        for r in results:
+            if r.tool_names:
+                for name in r.tool_names:
+                    aggregated_counts[name] = aggregated_counts.get(name, 0) + 1
+        averaged_freq: dict[str, float] = {
+            f"morehopqa_tool_frequency_usage.{name}": (count / total_tasks) for name, count in aggregated_counts.items()
+        }
+
+        return {
+            "morehopqa_tool_trigger_rate": float(tool_trigger_rate),
+            "morehopqa_avg_num_tool_calls": avg_tool_calls,
+            "morehopqa_avg_tool_error_count": avg_tool_errors,
+            **averaged_freq,
+        }
+
+
+class MoreHopQAEfficiency(Metric[MoreHopQAResult]):
+    """
+    Efficiency and resource usage metrics:
+    - morehopqa_avg_latency_ms: average response latency in milliseconds
+    """
+
+    @staticmethod
+    async def compute(results: list[MoreHopQAResult]) -> dict:
+        """Compute efficiency and resource usage metrics.
+
+        Returns:
+            Dictionary with average latency.
+        """
+        avg_latency = float(mean(r.total_latency_ms for r in results)) if results else 0.0
+
+        return {
+            "morehopqa_avg_latency_ms": avg_latency,
+        }