diff --git a/examples/README.md b/examples/README.md index bb90f28e2a..b5f2adf0e7 100644 --- a/examples/README.md +++ b/examples/README.md @@ -108,3 +108,4 @@ All necessary details are provided in the comments at the top of each script. | [Agent Benchmark: HotpotQA](/examples/evaluate/agent-benchmarking/run_hotpot.py) | [ragbits-evaluate](/packages/ragbits-evaluate) | Example of how to benchmark on HotpotQA comparing `Agent` and `TodoAgent`. | | [Agent Benchmark: GAIA](/examples/evaluate/agent-benchmarking/run_gaia.py) | [ragbits-evaluate](/packages/ragbits-evaluate) | Example of how to benchmark on GAIA with tools comparing `Agent` and `TodoAgent`. | | [Agent Benchmark: HumanEval](/examples/evaluate/agent-benchmarking/run_humaneval.py) | [ragbits-evaluate](/packages/ragbits-evaluate) | Example of how to benchmark on HumanEval comparing `Agent` and `TodoAgent`. | +| [Agent Benchmark: MoreHopQA](/examples/evaluate/agent-benchmarking/run_morehopqa.py) | [ragbits-evaluate](/packages/ragbits-evaluate) | Example of how to benchmark on MoreHopQA comparing `Agent` and `TodoAgent`. | diff --git a/examples/evaluate/agent-benchmarking/example_agents.py b/examples/evaluate/agent-benchmarking/example_agents.py index 824b4079a0..6f6d557af9 100644 --- a/examples/evaluate/agent-benchmarking/example_agents.py +++ b/examples/evaluate/agent-benchmarking/example_agents.py @@ -243,6 +243,59 @@ def parse_final_answer(text: str) -> str: return candidate +class AgentMoreHopQA: + """Factory for the MoreHopQA base agent for multi-hop reasoning.""" + + @staticmethod + def add(a: int, b: int) -> int: + """Add two integers.""" + return a + b + + @staticmethod + def subtract(a: int, b: int) -> int: + """Subtract two integers.""" + return a - b + + @staticmethod + def multiply(a: int, b: int) -> int: + """Multiply two integers.""" + return a * b + + @staticmethod + def divide(a: int, b: int) -> float: + """Divide two integers as float.""" + if b == 0: + raise ValueError("Cannot divide by zero.") + return a / b + + @staticmethod + def build_system_prompt() -> str: + """Return the MoreHopQA system prompt.""" + return "You are a helpful AI assistant. Answer the question concisely and accurately.\n" + + @classmethod + def build_tools(cls) -> list[Callable[..., Any]]: + """Return the callable toolset used by the MoreHopQA agent.""" + tools: list[Callable[..., Any]] = [ + cls.add, + cls.subtract, + cls.multiply, + cls.divide, + ] + return tools + + @classmethod + def build(cls) -> Agent: + """Build the MoreHopQA agent.""" + agent: Agent = Agent( + llm=LiteLLM("gpt-4.1-mini"), + prompt=cls.build_system_prompt(), + tools=cls.build_tools(), + default_options=AgentOptions(max_turns=30), + ) + return agent + + # Export agent instances for import humaneval_agent: Agent = AgentHumanEval.build() humaneval_planning_agent: Agent = build_planning_agent( @@ -262,3 +315,10 @@ def parse_final_answer(text: str) -> str: tools=AgentGAIA.build_tools(), max_turns=50, ) + +morehopqa_agent: Agent = AgentMoreHopQA.build() +morehopqa_planning_agent: Agent = build_planning_agent( + base_prompt=AgentMoreHopQA.build_system_prompt(), + tools=AgentMoreHopQA.build_tools(), + max_turns=50, +) diff --git a/examples/evaluate/agent-benchmarking/run_morehopqa.py b/examples/evaluate/agent-benchmarking/run_morehopqa.py new file mode 100644 index 0000000000..6c7a092396 --- /dev/null +++ b/examples/evaluate/agent-benchmarking/run_morehopqa.py @@ -0,0 +1,120 @@ +""" +Ragbits Evaluation Example: MoreHopQA + +This example evaluates MoreHopQA using any agent provided via a path compatible with +the CLI's `import_agent_from_path` utility, e.g.: + +uv run python examples/evaluate/agent-benchmarking/run_morehopqa.py \ + --agent_path examples/evaluate/agent-benchmarking/example_agents.py:morehopqa_agent + +You can also specify a range of examples: + +uv run python examples/evaluate/agent-benchmarking/run_morehopqa.py \ + --agent_path examples/evaluate/agent-benchmarking/example_agents.py:morehopqa_agent \ + --start_idx 0 \ + --end_idx 10 + +To use the planning-enabled variant, pass `morehopqa_planning_agent`. The script will +conditionally print planning stats when the agent is planning-based and extended logs +are enabled. + +MoreHopQA is a multi-hop reasoning benchmark that tests an agent's ability to follow +chains of reasoning over several steps. +""" + +# /// script +# requires-python = ">=3.10" +# dependencies = [ +# "ragbits-agents", +# "ragbits-core", +# "ragbits-evaluate", +# ] +# /// + +import argparse +import asyncio +import logging +from pathlib import Path + +from ragbits.agents import Agent +from ragbits.agents.cli import import_agent_from_path +from ragbits.core.sources.git import GitSource +from ragbits.evaluate.dataloaders.morehopqa import MoreHopQADataLoader +from ragbits.evaluate.evaluator import Evaluator +from ragbits.evaluate.metrics.base import MetricSet +from ragbits.evaluate.metrics.morehopqa import MoreHopQAEfficiency, MoreHopQAOutcome, MoreHopQATooling +from ragbits.evaluate.pipelines.morehopqa import MoreHopQAPipeline + + +async def main( + agent_path: str | None, + start_idx: int | None, + end_idx: int | None, +) -> None: + """Run MoreHopQA evaluation with a pluggable agent.""" + logging.getLogger("LiteLLM").setLevel(logging.ERROR) + + default_agent_path = "examples/evaluate/agent-benchmarking/example_agents.py:morehopqa_agent" + evaluation_target: Agent = import_agent_from_path(agent_path or default_agent_path) + + source = GitSource( + repo_url="https://github.com/Alab-NII/morehopqa.git", file_path="datasets/files/morehopqa_final_150samples.json" + ) + + dataloader = MoreHopQADataLoader( + source=source, + split="data", + start_idx=start_idx, + end_idx=end_idx, + ) + + # Pipeline + use_planning = "planning" in (agent_path or default_agent_path) + log_file = "morehopqa_planning_examples.ndjson" if use_planning else "morehopqa_examples.ndjson" + log_path = Path(__file__).with_name(log_file) + + pipeline = MoreHopQAPipeline( + evaluation_target=evaluation_target, + per_example_log_file=log_path, + extended_logs=use_planning, + ) + + # Metrics + metrics = MetricSet(MoreHopQAOutcome(), MoreHopQATooling(), MoreHopQAEfficiency()) + + evaluator = Evaluator(batch_size=5, parallelize_batches=True) + results = await evaluator.compute(pipeline=pipeline, dataloader=dataloader, metricset=metrics) + + print("\nMetrics:") + for key, value in results.metrics.items(): + print(f" {key}: {value:.4f}") + + if use_planning: + from utils import print_planning_stats + + await print_planning_stats(log_path, dataloader) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="MoreHopQA evaluation") + parser.add_argument( + "--agent_path", + type=str, + default=None, + help=("Path to agent 'path/to/file.py:var'. " "Defaults to exported morehopqa_agent in example_agents."), + ) + parser.add_argument( + "--start_idx", + type=int, + default=None, + help="Starting index for examples to evaluate. Defaults to 0.", + ) + parser.add_argument( + "--end_idx", + type=int, + default=None, + help="Ending index for examples to evaluate. If not specified, all examples are used.", + ) + args = parser.parse_args() + + asyncio.run(main(args.agent_path, args.start_idx, args.end_idx)) diff --git a/packages/ragbits-evaluate/CHANGELOG.md b/packages/ragbits-evaluate/CHANGELOG.md index b0020e412d..88523152d2 100644 --- a/packages/ragbits-evaluate/CHANGELOG.md +++ b/packages/ragbits-evaluate/CHANGELOG.md @@ -14,6 +14,8 @@ - Feat: add support for response adapters from `ragbits.chat.adapters` in agent simulation, enabling production chat interfaces to be used directly without wrapper classes +- Feat: add MoreHopQA multi-hop reasoning benchmark for agent evaluation with dataloader, metrics, and evaluation pipeline + ## 1.6.2 (2026-03-26) - ragbits-document-search updated to version v1.6.2 diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/dataloaders/morehopqa.py b/packages/ragbits-evaluate/src/ragbits/evaluate/dataloaders/morehopqa.py new file mode 100644 index 0000000000..d44c109361 --- /dev/null +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/dataloaders/morehopqa.py @@ -0,0 +1,93 @@ +import json +from collections.abc import Iterable + +from ragbits.core.sources.base import Source +from ragbits.evaluate.dataloaders.base import DataLoader +from ragbits.evaluate.pipelines.morehopqa import MoreHopQAData + + +class MoreHopQADataLoader(DataLoader[MoreHopQAData]): + """ + MoreHopQA benchmark evaluation data loader. + + The source should point to a local/remote file exported from the + GitHub repository. Rows are expected to contain at least: + - "question" (str) - the question + - "answer" (str) - the correct answer + """ + + def __init__( + self, + source: Source, + *, + split: str = "data", + question_key: str = "question", + answer_key: str = "answer", + start_idx: int | None = None, + end_idx: int | None = None, + ) -> None: + """ + Initialize the MoreHopQA data loader. + + Args: + source: The source to load the data from. + split: The split to load the data from (file name generated by the source helper). + question_key: Column name for the question. + answer_key: Column name for the answer. + start_idx: The starting index for the examples to load. If None, starts from 0. + end_idx: The ending index for the examples to load. If None, loads all examples. + """ + required = {question_key, answer_key} + super().__init__(source=source, split=split, required_keys=required) + self.question_key = question_key + self.answer_key = answer_key + self.start_idx = start_idx + self.end_idx = end_idx + + async def load(self) -> Iterable[MoreHopQAData]: + """ + Load the data from a JSON array file. + + Returns: + The loaded evaluation data. + """ + data_path = await self.source.fetch() + with open(data_path, encoding="utf-8") as f: + dataset = json.load(f) + + if not isinstance(dataset, list): + raise ValueError(f"Expected JSON array, got {type(dataset).__name__}") + + return await self.map(dataset) + + async def map(self, dataset: Iterable[dict]) -> Iterable[MoreHopQAData]: + """ + Map the dataset to the MoreHopQA evaluation data schema. + + Args: + dataset: The dataset to map. + + Returns: + The MoreHopQA evaluation data rows. + """ + # Apply slicing if start_idx or end_idx is specified + start = self.start_idx if self.start_idx is not None else 0 + dataset_list = list(dataset) + end = self.end_idx if self.end_idx is not None else len(dataset_list) + dataset = dataset_list[start:end] + + result = [] + for idx, row in enumerate(dataset): + task_id = str(idx) + question = str(row.get(self.question_key, "")) + answer = str(row.get(self.answer_key, "")) + + result.append( + MoreHopQAData( + task_id=task_id, + question=question, + answer=answer, + ) + ) + + return result diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/metrics/morehopqa.py b/packages/ragbits-evaluate/src/ragbits/evaluate/metrics/morehopqa.py new file mode 100644 index 0000000000..9c65b694b4 --- /dev/null +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/metrics/morehopqa.py @@ -0,0 +1,84 @@ +from statistics import mean + +from ragbits.evaluate.metrics.base import Metric +from ragbits.evaluate.pipelines.morehopqa import MoreHopQAResult + + +class MoreHopQAOutcome(Metric[MoreHopQAResult]): + """ + Computes task success rate over MoreHopQA tasks. + Measures the fraction of tasks that were successfully solved. + """ + + @staticmethod + async def compute(results: list[MoreHopQAResult]) -> dict: + """Compute task success rate. + + Returns: + Dictionary with morehopqa_task_success_rate: fraction of successfully solved tasks. + """ + success_count = sum(1 for r in results if r.task_success) + success_rate = (success_count / len(results)) if results else 0.0 + + return {"morehopqa_task_success_rate": float(success_rate)} + + +class MoreHopQATooling(Metric[MoreHopQAResult]): + """ + Tool utilization and performance metrics: + - morehopqa_tool_trigger_rate: fraction of tasks where tools were used + - morehopqa_avg_num_tool_calls: average number of tool calls per task + - morehopqa_avg_tool_error_count: average number of tool errors per task + - averaged_freq: average tool usage/calls per task + """ + + @staticmethod + async def compute(results: list[MoreHopQAResult]) -> dict: + """Compute tool utilization and performance metrics. + + Returns: + Dictionary with tool trigger rate, average tool calls, average errors, + and flattened tool frequency usage as numeric metrics. + """ + tool_triggered_count = sum(1 for r in results if r.tool_triggered) + tool_trigger_rate = (tool_triggered_count / len(results)) if results else 0.0 + avg_tool_calls = float(mean(r.num_tool_calls for r in results)) if results else 0.0 + avg_tool_errors = float(mean(r.tool_error_count for r in results)) if results else 0.0 + + # tool frequency as average per task (mean calls per task per tool) + total_tasks = len(results) if results else 1 + aggregated_counts: dict[str, int] = {} + for r in results: + if r.tool_names: + for name in r.tool_names: + aggregated_counts[name] = aggregated_counts.get(name, 0) + 1 + averaged_freq: dict[str, float] = { + f"morehopqa_tool_frequency_usage.{name}": (count / total_tasks) for name, count in aggregated_counts.items() + } + + return { + "morehopqa_tool_trigger_rate": float(tool_trigger_rate), + "morehopqa_avg_num_tool_calls": avg_tool_calls, + "morehopqa_avg_tool_error_count": avg_tool_errors, + **averaged_freq, + } + + +class MoreHopQAEfficiency(Metric[MoreHopQAResult]): + """ + Efficiency and resource usage metrics: + - morehopqa_avg_latency_ms: average response latency in milliseconds + """ + + @staticmethod + async def compute(results: list[MoreHopQAResult]) -> dict: + """Compute efficiency and resource usage metrics. + + Returns: + Dictionary with average latency. + """ + avg_latency = float(mean(r.total_latency_ms for r in results)) if results else 0.0 + + return { + "morehopqa_avg_latency_ms": avg_latency, + } diff --git a/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/morehopqa.py b/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/morehopqa.py new file mode 100644 index 0000000000..1412a13eae --- /dev/null +++ b/packages/ragbits-evaluate/src/ragbits/evaluate/pipelines/morehopqa.py @@ -0,0 +1,234 @@ +import json +import logging +import time +from collections.abc import Iterable +from dataclasses import dataclass +from pathlib import Path +from typing import cast + +from typing_extensions import Self + +from ragbits.agents import Agent +from ragbits.core.llms.base import LLM, LLMClientOptionsT, LLMResponseWithMetadata, Usage +from ragbits.evaluate.pipelines.base import EvaluationData, EvaluationPipeline, EvaluationResult + + +class MoreHopQAData(EvaluationData): + """ + Represents a single MoreHopQA task. + """ + + task_id: str + question: str + answer: str + + +@dataclass +class MoreHopQAResult(EvaluationResult): + """ + Represents the result of evaluating a single MoreHopQA task. + """ + + task_id: str + question: str + answer: str + predicted_result: str + task_success: bool + tool_triggered: bool + num_tool_calls: int + tool_error_count: int + total_latency_ms: int + usage: Usage | None = None + tool_names: list[str] | None = None + + +class MoreHopQAPipeline( + EvaluationPipeline[Agent[LLMClientOptionsT, None, str] | LLM[LLMClientOptionsT], MoreHopQAData, MoreHopQAResult] +): + """MoreHopQA evaluation pipeline for multi-hop question answering models/agents.""" + + def __init__( + self, + evaluation_target: Agent[LLMClientOptionsT, None, str] | LLM[LLMClientOptionsT], + *, + system_prompt: str | None = None, + per_example_log_file: Path | None = None, + extended_logs: bool = False, + ) -> None: + super().__init__(evaluation_target=evaluation_target) + self.system_prompt = system_prompt + self.per_example_log_file = per_example_log_file + self.extended_logs = extended_logs + self._init_log_file() + + @classmethod + def from_config(cls, config: dict) -> Self: + """Create pipeline from config. + Attempts Agent first, falls back to raw LLM construction. + """ + if "evaluation_target" not in config: + try: + config["evaluation_target"] = Agent.from_config(config) + except Exception: + config["evaluation_target"] = LLM.from_config(config) + return super().from_config(config) + + @staticmethod + def _count_tool_errors(tool_calls: list) -> int: + """Count tool errors from tool_calls.""" + tool_error_count = 0 + for call in tool_calls: + try: + if isinstance(call.result, dict) and "error" in call.result: + tool_error_count += 1 + except Exception as exc: + logging.getLogger(__name__).debug("Error while parsing tool call result: %s", exc) + return tool_error_count + + @staticmethod + def _extract_tool_names(tool_calls: list) -> list[str] | None: + """Extract tool names from tool_calls.""" + if not tool_calls: + return None + tool_names = [] + for call in tool_calls: + try: + name = getattr(call, "name", None) + if name is None and isinstance(call, dict): + name = call.get("name") + if name: + tool_names.append(str(name)) + except Exception as exc: + logging.getLogger(__name__).debug("Tool name extraction error: %s", exc) + return tool_names + + async def __call__(self, data: Iterable[MoreHopQAData]) -> Iterable[MoreHopQAResult]: + """Generate answer completions per task and evaluate them. + Returns list of `MoreHopQAResult`, one per input task. + """ + results: list[MoreHopQAResult] = [] + + for row in data: + start_time = time.perf_counter() + + prompt_input = row.question + debug_traces: list[dict | None] | None = [] if self.extended_logs else None + + usage: Usage | None = None + try: + if self.extended_logs: + content, dbg = await self._generate_with_debug(prompt_input) + if debug_traces is not None: + debug_traces.append(dbg) + tool_calls = cast(list, (dbg or {}).get("tool_calls") or []) + usage = (dbg or {}).get("usage") + else: + content, usage, tool_calls = await self._generate_answer(prompt_input) + + except Exception as generation_exc: + content = "" + tool_calls = [] + usage = None + err_msg = f"GenerationError: {generation_exc.__class__.__name__}: {generation_exc}" + if self.extended_logs and debug_traces is not None: + debug_traces.append({"error": err_msg}) + + end_time = time.perf_counter() + + # Compute metrics + predicted = str(content).strip() + + # Check if the answer is in the predicted result (normalized) + task_success = self._normalize(row.answer) in self._normalize(predicted) + + tool_triggered = bool(tool_calls) + num_tool_calls = len(tool_calls) + tool_error_count = MoreHopQAPipeline._count_tool_errors(tool_calls) + tool_names = MoreHopQAPipeline._extract_tool_names(tool_calls) + total_latency_ms = int((end_time - start_time) * 1000) + + result = MoreHopQAResult( + task_id=row.task_id, + question=row.question, + answer=row.answer, + predicted_result=content, + task_success=task_success, + tool_triggered=tool_triggered, + num_tool_calls=num_tool_calls, + tool_error_count=tool_error_count, + total_latency_ms=total_latency_ms, + usage=usage, + tool_names=tool_names, + ) + results.append(result) + ext_log_str = ( + json.dumps(debug_traces, ensure_ascii=False, default=str) + if (self.extended_logs and debug_traces is not None) + else None + ) + self._log_example(row, result, ext_log_str) + + return results + + def _init_log_file(self) -> None: + """Ensure the per-example log file exists if logging is enabled.""" + if self.per_example_log_file is None: + return + self.per_example_log_file.parent.mkdir(parents=True, exist_ok=True) + with open(self.per_example_log_file, "w", encoding="utf-8") as _: + pass + + def _log_example(self, row: MoreHopQAData, result: MoreHopQAResult, extended_log: str | None = None) -> None: + """Append a single NDJSON record for debugging if enabled.""" + if self.per_example_log_file is None: + return + # per-task tool frequency map from tool names + tool_frequency_usage: dict[str, int] = {} + if result.tool_names: + for name in result.tool_names: + tool_frequency_usage[name] = tool_frequency_usage.get(name, 0) + 1 + record: dict[str, object] = { + "task_id": row.task_id, + "question": row.question, + "answer": row.answer, + "predicted": str(result.predicted_result), + "task_success": result.task_success, + "tool_triggered": result.tool_triggered, + "num_tool_calls": result.num_tool_calls, + "tool_error_count": result.tool_error_count, + "total_latency_ms": result.total_latency_ms, + "tool_frequency_usage": tool_frequency_usage, + } + record["extended_debug_logging"] = extended_log or "[]" + with open(self.per_example_log_file, "a", encoding="utf-8") as f: + f.write(json.dumps(record, ensure_ascii=False) + "\n") + + async def _generate_answer(self, prompt: str) -> tuple[str, Usage, list]: + """Generate final answer from Agent or raw LLM and capture usage and tool calls.""" + target = self.evaluation_target + if isinstance(target, Agent): + res = await target.run(prompt) + return str(res.content), res.usage, (res.tool_calls or []) + + resp = cast(LLMResponseWithMetadata[str], await target.generate_with_metadata(prompt)) + return str(resp.content), (resp.usage or Usage()), [] + + async def _generate_with_debug(self, prompt: str) -> tuple[str, dict | None]: + """Generate answer and capture tool/history/usage for logging (as raw content).""" + target = self.evaluation_target + if isinstance(target, Agent): + res = await target.run(prompt) + dbg = { + "history": res.history, + "tool_calls": res.tool_calls, + "usage": res.usage, + "metadata": res.metadata, + } + return str(res.content), dbg + resp = await target.generate(prompt) + return str(resp), None + + @staticmethod + def _normalize(text: str) -> str: + """Basic normalization for answer equality checks: lowercase, strip spaces.""" + return "".join(ch.lower() for ch in text.strip() if not ch.isspace())