From 1db7e15311948fbca9d87c6fa9d4139e4c3ecd5c Mon Sep 17 00:00:00 2001 From: Jace Xia Date: Wed, 18 Nov 2026 18:48:55 +0800 Subject: [PATCH] fix(agent): compare strict step parameters as key-value pairs ToolCallStepSequenceMatchGrader's strict step-by-step mode scored parameters as an F1 over a flat set of argument names and values, so a call with swapped values (origin/destination) scored 1.0, a value matched regardless of which argument it belonged to, and the same nested object with a different key order scored 0.5. Build the set from (name, value) pairs and render nested dict/list values with json.dumps(sort_keys=True). --- docs/built_in_graders/agent_graders.md | 2 +- .../tool/tool_call_step_sequence_match.py | 12 ++++---- .../test_tool_call_step_sequence_match.py | 29 +++++++++++++++++-- 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/docs/built_in_graders/agent_graders.md b/docs/built_in_graders/agent_graders.md index c5c2ab5c1..27fdddf4f 100644 --- a/docs/built_in_graders/agent_graders.md +++ b/docs/built_in_graders/agent_graders.md @@ -470,7 +470,7 @@ Evaluates multi-step tool call sequences with step-by-step alignment against ref **Scoring:** - **Strict mode with Jaccard**: Intersection over union of (tool_name, parameters) pairs. JSON object key order is ignored, including nested objects; array order remains significant. - **Loose mode with Jaccard**: Intersection over union of tool names -- **Step-by-step mode**: Average F1 score across steps. In strict mode, repeated calls to the same tool are matched one-to-one within each step to maximize total parameter similarity; a prediction is never reused. +- **Step-by-step mode**: Average F1 score across steps. In strict mode, repeated calls to the same tool are matched one-to-one within each step to maximize total parameter similarity; a prediction is never reused. Parameter similarity is the F1 over (argument name, value) pairs, so a value only matches under its own argument name; nested object key order is ignored, while array order remains significant. - Range: 0.0 (no match) to 1.0 (perfect match) **Example:** diff --git a/openjudge/graders/agent/tool/tool_call_step_sequence_match.py b/openjudge/graders/agent/tool/tool_call_step_sequence_match.py index b409a1db3..be4f9e989 100644 --- a/openjudge/graders/agent/tool/tool_call_step_sequence_match.py +++ b/openjudge/graders/agent/tool/tool_call_step_sequence_match.py @@ -156,20 +156,22 @@ def extract_reference_tool_sequence( step += 1 return step_tools - def create_tool_elements(self, tool: Dict[str, Any]) -> Set[str]: + def create_tool_elements(self, tool: Dict[str, Any]) -> Set[Tuple[str, str]]: """ Create elements for tool matching based on matching mode. Args: tool: Tool information dictionary Returns: - Set of elements representing the tool + Set of (argument name, argument value) pairs representing the tool. + Keeping each value paired with its key means swapped values do not + match, and nested objects are compared independently of key order. """ elements = set() - # Add flattened arguments params = tool.get("arguments", {}) for key, value in params.items(): - elements.add(str(key)) - elements.add(str(value)) + if isinstance(value, (dict, list)): + value = json.dumps(value, sort_keys=True, ensure_ascii=False, default=str) + elements.add((str(key), str(value))) return elements def calculate_param_similarity( diff --git a/tests/graders/agent/tool/test_tool_call_step_sequence_match.py b/tests/graders/agent/tool/test_tool_call_step_sequence_match.py index 3901b2bde..51016849a 100644 --- a/tests/graders/agent/tool/test_tool_call_step_sequence_match.py +++ b/tests/graders/agent/tool/test_tool_call_step_sequence_match.py @@ -326,8 +326,8 @@ async def test_tool_call_sequence_match_grader_recall_vs_precision(): (["A", "B"], ["A", "B"], 1.0), (["B", "A"], ["A", "B"], 1.0), (["A", "B", "C"], ["C", "A", "B"], 1.0), - (["A", "B"], ["A", "A"], 0.75), - (["A", "A"], ["A", "B"], 0.75), + (["A", "B"], ["A", "A"], 0.5), + (["A", "A"], ["A", "B"], 0.5), (["C", "B", "A"], ["A", "B"], 1.0), (["A"], ["A", "A"], 0.0), (["A", "A"], ["A", "A"], 1.0), @@ -395,7 +395,7 @@ async def test_strict_steps_keep_different_tool_names_separate(): ], ) - assert result.score == pytest.approx(0.5) + assert result.score == pytest.approx(0.0) @pytest.mark.asyncio @@ -432,3 +432,26 @@ async def test_strict_jaccard_ignores_only_object_key_order(as_json, predicted, assert result.score == expected assert result.metadata["predicted_tool_count"] == 1 assert result.metadata["reference_tool_count"] == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "predicted, reference, expected", + [ + ({"origin": "上海", "destination": "北京"}, {"origin": "北京", "destination": "上海"}, 0.0), + ({"a": 2, "b": 1}, {"a": 1, "b": 2}, 0.0), + ({"origin": "destination", "北京": "上海"}, {"origin": "北京", "destination": "上海"}, 0.0), + ({"a": 1, "b": 3}, {"a": 1, "b": 2}, 0.5), + ({"filter": {"year": 2024, "lang": "en"}}, {"filter": {"lang": "en", "year": 2024}}, 1.0), + ({"tags": ["b", "a"]}, {"tags": ["a", "b"]}, 0.0), + ], +) +async def test_strict_steps_compare_parameters_as_key_value_pairs(predicted, reference, expected): + """Values only match under their own key, and nested object key order is ignored.""" + grader = ToolCallStepSequenceMatchGrader(strict_mode=True, use_jaccard_similarity=False) + result = await grader.aevaluate( + messages=[{"role": "assistant", "tool_calls": [{"name": "lookup", "arguments": predicted}]}], + reference_tool_calls=[[{"name": "lookup", "arguments": reference}]], + ) + + assert result.score == pytest.approx(expected)