Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/built_in_graders/agent_graders.md
Original file line number Diff line number Diff line change
Expand Up @@ -470,7 +470,7 @@ Evaluates multi-step tool call sequences with step-by-step alignment against ref
**Scoring:**
- **Strict mode with Jaccard**: Intersection over union of (tool_name, parameters) pairs. JSON object key order is ignored, including nested objects; array order remains significant.
- **Loose mode with Jaccard**: Intersection over union of tool names
- **Step-by-step mode**: Average F1 score across steps. In strict mode, repeated calls to the same tool are matched one-to-one within each step to maximize total parameter similarity; a prediction is never reused.
- **Step-by-step mode**: Average F1 score across steps. In strict mode, repeated calls to the same tool are matched one-to-one within each step to maximize total parameter similarity; a prediction is never reused. Parameter similarity is the F1 over (argument name, value) pairs, so a value only matches under its own argument name; nested object key order is ignored, while array order remains significant.
- Range: 0.0 (no match) to 1.0 (perfect match)

**Example:**
Expand Down
12 changes: 7 additions & 5 deletions openjudge/graders/agent/tool/tool_call_step_sequence_match.py
Original file line number Diff line number Diff line change
Expand Up @@ -156,20 +156,22 @@ def extract_reference_tool_sequence(
step += 1
return step_tools

def create_tool_elements(self, tool: Dict[str, Any]) -> Set[str]:
def create_tool_elements(self, tool: Dict[str, Any]) -> Set[Tuple[str, str]]:
"""
Create elements for tool matching based on matching mode.
Args:
tool: Tool information dictionary
Returns:
Set of elements representing the tool
Set of (argument name, argument value) pairs representing the tool.
Keeping each value paired with its key means swapped values do not
match, and nested objects are compared independently of key order.
"""
elements = set()
# Add flattened arguments
params = tool.get("arguments", {})
for key, value in params.items():
elements.add(str(key))
elements.add(str(value))
if isinstance(value, (dict, list)):
value = json.dumps(value, sort_keys=True, ensure_ascii=False, default=str)
elements.add((str(key), str(value)))
return elements

def calculate_param_similarity(
Expand Down
29 changes: 26 additions & 3 deletions tests/graders/agent/tool/test_tool_call_step_sequence_match.py
Original file line number Diff line number Diff line change
Expand Up @@ -326,8 +326,8 @@ async def test_tool_call_sequence_match_grader_recall_vs_precision():
(["A", "B"], ["A", "B"], 1.0),
(["B", "A"], ["A", "B"], 1.0),
(["A", "B", "C"], ["C", "A", "B"], 1.0),
(["A", "B"], ["A", "A"], 0.75),
(["A", "A"], ["A", "B"], 0.75),
(["A", "B"], ["A", "A"], 0.5),
(["A", "A"], ["A", "B"], 0.5),
(["C", "B", "A"], ["A", "B"], 1.0),
(["A"], ["A", "A"], 0.0),
(["A", "A"], ["A", "A"], 1.0),
Expand Down Expand Up @@ -395,7 +395,7 @@ async def test_strict_steps_keep_different_tool_names_separate():
],
)

assert result.score == pytest.approx(0.5)
assert result.score == pytest.approx(0.0)


@pytest.mark.asyncio
Expand Down Expand Up @@ -432,3 +432,26 @@ async def test_strict_jaccard_ignores_only_object_key_order(as_json, predicted,
assert result.score == expected
assert result.metadata["predicted_tool_count"] == 1
assert result.metadata["reference_tool_count"] == 1


@pytest.mark.asyncio
@pytest.mark.parametrize(
"predicted, reference, expected",
[
({"origin": "上海", "destination": "北京"}, {"origin": "北京", "destination": "上海"}, 0.0),
({"a": 2, "b": 1}, {"a": 1, "b": 2}, 0.0),
({"origin": "destination", "北京": "上海"}, {"origin": "北京", "destination": "上海"}, 0.0),
({"a": 1, "b": 3}, {"a": 1, "b": 2}, 0.5),
({"filter": {"year": 2024, "lang": "en"}}, {"filter": {"lang": "en", "year": 2024}}, 1.0),
({"tags": ["b", "a"]}, {"tags": ["a", "b"]}, 0.0),
],
)
async def test_strict_steps_compare_parameters_as_key_value_pairs(predicted, reference, expected):
"""Values only match under their own key, and nested object key order is ignored."""
grader = ToolCallStepSequenceMatchGrader(strict_mode=True, use_jaccard_similarity=False)
result = await grader.aevaluate(
messages=[{"role": "assistant", "tool_calls": [{"name": "lookup", "arguments": predicted}]}],
reference_tool_calls=[[{"name": "lookup", "arguments": reference}]],
)

assert result.score == pytest.approx(expected)
Loading