From c3fd286adff9106b8e2fd5ad21cf3ff2aa71aa4f Mon Sep 17 00:00:00 2001
From: WatchTree-19 <119982314+WatchTree-19@users.noreply.github.com>
Date: Fri, 25 Sep 2026 20:41:51 +0000
Subject: [PATCH 1/3] FEAT: add MessageToolCallScorer for tool calls recorded
 in stored messages

Scores the ToolsCalled condition from function_call and function_call_output
pieces, with no trace pipeline. A tool counts when a model-authored call is
paired by call ID with a later output in the conversation through the scored
response. PyRIT tolerant-mode dispatch errors do not count. Stored messages
are partial evidence, so the scorer returns true or undetermined, never false.

Handles both the Responses and Chat Completions call formats. No observation
payload or replay yet; that needs a message-scoped tool-event payload.
---
 doc/code/scoring/5_tool_call_scorer.ipynb     |  24 ++
 doc/code/scoring/5_tool_call_scorer.py        |  18 +
 pyrit/score/__init__.py                       |   2 +
 .../true_false/message_tool_call_scorer.py    | 153 ++++++++
 .../score/test_message_tool_call_scorer.py    | 354 ++++++++++++++++++
 5 files changed, 551 insertions(+)
 create mode 100644 pyrit/score/true_false/message_tool_call_scorer.py
 create mode 100644 tests/unit/score/test_message_tool_call_scorer.py

diff --git a/doc/code/scoring/5_tool_call_scorer.ipynb b/doc/code/scoring/5_tool_call_scorer.ipynb
index 79b405599a..4a0f421850 100644
--- a/doc/code/scoring/5_tool_call_scorer.ipynb
+++ b/doc/code/scoring/5_tool_call_scorer.ipynb
@@ -387,6 +387,30 @@
     "The caller owns instrumentation and trace retrieval. Automatic request\n",
     "correlation does not install a remote collector or a backend adapter."
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0db338ce",
+   "metadata": {},
+   "source": [
+    "## Score tool calls without traces\n",
+    "\n",
+    "`MessageToolCallScorer` reads the same `ToolsCalled` condition from stored\n",
+    "messages instead of spans. A tool counts only when a model-authored\n",
+    "`function_call` piece is paired, by call ID, with a later `function_call_output`\n",
+    "piece in the conversation through the scored response. PyRIT's own\n",
+    "\"function not found\" and \"malformed arguments\" outputs do not count, because\n",
+    "the function never ran.\n",
+    "Injected history in the `simulated_assistant` and `simulated_tool` roles does\n",
+    "not count either.\n",
+    "\n",
+    "Stored messages are partial evidence. Hosted tools, several response section\n",
+    "types, and targets that execute tools themselves leave no output pieces, so the\n",
+    "scorer returns true or undetermined and never false. `OpenAIResponseTarget`\n",
+    "records these pieces when it runs `custom_functions`. Combine it with\n",
+    "`OtelToolCallScorer` under `TrueFalseScoreAggregator.OR` when trace evidence is\n",
+    "also available."
+   ]
   }
  ],
  "metadata": {
diff --git a/doc/code/scoring/5_tool_call_scorer.py b/doc/code/scoring/5_tool_call_scorer.py
index 3682b84f27..98f455c849 100644
--- a/doc/code/scoring/5_tool_call_scorer.py
+++ b/doc/code/scoring/5_tool_call_scorer.py
@@ -270,3 +270,21 @@ async def local_agent_async(request: httpx.Request) -> httpx.Response:
 # Other sources can use the same protocol with their own scorable types.
 # The caller owns instrumentation and trace retrieval. Automatic request
 # correlation does not install a remote collector or a backend adapter.
+
+# %% [markdown]
+# ## Score tool calls without traces
+#
+# `MessageToolCallScorer` reads the same `ToolsCalled` condition from stored
+# messages instead of spans. A tool counts only when a model-authored
+# `function_call` piece is paired, by call ID, with a later `function_call_output`
+# piece in the conversation through the scored response. PyRIT's own
+# "function not found" and "malformed arguments" outputs do not count, because
+# the function never ran. Injected history in the `simulated_assistant` and
+# `simulated_tool` roles does not count either.
+#
+# Stored messages are partial evidence. Hosted tools, several response section
+# types, and targets that execute tools themselves leave no output pieces, so the
+# scorer returns true or undetermined and never false. `OpenAIResponseTarget`
+# records these pieces when it runs `custom_functions`. Combine it with
+# `OtelToolCallScorer` under `TrueFalseScoreAggregator.OR` when trace evidence is
+# also available.
diff --git a/pyrit/score/__init__.py b/pyrit/score/__init__.py
index dd55b49da3..fef3015d68 100644
--- a/pyrit/score/__init__.py
+++ b/pyrit/score/__init__.py
@@ -90,6 +90,7 @@
     )
     from pyrit.score.true_false.local_refusal_classifier_scorer import LocalRefusalClassifierScorer
     from pyrit.score.true_false.manual_scorer import ManualScorer
+    from pyrit.score.true_false.message_tool_call_scorer import MessageToolCallScorer
     from pyrit.score.true_false.otel_tool_call_scorer import OtelToolCallScorer
     from pyrit.score.true_false.prompt_shield_scorer import PromptShieldScorer
     from pyrit.score.true_false.question_answer_scorer import QuestionAnswerScorer
@@ -207,6 +208,7 @@
     "LlamaGuardScorer": "pyrit.score.true_false.llamaguard_scorer",
     "MarkdownInjectionScorer": "pyrit.score.true_false.regex.markdown_injection",
     "ManualScorer": "pyrit.score.true_false.manual_scorer",
+    "MessageToolCallScorer": "pyrit.score.true_false.message_tool_call_scorer",
     "MessageScorableResolver": "pyrit.score.message_scorable_resolver",
     "MessageScorable": "pyrit.score.scorable",
     "MessageScorer": "pyrit.score.message_scorer",
diff --git a/pyrit/score/true_false/message_tool_call_scorer.py b/pyrit/score/true_false/message_tool_call_scorer.py
new file mode 100644
index 0000000000..24c64e4360
--- /dev/null
+++ b/pyrit/score/true_false/message_tool_call_scorer.py
@@ -0,0 +1,153 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+"""Tool invocation scoring over the function-call pieces a target stored in memory."""
+
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING, Any
+
+from pydantic import ValidationError
+
+from pyrit.models import MessageScorable, Score, ToolsCalled
+from pyrit.models.messages.tool_content import FunctionCallContent, FunctionOutputContent
+from pyrit.score.message_scorable_resolver import MessageScorableResolver
+from pyrit.score.true_false.true_false_scorer import TrueFalseScorer
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
+    from pyrit.models import ComponentIdentifier, MessagePiece, Scorable, ScoringExpectation
+
+# Outputs PyRIT writes in tolerant mode when it could not run the requested function. Each code is
+# paired with a key the dispatcher always sets, so a tool's own "error" field is not mistaken for one.
+_DISPATCH_ERRORS = {
+    "function_not_found": "missing_function",
+    "missing_function_name": "tool_call_section",
+    "malformed_arguments": "raw_arguments",
+}
+
+_INCOMPLETE_EVIDENCE_REASON = (
+    "Stored messages cannot rule out a call: hosted tools and several response section types are not "
+    "persisted, and targets that execute tools themselves store no outputs."
+)
+
+
+def _json_object(value: str) -> dict[str, Any] | None:
+    try:
+        parsed = json.loads(value)
+    except (TypeError, ValueError):
+        return None
+    return parsed if isinstance(parsed, dict) else None
+
+
+def _requested_call(piece: MessagePiece) -> tuple[str, str] | None:
+    """
+    Read the call id and function name from a model-authored function_call piece.
+
+    Simulated history is skipped: an injected call is not a request this run made.
+
+    Returns:
+        tuple[str, str] | None: The call id and name, or None if the piece is not a readable call.
+    """
+    if piece.role != "assistant" or piece.converted_value_data_type != "function_call":
+        return None
+    try:
+        # The shared model reads both the Chat Completions and the Responses shapes.
+        content = FunctionCallContent.model_validate_json(piece.converted_value)
+        return content.validated_call_id(), content.validated_function().name
+    except (ValidationError, ValueError):
+        return None
+
+
+def _executed_call_id(piece: MessagePiece) -> str | None:
+    """
+    Read the call id of a function_call_output piece whose function actually ran.
+
+    Returns:
+        str | None: The call id, or None for other pieces and for dispatch failures.
+    """
+    if piece.role != "tool" or piece.converted_value_data_type != "function_call_output":
+        return None
+    try:
+        content = FunctionOutputContent.model_validate_json(piece.converted_value)
+    except ValidationError:
+        return None
+    output = content.output
+    result = _json_object(output) if isinstance(output, str) else output if isinstance(output, dict) else None
+    if result is not None and _DISPATCH_ERRORS.get(result.get("error")) in result:
+        return None
+    return content.call_id
+
+
+def match_message_tool_calls(*, pieces: Iterable[MessagePiece]) -> set[str]:
+    """
+    Return the names of functions that ran, pairing each request with its output by call id.
+
+    A request with no output, or whose output reports that PyRIT could not dispatch it, is not
+    counted: a model asking for a tool is not evidence that the tool was invoked.
+
+    Returns:
+        set[str]: Names of functions with an execution attempt in the given pieces.
+    """
+    requested: dict[str, str] = {}
+    executed: set[str] = set()
+    # Walking in sequence order means an output only pairs with a request made before it.
+    for piece in sorted(pieces, key=lambda piece: piece.sequence):
+        if (call := _requested_call(piece)) is not None:
+            requested.setdefault(*call)
+        elif (call_id := _executed_call_id(piece)) is not None and call_id in requested:
+            executed.add(requested[call_id])
+    return executed
+
+
+class MessageToolCallScorer(TrueFalseScorer):
+    """
+    Score tool invocations recorded as function_call and function_call_output message pieces.
+
+    This scorer needs no trace pipeline. Stored messages are partial evidence, so it reports true
+    when every required tool ran and undetermined otherwise; it never reports false. Compose it
+    with ``OtelToolCallScorer`` under an OR aggregator when trace evidence is also available.
+    """
+
+    CONDITION_TYPE = ToolsCalled
+
+    def _build_identifier(self) -> ComponentIdentifier:
+        return self._create_identifier(params={"matching_version": 1, "message_scope_version": 1})
+
+    async def _score_scorable_async(self, *, scorable: Scorable, expectation: ScoringExpectation | None) -> list[Score]:
+        if not isinstance(scorable, MessageScorable):
+            raise TypeError("MessageToolCallScorer requires a MessageScorable.")
+        condition = self._get_required_condition(expectation=expectation, condition_type=ToolsCalled)
+        message = await MessageScorableResolver().resolve_async(scorable=scorable, memory=self._memory)
+        piece = message.message_pieces[0]
+        if not piece.conversation_id or piece.sequence < 0:
+            raise ValueError("Message tool-call scoring requires a stored conversation and message sequence.")
+        conversation = await self._memory.get_message_pieces_async(conversation_id=piece.conversation_id)
+        executed = match_message_tool_calls(pieces=(p for p in conversation if p.sequence <= piece.sequence))
+        missing = [tool.name for tool in condition.tools if tool.name not in executed]
+        piece_id = self._piece_id_from_scorable(scorable)
+        names = ", ".join(tool.name for tool in condition.tools)
+        if missing:
+            return [
+                self._build_undetermined_score(
+                    rationale=(
+                        f"Stored messages show no execution of: {', '.join(missing)}. {_INCOMPLETE_EVIDENCE_REASON}"
+                    ),
+                    description="Tool invocation; successful completion is not required.",
+                    scorable=scorable,
+                    message_piece_id=piece_id,
+                )
+            ]
+        return [
+            Score(
+                score_value="true",
+                score_type="true_false",
+                score_rationale=f"Stored function call outputs show execution of all required tools: {names}.",
+                score_value_description="Tool invocation; successful completion is not required.",
+                scorer_class_identifier=self.get_identifier(),
+                scorable=scorable,
+                message_piece_id=piece_id,
+            )
+        ]
diff --git a/tests/unit/score/test_message_tool_call_scorer.py b/tests/unit/score/test_message_tool_call_scorer.py
new file mode 100644
index 0000000000..896f5303d3
--- /dev/null
+++ b/tests/unit/score/test_message_tool_call_scorer.py
@@ -0,0 +1,354 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+import json
+import uuid
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from openai.types.responses import ResponseOutputText
+
+from pyrit.executor.attack import AttackScoringConfig, PromptSendingAttack
+from pyrit.memory import SQLiteMemory
+from pyrit.models import (
+    AttackOutcome,
+    ContentScorable,
+    Message,
+    MessagePiece,
+    MessageScorable,
+    ScoreStatus,
+    ScoringExpectation,
+    ToolCallRequirement,
+    ToolsCalled,
+)
+from pyrit.prompt_target import OpenAIResponseTarget
+from pyrit.score import (
+    InMemoryTraceClient,
+    MessageToolCallScorer,
+    OtelToolCallScorer,
+    OtelTraceSource,
+    TrueFalseCompositeScorer,
+    TrueFalseScoreAggregator,
+)
+
+pytestmark = pytest.mark.usefixtures("patch_central_database")
+
+
+def _expectation(*names: str) -> ScoringExpectation:
+    return ScoringExpectation(conditions=(ToolsCalled(tools=tuple(ToolCallRequirement(name=name) for name in names)),))
+
+
+def _responses_call(call_id: str, name: str) -> dict:
+    return {"type": "function_call", "call_id": call_id, "name": name, "arguments": "{}"}
+
+
+def _chat_completions_call(call_id: str, name: str) -> dict:
+    return {"type": "function", "id": call_id, "function": {"name": name, "arguments": "{}"}}
+
+
+def _output(call_id: str, output: object) -> dict:
+    text = output if isinstance(output, str) else json.dumps(output, separators=(",", ":"))
+    return {"type": "function_call_output", "call_id": call_id, "output": text}
+
+
+class _Conversation:
+    """Store pieces in one conversation, one message per piece, in call order."""
+
+    def __init__(self, memory: SQLiteMemory) -> None:
+        self._memory = memory
+        self.conversation_id = str(uuid.uuid4())
+        self.add("user", "text", "use the tools")
+
+    def add(self, role: str, data_type: str, value: object) -> Message:
+        text = value if isinstance(value, str) else json.dumps(value, separators=(",", ":"))
+        piece = MessagePiece(
+            role=role,
+            original_value=text,
+            original_value_data_type=data_type,
+            conversation_id=self.conversation_id,
+        )
+        message = piece.to_message()
+        self._memory.add_message_to_memory(request=message)
+        return message
+
+    def call(self, call: dict, *, role: str = "assistant") -> Message:
+        return self.add(role, "function_call", call)
+
+    def output(self, call_id: str, output: object = "ok", *, role: str = "tool") -> Message:
+        return self.add(role, "function_call_output", _output(call_id, output))
+
+    def reply(self, text: str = "done") -> Message:
+        return self.add("assistant", "text", text)
+
+
+async def _score(message: Message, *names: str):
+    scorer = MessageToolCallScorer()
+    scores = await scorer.score_async(scorable=MessageScorable.from_message(message), expectation=_expectation(*names))
+    assert len(scores) == 1
+    return scores[0]
+
+
+@pytest.mark.parametrize("call_format", [_responses_call, _chat_completions_call])
+async def test_paired_output_is_true_async(sqlite_instance: SQLiteMemory, call_format) -> None:
+    conversation = _Conversation(sqlite_instance)
+    conversation.call(call_format("call_1", "lookup"))
+    conversation.output("call_1")
+    reply = conversation.reply()
+
+    score = await _score(reply, "lookup")
+
+    assert score.status == ScoreStatus.COMPLETE
+    assert score.get_value() is True
+    assert score.message_piece_id == reply.get_piece().id
+    assert score.scorable == MessageScorable.from_message(reply)
+    assert "lookup" in score.score_rationale
+
+
+@pytest.mark.parametrize("call_format", [_responses_call, _chat_completions_call])
+async def test_requested_call_without_output_is_undetermined_async(sqlite_instance: SQLiteMemory, call_format) -> None:
+    conversation = _Conversation(sqlite_instance)
+    request = conversation.call(call_format("call_1", "lookup"))
+
+    score = await _score(request, "lookup")
+
+    assert score.status == ScoreStatus.UNDETERMINED
+    assert score.score_value is None
+    assert "lookup" in score.score_rationale
+
+
+async def test_no_tool_evidence_is_undetermined_not_false_async(sqlite_instance: SQLiteMemory) -> None:
+    reply = _Conversation(sqlite_instance).reply()
+
+    score = await _score(reply, "lookup")
+
+    assert score.status == ScoreStatus.UNDETERMINED
+    assert score.score_value is None
+
+
+@pytest.mark.parametrize(
+    "dispatch_error",
+    [
+        {"error": "function_not_found", "missing_function": "lookup", "available_functions": []},
+        {"error": "missing_function_name", "tool_call_section": {"type": "function_call"}},
+        {"error": "malformed_arguments", "function": "lookup", "raw_arguments": "{"},
+    ],
+)
+async def test_tolerant_dispatch_error_is_not_execution_async(sqlite_instance: SQLiteMemory, dispatch_error) -> None:
+    conversation = _Conversation(sqlite_instance)
+    conversation.call(_responses_call("call_1", "lookup"))
+    conversation.output("call_1", dispatch_error)
+    reply = conversation.reply()
+
+    score = await _score(reply, "lookup")
+
+    assert score.status == ScoreStatus.UNDETERMINED
+
+
+@pytest.mark.parametrize("output", [{"error": "function_not_found"}, {"error": "rate limited"}, "not json", ""])
+async def test_tool_reported_error_still_counts_as_execution_async(sqlite_instance: SQLiteMemory, output) -> None:
+    conversation = _Conversation(sqlite_instance)
+    conversation.call(_responses_call("call_1", "lookup"))
+    conversation.output("call_1", output)
+    reply = conversation.reply()
+
+    assert (await _score(reply, "lookup")).get_value() is True
+
+
+async def test_output_must_pair_by_call_id_async(sqlite_instance: SQLiteMemory) -> None:
+    conversation = _Conversation(sqlite_instance)
+    conversation.call(_responses_call("call_1", "lookup"))
+    conversation.call(_responses_call("call_2", "delete"))
+    conversation.output("call_2")
+    reply = conversation.reply()
+
+    assert (await _score(reply, "delete")).get_value() is True
+    assert (await _score(reply, "lookup")).status == ScoreStatus.UNDETERMINED
+
+
+async def test_output_for_an_unknown_call_is_ignored_async(sqlite_instance: SQLiteMemory) -> None:
+    conversation = _Conversation(sqlite_instance)
+    conversation.call(_responses_call("call_1", "lookup"))
+    conversation.output("call_9")
+    reply = conversation.reply()
+
+    assert (await _score(reply, "lookup")).status == ScoreStatus.UNDETERMINED
+
+
+@pytest.mark.parametrize(("role", "data_type"), [("assistant", "function_call_output"), ("tool", "text")])
+async def test_only_tool_role_function_call_outputs_count_async(
+    sqlite_instance: SQLiteMemory, role: str, data_type: str
+) -> None:
+    conversation = _Conversation(sqlite_instance)
+    conversation.call(_responses_call("call_1", "lookup"))
+    conversation.add(role, data_type, _output("call_1", "ok"))
+    reply = conversation.reply()
+
+    assert (await _score(reply, "lookup")).status == ScoreStatus.UNDETERMINED
+
+
+async def test_output_before_its_request_is_ignored_async(sqlite_instance: SQLiteMemory) -> None:
+    conversation = _Conversation(sqlite_instance)
+    conversation.output("call_1")
+    conversation.call(_responses_call("call_1", "lookup"))
+    reply = conversation.reply()
+
+    assert (await _score(reply, "lookup")).status == ScoreStatus.UNDETERMINED
+
+
+async def test_every_required_tool_must_run_async(sqlite_instance: SQLiteMemory) -> None:
+    conversation = _Conversation(sqlite_instance)
+    conversation.call(_responses_call("call_1", "lookup"))
+    conversation.output("call_1")
+    partial = conversation.reply()
+    conversation.call(_responses_call("call_2", "summarize"))
+    conversation.output("call_2")
+    full = conversation.reply()
+
+    missing = await _score(partial, "lookup", "summarize")
+    assert missing.status == ScoreStatus.UNDETERMINED
+    assert "summarize" in missing.score_rationale
+    assert "lookup" not in missing.score_rationale.split(".")[0]
+    assert (await _score(full, "lookup", "summarize")).get_value() is True
+
+
+async def test_tool_names_match_exactly_async(sqlite_instance: SQLiteMemory) -> None:
+    conversation = _Conversation(sqlite_instance)
+    conversation.call(_responses_call("call_1", "Lookup"))
+    conversation.output("call_1")
+    reply = conversation.reply()
+
+    assert (await _score(reply, "lookup")).status == ScoreStatus.UNDETERMINED
+
+
+async def test_later_turns_are_not_evidence_for_an_earlier_message_async(sqlite_instance: SQLiteMemory) -> None:
+    conversation = _Conversation(sqlite_instance)
+    earlier = conversation.reply()
+    conversation.call(_responses_call("call_1", "lookup"))
+    conversation.output("call_1")
+
+    assert (await _score(earlier, "lookup")).status == ScoreStatus.UNDETERMINED
+
+
+async def test_other_conversations_are_not_evidence_async(sqlite_instance: SQLiteMemory) -> None:
+    other = _Conversation(sqlite_instance)
+    other.call(_responses_call("call_1", "lookup"))
+    other.output("call_1")
+    conversation = _Conversation(sqlite_instance)
+    for _ in range(3):
+        reply = conversation.reply()
+
+    assert (await _score(reply, "lookup")).status == ScoreStatus.UNDETERMINED
+
+
+@pytest.mark.parametrize("role", ["simulated_assistant", "user"])
+async def test_calls_the_model_did_not_make_are_ignored_async(sqlite_instance: SQLiteMemory, role: str) -> None:
+    conversation = _Conversation(sqlite_instance)
+    conversation.call(_responses_call("call_1", "lookup"), role=role)
+    conversation.output("call_1")
+    reply = conversation.reply()
+
+    assert (await _score(reply, "lookup")).status == ScoreStatus.UNDETERMINED
+
+
+async def test_simulated_tool_output_is_not_execution_async(sqlite_instance: SQLiteMemory) -> None:
+    conversation = _Conversation(sqlite_instance)
+    conversation.call(_responses_call("call_1", "lookup"))
+    conversation.output("call_1", role="simulated_tool")
+    reply = conversation.reply()
+
+    assert (await _score(reply, "lookup")).status == ScoreStatus.UNDETERMINED
+
+
+async def test_function_call_text_in_a_reply_is_not_a_call_async(sqlite_instance: SQLiteMemory) -> None:
+    conversation = _Conversation(sqlite_instance)
+    conversation.add("assistant", "text", _responses_call("call_1", "lookup"))
+    conversation.output("call_1")
+    reply = conversation.reply()
+
+    assert (await _score(reply, "lookup")).status == ScoreStatus.UNDETERMINED
+
+
+async def test_requires_a_tools_called_condition_async(sqlite_instance: SQLiteMemory) -> None:
+    reply = _Conversation(sqlite_instance).reply()
+
+    with pytest.raises(ValueError, match="ToolsCalled"):
+        await MessageToolCallScorer().score_async(
+            scorable=MessageScorable.from_message(reply), expectation=ScoringExpectation()
+        )
+
+
+async def test_rejects_loose_content_async() -> None:
+    with pytest.raises(RuntimeError, match="requires a MessageScorable") as error:
+        await MessageToolCallScorer().score_async(
+            scorable=ContentScorable(value="I called lookup"), expectation=_expectation("lookup")
+        )
+    assert isinstance(error.value.__cause__, TypeError)
+
+
+@pytest.mark.parametrize("executed", [True, False])
+async def test_composes_under_or_with_trace_scoring_async(sqlite_instance: SQLiteMemory, executed: bool) -> None:
+    conversation = _Conversation(sqlite_instance)
+    conversation.call(_responses_call("call_1", "lookup"))
+    if executed:
+        conversation.output("call_1")
+    reply = conversation.reply()
+    client = InMemoryTraceClient()
+    composite = TrueFalseCompositeScorer(
+        scorers=[OtelToolCallScorer(source=OtelTraceSource(trace_client=client)), MessageToolCallScorer()],
+        aggregator=TrueFalseScoreAggregator.OR,
+    )
+
+    scores = await composite.score_async(
+        scorable=MessageScorable.from_message(reply), expectation=_expectation("lookup")
+    )
+
+    assert len(scores) == 1
+    if executed:
+        assert scores[0].get_value() is True
+    else:
+        assert scores[0].status == ScoreStatus.UNDETERMINED
+    client.close()
+
+
+def _sdk_function_call(call_id: str, name: str) -> MagicMock:
+    section = MagicMock()
+    section.type = "function_call"
+    section.call_id = call_id
+    section.name = name
+    section.arguments = "{}"
+    return MagicMock(status="completed", error=None, output=[section])
+
+
+def _sdk_text(text: str) -> MagicMock:
+    section = MagicMock()
+    section.type = "message"
+    section.content = [ResponseOutputText(annotations=[], text=text, type="output_text")]
+    return MagicMock(status="completed", error=None, output=[section])
+
+
+@pytest.mark.parametrize(
+    ("registered", "outcome"),
+    [(True, AttackOutcome.SUCCESS), (False, AttackOutcome.UNDETERMINED)],
+)
+async def test_scores_the_response_target_tool_loop_async(registered: bool, outcome: AttackOutcome) -> None:
+    target = OpenAIResponseTarget(
+        model_name="gpt-4", endpoint="https://mock.azure.com", api_key="mock-key", fail_on_missing_function=False
+    )
+    if registered:
+
+        async def lookup(args: dict) -> dict:
+            return {"found": True}
+
+        target._custom_functions["lookup"] = lookup
+    attack = PromptSendingAttack(
+        objective_target=target,
+        attack_scoring_config=AttackScoringConfig(objective_scorer=MessageToolCallScorer()),
+        max_attempts_on_failure=0,
+    )
+    responses = [_sdk_function_call("call_1", "lookup"), _sdk_text("found it")]
+    with patch.object(target._async_client.responses, "create", new_callable=AsyncMock, side_effect=responses):
+        result = await attack.execute_async(objective="look it up", expectation=_expectation("lookup"))
+
+    assert result.outcome is outcome
+    assert result.automated_score is not None
+    assert result.automated_score.message_piece_id == result.last_response.id

From 412ebec6a853ce69e76badcc0c4ec98d8836c8f5 Mon Sep 17 00:00:00 2001
From: Richard Lundeen <rlundeen@microsoft.com>
Date: Thu, 1 Oct 2026 16:25:48 -0700
Subject: [PATCH 2/3] FIX: record tool execution metadata for message scoring

Separate dispatch status from tool-returned data and guard legacy structured errors. Add metadata serialization, regression coverage, and documentation.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
---
 doc/code/scoring/5_tool_call_scorer.ipynb     | 11 ++-
 doc/code/scoring/5_tool_call_scorer.py        | 12 ++-
 pyrit/models/__init__.py                      |  2 +
 .../models/target/tool_execution_metadata.py  | 36 +++++++
 .../openai/openai_response_target.py          | 69 ++++++++------
 .../true_false/message_tool_call_scorer.py    | 18 ++--
 .../models/test_tool_execution_metadata.py    | 29 ++++++
 .../target/test_mcp_tool_provider.py          |  3 +-
 .../target/test_openai_response_target.py     | 44 +++++++--
 tests/unit/prompt_target/target/test_tool.py  | 22 ++++-
 .../score/test_message_tool_call_scorer.py    | 94 +++++++++++++++++--
 11 files changed, 276 insertions(+), 64 deletions(-)
 create mode 100644 pyrit/models/target/tool_execution_metadata.py
 create mode 100644 tests/unit/models/test_tool_execution_metadata.py

diff --git a/doc/code/scoring/5_tool_call_scorer.ipynb b/doc/code/scoring/5_tool_call_scorer.ipynb
index 4a0f421850..bb7e81f6d0 100644
--- a/doc/code/scoring/5_tool_call_scorer.ipynb
+++ b/doc/code/scoring/5_tool_call_scorer.ipynb
@@ -390,7 +390,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "0db338ce",
+   "id": "14",
    "metadata": {},
    "source": [
     "## Score tool calls without traces\n",
@@ -398,9 +398,12 @@
     "`MessageToolCallScorer` reads the same `ToolsCalled` condition from stored\n",
     "messages instead of spans. A tool counts only when a model-authored\n",
     "`function_call` piece is paired, by call ID, with a later `function_call_output`\n",
-    "piece in the conversation through the scored response. PyRIT's own\n",
-    "\"function not found\" and \"malformed arguments\" outputs do not count, because\n",
-    "the function never ran.\n",
+    "piece in the conversation through the scored response. `OpenAIResponseTarget`\n",
+    "records `ToolExecutionMetadata` on each output piece. The scorer uses this\n",
+    "dispatch status, not the returned payload: a tool-reported error still counts\n",
+    "as an invocation, while a dispatch failure does not. Older outputs without\n",
+    "metadata use a conservative fallback; payloads shaped like PyRIT dispatch\n",
+    "errors cannot prove invocation and remain undetermined.\n",
     "Injected history in the `simulated_assistant` and `simulated_tool` roles does\n",
     "not count either.\n",
     "\n",
diff --git a/doc/code/scoring/5_tool_call_scorer.py b/doc/code/scoring/5_tool_call_scorer.py
index 98f455c849..d45fad0cde 100644
--- a/doc/code/scoring/5_tool_call_scorer.py
+++ b/doc/code/scoring/5_tool_call_scorer.py
@@ -277,10 +277,14 @@ async def local_agent_async(request: httpx.Request) -> httpx.Response:
 # `MessageToolCallScorer` reads the same `ToolsCalled` condition from stored
 # messages instead of spans. A tool counts only when a model-authored
 # `function_call` piece is paired, by call ID, with a later `function_call_output`
-# piece in the conversation through the scored response. PyRIT's own
-# "function not found" and "malformed arguments" outputs do not count, because
-# the function never ran. Injected history in the `simulated_assistant` and
-# `simulated_tool` roles does not count either.
+# piece in the conversation through the scored response. `OpenAIResponseTarget`
+# records `ToolExecutionMetadata` on each output piece. The scorer uses this
+# dispatch status, not the returned payload: a tool-reported error still counts
+# as an invocation, while a dispatch failure does not. Older outputs without
+# metadata use a conservative fallback; payloads shaped like PyRIT dispatch
+# errors cannot prove invocation and remain undetermined.
+# Injected history in the `simulated_assistant` and `simulated_tool` roles does
+# not count either.
 #
 # Stored messages are partial evidence. Hosted tools, several response section
 # types, and targets that execute tools themselves leave no output pieces, so the
diff --git a/pyrit/models/__init__.py b/pyrit/models/__init__.py
index bc34c01116..242b6173b8 100644
--- a/pyrit/models/__init__.py
+++ b/pyrit/models/__init__.py
@@ -229,9 +229,11 @@
         unregister_common_json_schema,
     )
     from pyrit.models.target.request_trace_context import RequestTraceContext
+    from pyrit.models.target.tool_execution_metadata import ToolExecutionMetadata
 
 _LAZY_EXPORTS: dict[str, str] = {
     "RequestTraceContext": "pyrit.models.target.request_trace_context",
+    "ToolExecutionMetadata": "pyrit.models.target.tool_execution_metadata",
     "AttackAnalyticsCell": "pyrit.models.analytics",
     "AttackAnalyticsConverterDirection": "pyrit.models.analytics",
     "AttackAnalyticsDimension": "pyrit.models.analytics",
diff --git a/pyrit/models/target/tool_execution_metadata.py b/pyrit/models/target/tool_execution_metadata.py
new file mode 100644
index 0000000000..47f9da1a83
--- /dev/null
+++ b/pyrit/models/target/tool_execution_metadata.py
@@ -0,0 +1,36 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+from __future__ import annotations
+
+from typing import Any, ClassVar
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class ToolExecutionMetadata(BaseModel):
+    """Target-recorded invocation status, separate from a tool's returned data."""
+
+    METADATA_KEY: ClassVar[str] = "pyrit_tool_execution"
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    invoked: bool = Field(strict=True)
+
+    def to_metadata(self) -> dict[str, Any]:
+        """Return the metadata fragment for a function-call output piece."""
+        return {self.METADATA_KEY: self.model_dump(mode="json")}
+
+    @classmethod
+    def from_metadata(cls, *, metadata: dict[str, Any]) -> ToolExecutionMetadata | None:
+        """
+        Read invocation status without interpreting tool-returned data.
+
+        Returns:
+            ToolExecutionMetadata | None: The recorded status, or None if absent.
+
+        Raises:
+            ValueError: If stored invocation metadata is malformed.
+        """
+        if cls.METADATA_KEY not in metadata:
+            return None
+        return cls.model_validate(metadata[cls.METADATA_KEY])
diff --git a/pyrit/prompt_target/openai/openai_response_target.py b/pyrit/prompt_target/openai/openai_response_target.py
index 393a69f898..eedcf7c0b8 100644
--- a/pyrit/prompt_target/openai/openai_response_target.py
+++ b/pyrit/prompt_target/openai/openai_response_target.py
@@ -33,6 +33,7 @@
     MessagePiece,
     PromptDataType,
     PromptResponseError,
+    ToolExecutionMetadata,
 )
 from pyrit.models.messages.chat_message import FunctionCall
 from pyrit.prompt_target.common.target_capabilities import TargetCapabilities
@@ -65,6 +66,12 @@ class _SerializedPiece:
     placement: Literal["inline", "top_level"]
 
 
+@dataclass(frozen=True)
+class _ToolDispatchResult:
+    output: object
+    invoked: bool
+
+
 class MessagePieceType(str, Enum):
     """Enumeration of different types of message pieces."""
 
@@ -661,8 +668,8 @@ async def _run_tool_call_loop_async(self, *, normalized_conversation: list[Messa
             for tool_call_section in tool_call_sections:
                 tool_output = await self._execute_call_section_async(tool_call_section)
                 tool_piece = self._make_tool_piece(
-                    tool_output,
-                    tool_call_section["call_id"],
+                    result=tool_output,
+                    call_id=tool_call_section["call_id"],
                     reference_piece=message_piece,
                 )
                 tool_message = Message(message_pieces=[tool_piece])
@@ -899,7 +906,7 @@ def _find_pending_tool_calls(self, reply: Message) -> list[dict[str, Any]]:
                     calls.append(cast("dict[str, Any]", section))
         return calls
 
-    async def _execute_call_section_async(self, tool_call_section: dict[str, Any]) -> object:
+    async def _execute_call_section_async(self, tool_call_section: dict[str, Any]) -> _ToolDispatchResult:
         """
         Execute a function call using the matching tool.
 
@@ -907,9 +914,8 @@ async def _execute_call_section_async(self, tool_call_section: dict[str, Any]) -
             tool_call_section: The function_call section dict.
 
         Returns:
-            A JSON-serializable payload that will be sent as function_call_output.
-            If fail_on_missing_function=False and a function is missing or no function is not called, returns:
-            {"error": "function_not_found", "missing_function": "<name>", "available_functions": [...]}
+            The output payload and whether the tool was invoked. Tolerant dispatch failures
+            retain their error payload with invoked=False.
 
         Raises:
             ValueError: If the function call section is missing a 'name' field.
@@ -920,10 +926,10 @@ async def _execute_call_section_async(self, tool_call_section: dict[str, Any]) -
         if not name:
             if self._fail_on_missing_function:
                 raise ValueError("Function call section missing 'name' field")
-            return {
-                "error": "missing_function_name",
-                "tool_call_section": tool_call_section,
-            }
+            return _ToolDispatchResult(
+                output={"error": "missing_function_name", "tool_call_section": tool_call_section},
+                invoked=False,
+            )
 
         args_json = tool_call_section.get("arguments", "{}")
         try:
@@ -933,50 +939,54 @@ async def _execute_call_section_async(self, tool_call_section: dict[str, Any]) -
             if self._fail_on_missing_function:
                 raise ValueError(f"Malformed arguments for function '{name}': {args_json}") from None
             logger.warning("Malformed arguments for function '%s': %s", name, args_json)
-            return {
-                "error": "malformed_arguments",
-                "function": name,
-                "raw_arguments": args_json,
-            }
+            return _ToolDispatchResult(
+                output={"error": "malformed_arguments", "function": name, "raw_arguments": args_json},
+                invoked=False,
+            )
         await self._initialize_tools_async()
         configured_tool = next((tool for tool in self._tools if tool.name == name), None)
         if configured_tool is not None:
             if not isinstance(args, dict):
                 if self._fail_on_missing_function:
                     raise ValueError(f"Arguments for function '{name}' must be a JSON object")
-                return {
-                    "error": "malformed_arguments",
-                    "function": name,
-                    "raw_arguments": args_json,
-                }
-            return await configured_tool.execute_async(arguments=args)
+                return _ToolDispatchResult(
+                    output={"error": "malformed_arguments", "function": name, "raw_arguments": args_json},
+                    invoked=False,
+                )
+            return _ToolDispatchResult(output=await configured_tool.execute_async(arguments=args), invoked=True)
 
         custom_function = self._custom_functions.get(name)
         if custom_function is not None:
-            return await custom_function(cast("dict[str, Any]", args))
+            return _ToolDispatchResult(output=await custom_function(cast("dict[str, Any]", args)), invoked=True)
 
         if self._fail_on_missing_function:
             raise KeyError(f"Function '{name}' is not registered")
         available_functions = sorted({*(tool.name for tool in self._tools), *self._custom_functions})
         logger.warning("Function '%s' not registered. Available: %s", name, available_functions)
-        return {
-            "error": "function_not_found",
-            "missing_function": name,
-            "available_functions": available_functions,
-        }
+        return _ToolDispatchResult(
+            output={
+                "error": "function_not_found",
+                "missing_function": name,
+                "available_functions": available_functions,
+            },
+            invoked=False,
+        )
 
-    def _make_tool_piece(self, output: object, call_id: str, *, reference_piece: MessagePiece) -> MessagePiece:
+    def _make_tool_piece(
+        self, *, result: _ToolDispatchResult, call_id: str, reference_piece: MessagePiece
+    ) -> MessagePiece:
         """
         Create a function_call_output MessagePiece.
 
         Args:
-            output: The tool output to wrap.
+            result: The dispatch result to record.
             call_id: The call ID for the function call.
             reference_piece: A reference piece to copy conversation context from.
 
         Returns:
             A MessagePiece containing the function call output.
         """
+        output = result.output
         output_str = output if isinstance(output, str) else json.dumps(output, separators=(",", ":"))
         return MessagePiece(
             role="tool",
@@ -986,4 +996,5 @@ def _make_tool_piece(self, output: object, call_id: str, *, reference_piece: Mes
             ),
             original_value_data_type="function_call_output",
             conversation_id=reference_piece.conversation_id,
+            prompt_metadata=ToolExecutionMetadata(invoked=result.invoked).to_metadata(),
         )
diff --git a/pyrit/score/true_false/message_tool_call_scorer.py b/pyrit/score/true_false/message_tool_call_scorer.py
index 24c64e4360..abd96d4eb8 100644
--- a/pyrit/score/true_false/message_tool_call_scorer.py
+++ b/pyrit/score/true_false/message_tool_call_scorer.py
@@ -10,7 +10,7 @@
 
 from pydantic import ValidationError
 
-from pyrit.models import MessageScorable, Score, ToolsCalled
+from pyrit.models import MessageScorable, Score, ToolExecutionMetadata, ToolsCalled
 from pyrit.models.messages.tool_content import FunctionCallContent, FunctionOutputContent
 from pyrit.score.message_scorable_resolver import MessageScorableResolver
 from pyrit.score.true_false.true_false_scorer import TrueFalseScorer
@@ -20,8 +20,7 @@
 
     from pyrit.models import ComponentIdentifier, MessagePiece, Scorable, ScoringExpectation
 
-# Outputs PyRIT writes in tolerant mode when it could not run the requested function. Each code is
-# paired with a key the dispatcher always sets, so a tool's own "error" field is not mistaken for one.
+# Legacy outputs have no dispatch metadata. Matching payloads remain ambiguous and cannot prove invocation.
 _DISPATCH_ERRORS = {
     "function_not_found": "missing_function",
     "missing_function_name": "tool_call_section",
@@ -74,9 +73,14 @@ def _executed_call_id(piece: MessagePiece) -> str | None:
         content = FunctionOutputContent.model_validate_json(piece.converted_value)
     except ValidationError:
         return None
+    execution = ToolExecutionMetadata.from_metadata(metadata=piece.prompt_metadata)
+    if execution is not None:
+        return content.call_id if execution.invoked else None
     output = content.output
     result = _json_object(output) if isinstance(output, str) else output if isinstance(output, dict) else None
-    if result is not None and _DISPATCH_ERRORS.get(result.get("error")) in result:
+    error_code = result.get("error") if result is not None else None
+    marker_key = _DISPATCH_ERRORS.get(error_code) if isinstance(error_code, str) else None
+    if result is not None and marker_key is not None and marker_key in result:
         return None
     return content.call_id
 
@@ -85,8 +89,8 @@ def match_message_tool_calls(*, pieces: Iterable[MessagePiece]) -> set[str]:
     """
     Return the names of functions that ran, pairing each request with its output by call id.
 
-    A request with no output, or whose output reports that PyRIT could not dispatch it, is not
-    counted: a model asking for a tool is not evidence that the tool was invoked.
+    A request with no output or a recorded dispatch failure is not counted. Legacy outputs shaped
+    like dispatch failures remain ambiguous: a requested call alone does not prove invocation.
 
     Returns:
         set[str]: Names of functions with an execution attempt in the given pieces.
@@ -114,7 +118,7 @@ class MessageToolCallScorer(TrueFalseScorer):
     CONDITION_TYPE = ToolsCalled
 
     def _build_identifier(self) -> ComponentIdentifier:
-        return self._create_identifier(params={"matching_version": 1, "message_scope_version": 1})
+        return self._create_identifier(params={"matching_version": 2, "message_scope_version": 1})
 
     async def _score_scorable_async(self, *, scorable: Scorable, expectation: ScoringExpectation | None) -> list[Score]:
         if not isinstance(scorable, MessageScorable):
diff --git a/tests/unit/models/test_tool_execution_metadata.py b/tests/unit/models/test_tool_execution_metadata.py
new file mode 100644
index 0000000000..51c3223743
--- /dev/null
+++ b/tests/unit/models/test_tool_execution_metadata.py
@@ -0,0 +1,29 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+import json
+
+import pytest
+from pydantic import ValidationError
+
+from pyrit.models import ToolExecutionMetadata
+
+
+@pytest.mark.parametrize("invoked", [True, False])
+def test_tool_execution_metadata_round_trip(invoked: bool) -> None:
+    execution = ToolExecutionMetadata(invoked=invoked)
+    metadata = json.loads(json.dumps({"unrelated": "retained", **execution.to_metadata()}))
+
+    assert metadata[ToolExecutionMetadata.METADATA_KEY] == {"invoked": invoked}
+    assert ToolExecutionMetadata.from_metadata(metadata=metadata) == execution
+    assert metadata["unrelated"] == "retained"
+
+
+def test_tool_execution_metadata_absent() -> None:
+    assert ToolExecutionMetadata.from_metadata(metadata={"unrelated": True}) is None
+
+
+@pytest.mark.parametrize("value", [None, True, {}, {"invoked": "false"}, {"invoked": 1}, {"invoked": True, "extra": 1}])
+def test_tool_execution_metadata_rejects_malformed_status(value: object) -> None:
+    with pytest.raises(ValidationError):
+        ToolExecutionMetadata.from_metadata(metadata={ToolExecutionMetadata.METADATA_KEY: value})
diff --git a/tests/unit/prompt_target/target/test_mcp_tool_provider.py b/tests/unit/prompt_target/target/test_mcp_tool_provider.py
index 5e7c4e8b91..27eb89a377 100644
--- a/tests/unit/prompt_target/target/test_mcp_tool_provider.py
+++ b/tests/unit/prompt_target/target/test_mcp_tool_provider.py
@@ -525,7 +525,8 @@ async def test_openai_response_target_uses_mcp_tools(patch_central_database) ->
     assert first_body["tools"] == second_body["tools"]
     assert first_body["tools"][0]["name"] == "get_note"
     provider.call_tool_async.assert_awaited_once_with(name="get_note", arguments={"id": "welcome"})
-    assert output == {"structured_content": {"text": "Welcome"}}
+    assert output.output == {"structured_content": {"text": "Welcome"}}
+    assert output.invoked is True
 
 
 async def test_openai_response_target_rejects_provider_name_conflict(patch_central_database) -> None:
diff --git a/tests/unit/prompt_target/target/test_openai_response_target.py b/tests/unit/prompt_target/target/test_openai_response_target.py
index 1566909ed9..109d000e10 100644
--- a/tests/unit/prompt_target/target/test_openai_response_target.py
+++ b/tests/unit/prompt_target/target/test_openai_response_target.py
@@ -34,10 +34,11 @@
     Message,
     MessagePiece,
     PromptDataType,
+    ToolExecutionMetadata,
     flatten_to_message_pieces,
 )
 from pyrit.prompt_target import OpenAIResponseTarget, PromptTarget
-from pyrit.prompt_target.openai.openai_response_target import token_usage_from_responses
+from pyrit.prompt_target.openai.openai_response_target import _ToolDispatchResult, token_usage_from_responses
 from pyrit.score import SelfAskRefusalScorer, TrueFalseInverterScorer
 
 
@@ -1098,14 +1099,18 @@ async def test_build_input_for_multi_modal_async_preserves_empty_conversation_er
     assert str(exc_info.value) == "Conversation cannot be empty"
 
 
-def test_make_tool_piece_serializes_output_and_sets_call_id(target: OpenAIResponseTarget):
+@pytest.mark.parametrize("invoked", [True, False])
+def test_make_tool_piece_serializes_output_and_sets_call_id(target: OpenAIResponseTarget, invoked: bool):
     out = {"answer": 42}
     reference_piece = MessagePiece(
         role="user",
         original_value="test",
         conversation_id="test-conv-123",
     )
-    piece = target._make_tool_piece(out, call_id="tool-1", reference_piece=reference_piece)
+    piece = target._make_tool_piece(
+        result=_ToolDispatchResult(output=out, invoked=invoked), call_id="tool-1", reference_piece=reference_piece
+    )
+    assert ToolExecutionMetadata.from_metadata(metadata=piece.prompt_metadata) == ToolExecutionMetadata(invoked=invoked)
     assert piece.original_value_data_type == "function_call_output"
     assert piece.conversation_id == "test-conv-123"
     payload = json.loads(piece.original_value)
@@ -1124,16 +1129,19 @@ async def add_fn(args: dict[str, Any]) -> dict[str, Any]:
 
     section = {"type": "function_call", "name": "add", "arguments": json.dumps({"a": 2, "b": 3})}
     result = await target._execute_call_section_async(section)
-    assert result == {"sum": 5}
+    assert result.output == {"sum": 5}
+    assert result.invoked is True
 
 
 async def test_execute_call_section_missing_function_tolerant_mode(target: OpenAIResponseTarget):
     # default fail_on_missing_function=False
     section = {"type": "function_call", "name": "unknown_tool", "arguments": "{}"}
     result = await target._execute_call_section_async(section)
-    assert result["error"] == "function_not_found"
-    assert result["missing_function"] == "unknown_tool"
-    assert "available_functions" in result
+    assert result.invoked is False
+    assert isinstance(result.output, dict)
+    assert result.output["error"] == "function_not_found"
+    assert result.output["missing_function"] == "unknown_tool"
+    assert "available_functions" in result.output
 
 
 async def test_execute_call_section_malformed_arguments_tolerant_mode(target: OpenAIResponseTarget):
@@ -1143,9 +1151,25 @@ async def echo_fn(args: dict[str, Any]) -> dict[str, Any]:
     target._custom_functions["echo"] = echo_fn
     section = {"type": "function_call", "name": "echo", "arguments": "{not-json"}
     result = await target._execute_call_section_async(section)
-    assert result["error"] == "malformed_arguments"
-    assert result["function"] == "echo"
-    assert result["raw_arguments"] == "{not-json"
+    assert result.invoked is False
+    assert result.output == {"error": "malformed_arguments", "function": "echo", "raw_arguments": "{not-json"}
+
+
+async def test_execute_call_section_missing_name_records_no_invocation_async(target: OpenAIResponseTarget) -> None:
+    section = {"type": "function_call", "arguments": "{}"}
+    result = await target._execute_call_section_async(section)
+
+    assert result.invoked is False
+    assert result.output == {"error": "missing_function_name", "tool_call_section": section}
+
+
+async def test_execute_call_section_preserves_tool_exception_async(target: OpenAIResponseTarget) -> None:
+    callback = AsyncMock(side_effect=ValueError("tool failed"))
+    target._custom_functions["lookup"] = callback
+
+    with pytest.raises(ValueError, match="tool failed"):
+        await target._execute_call_section_async({"name": "lookup", "arguments": "{}"})
+    callback.assert_awaited_once()
 
 
 async def test_execute_call_section_missing_function_strict_mode(target: OpenAIResponseTarget):
diff --git a/tests/unit/prompt_target/target/test_tool.py b/tests/unit/prompt_target/target/test_tool.py
index 9f68799e2e..daa893af89 100644
--- a/tests/unit/prompt_target/target/test_tool.py
+++ b/tests/unit/prompt_target/target/test_tool.py
@@ -104,7 +104,8 @@ async def test_openai_response_target_advertises_and_executes_tool(patch_central
             "strict": False,
         },
     ]
-    assert result == 5
+    assert result.output == 5
+    assert result.invoked is True
 
 
 def test_openai_response_target_without_tools_preserves_identifier(patch_central_database) -> None:
@@ -223,10 +224,27 @@ async def test_legacy_declaration_and_callback_remain_compatible(patch_central_d
     result = await target._execute_call_section_async({"name": "legacy_add", "arguments": '{"x": 2}'})
     assert body["tools"][0] == declaration
     assert len(body["tools"]) == 2
-    assert result == {"value": 3}
+    assert result.output == {"value": 3}
+    assert result.invoked is True
     callback.assert_awaited_once_with({"x": 2})
 
 
+async def test_non_object_arguments_record_no_invocation_async(patch_central_database) -> None:
+    target = OpenAIResponseTarget(
+        model_name="gpt-4",
+        endpoint="https://mock.azure.com",
+        api_key="mock-key",
+        tools=[add],
+        fail_on_missing_function=False,
+    )
+    with patch.object(add, "execute_async", new_callable=AsyncMock) as execute:
+        result = await target._execute_call_section_async({"name": "add", "arguments": "[]"})
+
+    execute.assert_not_awaited()
+    assert result.invoked is False
+    assert result.output == {"error": "malformed_arguments", "function": "add", "raw_arguments": "[]"}
+
+
 async def test_model_retries_pace_each_request_without_repeating_tools(patch_central_database) -> None:
     executed: list[int] = []
 
diff --git a/tests/unit/score/test_message_tool_call_scorer.py b/tests/unit/score/test_message_tool_call_scorer.py
index 896f5303d3..fd4b8e7896 100644
--- a/tests/unit/score/test_message_tool_call_scorer.py
+++ b/tests/unit/score/test_message_tool_call_scorer.py
@@ -3,6 +3,7 @@
 
 import json
 import uuid
+from typing import Any
 from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
@@ -19,6 +20,7 @@
     ScoreStatus,
     ScoringExpectation,
     ToolCallRequirement,
+    ToolExecutionMetadata,
     ToolsCalled,
 )
 from pyrit.prompt_target import OpenAIResponseTarget
@@ -59,13 +61,14 @@ def __init__(self, memory: SQLiteMemory) -> None:
         self.conversation_id = str(uuid.uuid4())
         self.add("user", "text", "use the tools")
 
-    def add(self, role: str, data_type: str, value: object) -> Message:
+    def add(self, role: str, data_type: str, value: object, *, metadata: dict[str, Any] | None = None) -> Message:
         text = value if isinstance(value, str) else json.dumps(value, separators=(",", ":"))
         piece = MessagePiece(
             role=role,
             original_value=text,
             original_value_data_type=data_type,
             conversation_id=self.conversation_id,
+            prompt_metadata=metadata or {},
         )
         message = piece.to_message()
         self._memory.add_message_to_memory(request=message)
@@ -144,7 +147,17 @@ async def test_tolerant_dispatch_error_is_not_execution_async(sqlite_instance: S
     assert score.status == ScoreStatus.UNDETERMINED
 
 
-@pytest.mark.parametrize("output", [{"error": "function_not_found"}, {"error": "rate limited"}, "not json", ""])
+@pytest.mark.parametrize(
+    "output",
+    [
+        {"error": "function_not_found"},
+        {"error": "rate limited"},
+        {"error": {"code": "timeout"}},
+        {"error": ["timeout"]},
+        "not json",
+        "",
+    ],
+)
 async def test_tool_reported_error_still_counts_as_execution_async(sqlite_instance: SQLiteMemory, output) -> None:
     conversation = _Conversation(sqlite_instance)
     conversation.call(_responses_call("call_1", "lookup"))
@@ -154,6 +167,55 @@ async def test_tool_reported_error_still_counts_as_execution_async(sqlite_instan
     assert (await _score(reply, "lookup")).get_value() is True
 
 
+async def test_unrelated_structured_error_does_not_abort_scoring_async(sqlite_instance: SQLiteMemory) -> None:
+    conversation = _Conversation(sqlite_instance)
+    conversation.call(_responses_call("other", "other_tool"))
+    conversation.output("other", {"error": {"code": "timeout"}})
+    conversation.call(_responses_call("call_1", "lookup"))
+    conversation.output("call_1")
+
+    assert (await _score(conversation.reply(), "lookup")).get_value() is True
+
+
+@pytest.mark.parametrize("invoked", [True, False])
+@pytest.mark.parametrize("output", ["ok", {"error": "malformed_arguments", "raw_arguments": "{"}])
+async def test_stored_execution_metadata_controls_verdict_async(
+    sqlite_instance: SQLiteMemory, invoked: bool, output: object
+) -> None:
+    conversation = _Conversation(sqlite_instance)
+    conversation.call(_responses_call("call_1", "lookup"))
+    result = conversation.add(
+        "tool",
+        "function_call_output",
+        _output("call_1", output),
+        metadata=ToolExecutionMetadata(invoked=invoked).to_metadata(),
+    )
+    stored = await sqlite_instance.get_message_pieces_async(prompt_ids=[str(result.get_piece().id)])
+    assert ToolExecutionMetadata.from_metadata(metadata=stored[0].prompt_metadata) == ToolExecutionMetadata(
+        invoked=invoked
+    )
+
+    score = await _score(conversation.reply(), "lookup")
+    if invoked:
+        assert score.get_value() is True
+    else:
+        assert score.status == ScoreStatus.UNDETERMINED
+
+
+async def test_invalid_execution_metadata_is_not_a_legacy_fallback_async(sqlite_instance: SQLiteMemory) -> None:
+    conversation = _Conversation(sqlite_instance)
+    conversation.call(_responses_call("call_1", "lookup"))
+    conversation.add(
+        "tool",
+        "function_call_output",
+        _output("call_1", "ok"),
+        metadata={ToolExecutionMetadata.METADATA_KEY: {"invoked": "false"}},
+    )
+
+    with pytest.raises(RuntimeError, match="invoked"):
+        await _score(conversation.reply(), "lookup")
+
+
 async def test_output_must_pair_by_call_id_async(sqlite_instance: SQLiteMemory) -> None:
     conversation = _Conversation(sqlite_instance)
     conversation.call(_responses_call("call_1", "lookup"))
@@ -253,7 +315,12 @@ async def test_calls_the_model_did_not_make_are_ignored_async(sqlite_instance: S
 async def test_simulated_tool_output_is_not_execution_async(sqlite_instance: SQLiteMemory) -> None:
     conversation = _Conversation(sqlite_instance)
     conversation.call(_responses_call("call_1", "lookup"))
-    conversation.output("call_1", role="simulated_tool")
+    conversation.add(
+        "simulated_tool",
+        "function_call_output",
+        _output("call_1", "ok"),
+        metadata=ToolExecutionMetadata(invoked=True).to_metadata(),
+    )
     reply = conversation.reply()
 
     assert (await _score(reply, "lookup")).status == ScoreStatus.UNDETERMINED
@@ -330,16 +397,18 @@ def _sdk_text(text: str) -> MagicMock:
     ("registered", "outcome"),
     [(True, AttackOutcome.SUCCESS), (False, AttackOutcome.UNDETERMINED)],
 )
-async def test_scores_the_response_target_tool_loop_async(registered: bool, outcome: AttackOutcome) -> None:
+async def test_scores_the_response_target_tool_loop_async(
+    registered: bool, outcome: AttackOutcome, sqlite_instance: SQLiteMemory
+) -> None:
     target = OpenAIResponseTarget(
         model_name="gpt-4", endpoint="https://mock.azure.com", api_key="mock-key", fail_on_missing_function=False
     )
     if registered:
 
-        async def lookup(args: dict) -> dict:
-            return {"found": True}
+        async def lookup_async(args: dict) -> dict:
+            return {"error": "function_not_found", "missing_function": "lookup", "available_functions": []}
 
-        target._custom_functions["lookup"] = lookup
+        target._custom_functions["lookup"] = lookup_async
     attack = PromptSendingAttack(
         objective_target=target,
         attack_scoring_config=AttackScoringConfig(objective_scorer=MessageToolCallScorer()),
@@ -352,3 +421,14 @@ async def lookup(args: dict) -> dict:
     assert result.outcome is outcome
     assert result.automated_score is not None
     assert result.automated_score.message_piece_id == result.last_response.id
+    pieces = await sqlite_instance.get_message_pieces_async(conversation_id=result.last_response.conversation_id)
+    outputs = [piece for piece in pieces if piece.original_value_data_type == "function_call_output"]
+    assert len(outputs) == 1
+    assert ToolExecutionMetadata.from_metadata(metadata=outputs[0].prompt_metadata) == ToolExecutionMetadata(
+        invoked=registered
+    )
+    assert json.loads(json.loads(outputs[0].original_value)["output"]) == {
+        "error": "function_not_found",
+        "missing_function": "lookup",
+        "available_functions": [],
+    }

From 9b0f84d05083c7babd905bffb897465be8205ddc Mon Sep 17 00:00:00 2001
From: Richard Lundeen <rlundeen@microsoft.com>
Date: Thu, 1 Oct 2026 21:43:43 -0700
Subject: [PATCH 3/3] FIX: use async memory in tool-call scorer tests

Update the scorer test helper to use the async memory API so deprecation warnings do not fail CI.

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
---
 .../score/test_message_tool_call_scorer.py    | 193 +++++++++---------
 1 file changed, 101 insertions(+), 92 deletions(-)

diff --git a/tests/unit/score/test_message_tool_call_scorer.py b/tests/unit/score/test_message_tool_call_scorer.py
index fd4b8e7896..4986c75635 100644
--- a/tests/unit/score/test_message_tool_call_scorer.py
+++ b/tests/unit/score/test_message_tool_call_scorer.py
@@ -59,9 +59,16 @@ class _Conversation:
     def __init__(self, memory: SQLiteMemory) -> None:
         self._memory = memory
         self.conversation_id = str(uuid.uuid4())
-        self.add("user", "text", "use the tools")
 
-    def add(self, role: str, data_type: str, value: object, *, metadata: dict[str, Any] | None = None) -> Message:
+    @classmethod
+    async def create_async(cls, *, memory: SQLiteMemory) -> "_Conversation":
+        conversation = cls(memory)
+        await conversation.add_async("user", "text", "use the tools")
+        return conversation
+
+    async def add_async(
+        self, role: str, data_type: str, value: object, *, metadata: dict[str, Any] | None = None
+    ) -> Message:
         text = value if isinstance(value, str) else json.dumps(value, separators=(",", ":"))
         piece = MessagePiece(
             role=role,
@@ -71,17 +78,17 @@ def add(self, role: str, data_type: str, value: object, *, metadata: dict[str, A
             prompt_metadata=metadata or {},
         )
         message = piece.to_message()
-        self._memory.add_message_to_memory(request=message)
+        await self._memory.add_message_to_memory_async(request=message)
         return message
 
-    def call(self, call: dict, *, role: str = "assistant") -> Message:
-        return self.add(role, "function_call", call)
+    async def call_async(self, call: dict, *, role: str = "assistant") -> Message:
+        return await self.add_async(role, "function_call", call)
 
-    def output(self, call_id: str, output: object = "ok", *, role: str = "tool") -> Message:
-        return self.add(role, "function_call_output", _output(call_id, output))
+    async def output_async(self, call_id: str, output: object = "ok", *, role: str = "tool") -> Message:
+        return await self.add_async(role, "function_call_output", _output(call_id, output))
 
-    def reply(self, text: str = "done") -> Message:
-        return self.add("assistant", "text", text)
+    async def reply_async(self, text: str = "done") -> Message:
+        return await self.add_async("assistant", "text", text)
 
 
 async def _score(message: Message, *names: str):
@@ -93,10 +100,10 @@ async def _score(message: Message, *names: str):
 
 @pytest.mark.parametrize("call_format", [_responses_call, _chat_completions_call])
 async def test_paired_output_is_true_async(sqlite_instance: SQLiteMemory, call_format) -> None:
-    conversation = _Conversation(sqlite_instance)
-    conversation.call(call_format("call_1", "lookup"))
-    conversation.output("call_1")
-    reply = conversation.reply()
+    conversation = await _Conversation.create_async(memory=sqlite_instance)
+    await conversation.call_async(call_format("call_1", "lookup"))
+    await conversation.output_async("call_1")
+    reply = await conversation.reply_async()
 
     score = await _score(reply, "lookup")
 
@@ -109,8 +116,8 @@ async def test_paired_output_is_true_async(sqlite_instance: SQLiteMemory, call_f
 
 @pytest.mark.parametrize("call_format", [_responses_call, _chat_completions_call])
 async def test_requested_call_without_output_is_undetermined_async(sqlite_instance: SQLiteMemory, call_format) -> None:
-    conversation = _Conversation(sqlite_instance)
-    request = conversation.call(call_format("call_1", "lookup"))
+    conversation = await _Conversation.create_async(memory=sqlite_instance)
+    request = await conversation.call_async(call_format("call_1", "lookup"))
 
     score = await _score(request, "lookup")
 
@@ -120,7 +127,8 @@ async def test_requested_call_without_output_is_undetermined_async(sqlite_instan
 
 
 async def test_no_tool_evidence_is_undetermined_not_false_async(sqlite_instance: SQLiteMemory) -> None:
-    reply = _Conversation(sqlite_instance).reply()
+    conversation = await _Conversation.create_async(memory=sqlite_instance)
+    reply = await conversation.reply_async()
 
     score = await _score(reply, "lookup")
 
@@ -137,10 +145,10 @@ async def test_no_tool_evidence_is_undetermined_not_false_async(sqlite_instance:
     ],
 )
 async def test_tolerant_dispatch_error_is_not_execution_async(sqlite_instance: SQLiteMemory, dispatch_error) -> None:
-    conversation = _Conversation(sqlite_instance)
-    conversation.call(_responses_call("call_1", "lookup"))
-    conversation.output("call_1", dispatch_error)
-    reply = conversation.reply()
+    conversation = await _Conversation.create_async(memory=sqlite_instance)
+    await conversation.call_async(_responses_call("call_1", "lookup"))
+    await conversation.output_async("call_1", dispatch_error)
+    reply = await conversation.reply_async()
 
     score = await _score(reply, "lookup")
 
@@ -159,22 +167,22 @@ async def test_tolerant_dispatch_error_is_not_execution_async(sqlite_instance: S
     ],
 )
 async def test_tool_reported_error_still_counts_as_execution_async(sqlite_instance: SQLiteMemory, output) -> None:
-    conversation = _Conversation(sqlite_instance)
-    conversation.call(_responses_call("call_1", "lookup"))
-    conversation.output("call_1", output)
-    reply = conversation.reply()
+    conversation = await _Conversation.create_async(memory=sqlite_instance)
+    await conversation.call_async(_responses_call("call_1", "lookup"))
+    await conversation.output_async("call_1", output)
+    reply = await conversation.reply_async()
 
     assert (await _score(reply, "lookup")).get_value() is True
 
 
 async def test_unrelated_structured_error_does_not_abort_scoring_async(sqlite_instance: SQLiteMemory) -> None:
-    conversation = _Conversation(sqlite_instance)
-    conversation.call(_responses_call("other", "other_tool"))
-    conversation.output("other", {"error": {"code": "timeout"}})
-    conversation.call(_responses_call("call_1", "lookup"))
-    conversation.output("call_1")
+    conversation = await _Conversation.create_async(memory=sqlite_instance)
+    await conversation.call_async(_responses_call("other", "other_tool"))
+    await conversation.output_async("other", {"error": {"code": "timeout"}})
+    await conversation.call_async(_responses_call("call_1", "lookup"))
+    await conversation.output_async("call_1")
 
-    assert (await _score(conversation.reply(), "lookup")).get_value() is True
+    assert (await _score(await conversation.reply_async(), "lookup")).get_value() is True
 
 
 @pytest.mark.parametrize("invoked", [True, False])
@@ -182,9 +190,9 @@ async def test_unrelated_structured_error_does_not_abort_scoring_async(sqlite_in
 async def test_stored_execution_metadata_controls_verdict_async(
     sqlite_instance: SQLiteMemory, invoked: bool, output: object
 ) -> None:
-    conversation = _Conversation(sqlite_instance)
-    conversation.call(_responses_call("call_1", "lookup"))
-    result = conversation.add(
+    conversation = await _Conversation.create_async(memory=sqlite_instance)
+    await conversation.call_async(_responses_call("call_1", "lookup"))
+    result = await conversation.add_async(
         "tool",
         "function_call_output",
         _output("call_1", output),
@@ -195,7 +203,7 @@ async def test_stored_execution_metadata_controls_verdict_async(
         invoked=invoked
     )
 
-    score = await _score(conversation.reply(), "lookup")
+    score = await _score(await conversation.reply_async(), "lookup")
     if invoked:
         assert score.get_value() is True
     else:
@@ -203,9 +211,9 @@ async def test_stored_execution_metadata_controls_verdict_async(
 
 
 async def test_invalid_execution_metadata_is_not_a_legacy_fallback_async(sqlite_instance: SQLiteMemory) -> None:
-    conversation = _Conversation(sqlite_instance)
-    conversation.call(_responses_call("call_1", "lookup"))
-    conversation.add(
+    conversation = await _Conversation.create_async(memory=sqlite_instance)
+    await conversation.call_async(_responses_call("call_1", "lookup"))
+    await conversation.add_async(
         "tool",
         "function_call_output",
         _output("call_1", "ok"),
@@ -213,25 +221,25 @@ async def test_invalid_execution_metadata_is_not_a_legacy_fallback_async(sqlite_
     )
 
     with pytest.raises(RuntimeError, match="invoked"):
-        await _score(conversation.reply(), "lookup")
+        await _score(await conversation.reply_async(), "lookup")
 
 
 async def test_output_must_pair_by_call_id_async(sqlite_instance: SQLiteMemory) -> None:
-    conversation = _Conversation(sqlite_instance)
-    conversation.call(_responses_call("call_1", "lookup"))
-    conversation.call(_responses_call("call_2", "delete"))
-    conversation.output("call_2")
-    reply = conversation.reply()
+    conversation = await _Conversation.create_async(memory=sqlite_instance)
+    await conversation.call_async(_responses_call("call_1", "lookup"))
+    await conversation.call_async(_responses_call("call_2", "delete"))
+    await conversation.output_async("call_2")
+    reply = await conversation.reply_async()
 
     assert (await _score(reply, "delete")).get_value() is True
     assert (await _score(reply, "lookup")).status == ScoreStatus.UNDETERMINED
 
 
 async def test_output_for_an_unknown_call_is_ignored_async(sqlite_instance: SQLiteMemory) -> None:
-    conversation = _Conversation(sqlite_instance)
-    conversation.call(_responses_call("call_1", "lookup"))
-    conversation.output("call_9")
-    reply = conversation.reply()
+    conversation = await _Conversation.create_async(memory=sqlite_instance)
+    await conversation.call_async(_responses_call("call_1", "lookup"))
+    await conversation.output_async("call_9")
+    reply = await conversation.reply_async()
 
     assert (await _score(reply, "lookup")).status == ScoreStatus.UNDETERMINED
 
@@ -240,31 +248,31 @@ async def test_output_for_an_unknown_call_is_ignored_async(sqlite_instance: SQLi
 async def test_only_tool_role_function_call_outputs_count_async(
     sqlite_instance: SQLiteMemory, role: str, data_type: str
 ) -> None:
-    conversation = _Conversation(sqlite_instance)
-    conversation.call(_responses_call("call_1", "lookup"))
-    conversation.add(role, data_type, _output("call_1", "ok"))
-    reply = conversation.reply()
+    conversation = await _Conversation.create_async(memory=sqlite_instance)
+    await conversation.call_async(_responses_call("call_1", "lookup"))
+    await conversation.add_async(role, data_type, _output("call_1", "ok"))
+    reply = await conversation.reply_async()
 
     assert (await _score(reply, "lookup")).status == ScoreStatus.UNDETERMINED
 
 
 async def test_output_before_its_request_is_ignored_async(sqlite_instance: SQLiteMemory) -> None:
-    conversation = _Conversation(sqlite_instance)
-    conversation.output("call_1")
-    conversation.call(_responses_call("call_1", "lookup"))
-    reply = conversation.reply()
+    conversation = await _Conversation.create_async(memory=sqlite_instance)
+    await conversation.output_async("call_1")
+    await conversation.call_async(_responses_call("call_1", "lookup"))
+    reply = await conversation.reply_async()
 
     assert (await _score(reply, "lookup")).status == ScoreStatus.UNDETERMINED
 
 
 async def test_every_required_tool_must_run_async(sqlite_instance: SQLiteMemory) -> None:
-    conversation = _Conversation(sqlite_instance)
-    conversation.call(_responses_call("call_1", "lookup"))
-    conversation.output("call_1")
-    partial = conversation.reply()
-    conversation.call(_responses_call("call_2", "summarize"))
-    conversation.output("call_2")
-    full = conversation.reply()
+    conversation = await _Conversation.create_async(memory=sqlite_instance)
+    await conversation.call_async(_responses_call("call_1", "lookup"))
+    await conversation.output_async("call_1")
+    partial = await conversation.reply_async()
+    await conversation.call_async(_responses_call("call_2", "summarize"))
+    await conversation.output_async("call_2")
+    full = await conversation.reply_async()
 
     missing = await _score(partial, "lookup", "summarize")
     assert missing.status == ScoreStatus.UNDETERMINED
@@ -274,69 +282,70 @@ async def test_every_required_tool_must_run_async(sqlite_instance: SQLiteMemory)
 
 
 async def test_tool_names_match_exactly_async(sqlite_instance: SQLiteMemory) -> None:
-    conversation = _Conversation(sqlite_instance)
-    conversation.call(_responses_call("call_1", "Lookup"))
-    conversation.output("call_1")
-    reply = conversation.reply()
+    conversation = await _Conversation.create_async(memory=sqlite_instance)
+    await conversation.call_async(_responses_call("call_1", "Lookup"))
+    await conversation.output_async("call_1")
+    reply = await conversation.reply_async()
 
     assert (await _score(reply, "lookup")).status == ScoreStatus.UNDETERMINED
 
 
 async def test_later_turns_are_not_evidence_for_an_earlier_message_async(sqlite_instance: SQLiteMemory) -> None:
-    conversation = _Conversation(sqlite_instance)
-    earlier = conversation.reply()
-    conversation.call(_responses_call("call_1", "lookup"))
-    conversation.output("call_1")
+    conversation = await _Conversation.create_async(memory=sqlite_instance)
+    earlier = await conversation.reply_async()
+    await conversation.call_async(_responses_call("call_1", "lookup"))
+    await conversation.output_async("call_1")
 
     assert (await _score(earlier, "lookup")).status == ScoreStatus.UNDETERMINED
 
 
 async def test_other_conversations_are_not_evidence_async(sqlite_instance: SQLiteMemory) -> None:
-    other = _Conversation(sqlite_instance)
-    other.call(_responses_call("call_1", "lookup"))
-    other.output("call_1")
-    conversation = _Conversation(sqlite_instance)
+    other = await _Conversation.create_async(memory=sqlite_instance)
+    await other.call_async(_responses_call("call_1", "lookup"))
+    await other.output_async("call_1")
+    conversation = await _Conversation.create_async(memory=sqlite_instance)
     for _ in range(3):
-        reply = conversation.reply()
+        reply = await conversation.reply_async()
 
     assert (await _score(reply, "lookup")).status == ScoreStatus.UNDETERMINED
 
 
 @pytest.mark.parametrize("role", ["simulated_assistant", "user"])
 async def test_calls_the_model_did_not_make_are_ignored_async(sqlite_instance: SQLiteMemory, role: str) -> None:
-    conversation = _Conversation(sqlite_instance)
-    conversation.call(_responses_call("call_1", "lookup"), role=role)
-    conversation.output("call_1")
-    reply = conversation.reply()
+    conversation = await _Conversation.create_async(memory=sqlite_instance)
+    await conversation.call_async(_responses_call("call_1", "lookup"), role=role)
+    await conversation.output_async("call_1")
+    reply = await conversation.reply_async()
 
     assert (await _score(reply, "lookup")).status == ScoreStatus.UNDETERMINED
 
 
 async def test_simulated_tool_output_is_not_execution_async(sqlite_instance: SQLiteMemory) -> None:
-    conversation = _Conversation(sqlite_instance)
-    conversation.call(_responses_call("call_1", "lookup"))
-    conversation.add(
+    conversation = await _Conversation.create_async(memory=sqlite_instance)
+    await conversation.call_async(_responses_call("call_1", "lookup"))
+    await conversation.add_async(
         "simulated_tool",
         "function_call_output",
         _output("call_1", "ok"),
         metadata=ToolExecutionMetadata(invoked=True).to_metadata(),
     )
-    reply = conversation.reply()
+    reply = await conversation.reply_async()
 
     assert (await _score(reply, "lookup")).status == ScoreStatus.UNDETERMINED
 
 
 async def test_function_call_text_in_a_reply_is_not_a_call_async(sqlite_instance: SQLiteMemory) -> None:
-    conversation = _Conversation(sqlite_instance)
-    conversation.add("assistant", "text", _responses_call("call_1", "lookup"))
-    conversation.output("call_1")
-    reply = conversation.reply()
+    conversation = await _Conversation.create_async(memory=sqlite_instance)
+    await conversation.add_async("assistant", "text", _responses_call("call_1", "lookup"))
+    await conversation.output_async("call_1")
+    reply = await conversation.reply_async()
 
     assert (await _score(reply, "lookup")).status == ScoreStatus.UNDETERMINED
 
 
 async def test_requires_a_tools_called_condition_async(sqlite_instance: SQLiteMemory) -> None:
-    reply = _Conversation(sqlite_instance).reply()
+    conversation = await _Conversation.create_async(memory=sqlite_instance)
+    reply = await conversation.reply_async()
 
     with pytest.raises(ValueError, match="ToolsCalled"):
         await MessageToolCallScorer().score_async(
@@ -354,11 +363,11 @@ async def test_rejects_loose_content_async() -> None:
 
 @pytest.mark.parametrize("executed", [True, False])
 async def test_composes_under_or_with_trace_scoring_async(sqlite_instance: SQLiteMemory, executed: bool) -> None:
-    conversation = _Conversation(sqlite_instance)
-    conversation.call(_responses_call("call_1", "lookup"))
+    conversation = await _Conversation.create_async(memory=sqlite_instance)
+    await conversation.call_async(_responses_call("call_1", "lookup"))
     if executed:
-        conversation.output("call_1")
-    reply = conversation.reply()
+        await conversation.output_async("call_1")
+    reply = await conversation.reply_async()
     client = InMemoryTraceClient()
     composite = TrueFalseCompositeScorer(
         scorers=[OtelToolCallScorer(source=OtelTraceSource(trace_client=client)), MessageToolCallScorer()],