Skip to content

Commit 633df85

Browse files
FIX: report canonical technique identity for PromptInject run summaries
PromptInject atomic attacks never set technique_name on AtomicAttack, only display_group (the goal text). Scenario._build_run_plan then failed to match the goal text against the technique enum and persisted technique_name=None, so run detail, list, and progress projections fell back to displaying the goal-based display_group as the technique. Set technique_name explicitly when building each atomic attack, and have the three techniques_used/techniques projections in scenario_run_service.py prefer technique_name over display_group. Fixes #2782. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
1 parent a8cf7d7 commit 633df85

4 files changed

Lines changed: 99 additions & 3 deletions

File tree

‎pyrit/backend/services/scenario_run_service.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1380,7 +1380,7 @@ def _build_response_from_db(
13801380
)
13811381
)
13821382
techniques_used = (
1383-
list(dict.fromkeys(group.display_group for group in plan.atomic_groups))
1383+
list(dict.fromkeys(group.technique_name or group.display_group for group in plan.atomic_groups))
13841384
if plan is not None
13851385
else scenario_result.get_techniques_used()
13861386
)
@@ -1581,7 +1581,7 @@ def _build_history_summary(
15811581
if terminal and record.completed_at is not None:
15821582
timestamps.append(record.completed_at)
15831583
techniques = (
1584-
list(dict.fromkeys(group.display_group for group in atomic_groups))
1584+
list(dict.fromkeys(group.technique_name or group.display_group for group in atomic_groups))
15851585
if atomic_groups is not None
15861586
else list(aggregate.atomic_attack_names)
15871587
)
@@ -1985,7 +1985,9 @@ def get_run_progress_from_storage(
19851985
scenario_identifier = header_result.scenario_identifier
19861986
target, datasets_used, scenario_parameters = self._safe_run_metadata(scenario_identifier=scenario_identifier)
19871987
if plan is not None:
1988-
techniques_used = list(dict.fromkeys(group.display_group for group in plan.atomic_groups))
1988+
techniques_used = list(
1989+
dict.fromkeys(group.technique_name or group.display_group for group in plan.atomic_groups)
1990+
)
19891991
else:
19901992
techniques_used = self._identifier_techniques(scenario_identifier)
19911993
return ScenarioRunProgress(

‎pyrit/scenario/scenarios/garak/prompt_inject.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -325,6 +325,7 @@ async def _build_atomic_attacks_async(self, *, context: ScenarioContext) -> list
325325
AtomicAttack(
326326
atomic_attack_name=f"{technique.value}__goal_{goal_index}",
327327
display_group=goal_text,
328+
technique_name=technique.value,
328329
attack_technique=AttackTechnique(attack=attack),
329330
seed_groups=seed_groups,
330331
memory_labels=context.memory_labels,

‎tests/unit/backend/test_scenario_run_service.py‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1479,6 +1479,40 @@ def test_get_run_detail_preserves_readability_across_plan_metadata(
14791479
assert fetched.techniques_used == (["Attack"] if expected_planned_total else ["legacy attack"])
14801480
assert ("using legacy run detail fields" in caplog.text) is expected_warning
14811481

1482+
def test_get_run_detail_techniques_used_prefers_technique_name_over_display_group(self, mock_memory) -> None:
1483+
"""A goal-based display_group must not stand in for the technique identity."""
1484+
plan = ScenarioRunPlan(
1485+
scenario_registry_name="garak.prompt_inject",
1486+
atomic_groups=[
1487+
ScenarioRunPlanAtomicGroup(
1488+
id="group-1",
1489+
atomic_attack_name="ignore_print__goal_0",
1490+
display_group="AUDIT_SAFE_MARKER",
1491+
technique_name="ignore_print",
1492+
technique_eval_hash="eval",
1493+
seed_group_ids=["seed-1"],
1494+
)
1495+
],
1496+
seed_groups=[
1497+
ScenarioRunPlanSeedGroup(
1498+
id="seed-1",
1499+
objective_sha256=to_sha256("objective"),
1500+
objective="objective",
1501+
)
1502+
],
1503+
).model_dump(mode="json")
1504+
db_result = make_scenario_result(
1505+
scenario_name="garak.prompt_inject",
1506+
attack_results={},
1507+
metadata={SCENARIO_RUN_PLAN_METADATA_KEY: plan},
1508+
)
1509+
mock_memory.get_scenario_results.return_value = [db_result]
1510+
1511+
fetched = ScenarioRunService().get_run(scenario_result_id=str(db_result.id))
1512+
1513+
assert fetched is not None
1514+
assert fetched.techniques_used == ["ignore_print"]
1515+
14821516
@pytest.mark.parametrize("run_state", list(ScenarioRunState))
14831517
def test_get_run_only_falls_back_to_persisted_error_for_failed_state(
14841518
self, *, mock_memory: MagicMock, run_state: ScenarioRunState
@@ -1752,6 +1786,28 @@ def test_history_scopes_duplicate_objective_hashes_to_atomic_groups(self, mock_m
17521786
assert summary.successful_attacks == 2
17531787
mock_memory.get_scenario_history_aggregates.assert_not_called()
17541788

1789+
def test_list_runs_techniques_used_prefers_technique_name_over_display_group(self, mock_memory) -> None:
1790+
"""A goal-based display_group must not stand in for the technique identity."""
1791+
record = _make_history_record(result_id="sr-goal-display-group", run_state=ScenarioRunState.COMPLETED)
1792+
group = ScenarioRunPlanAtomicGroup(
1793+
id="group-1",
1794+
atomic_attack_name="ignore_print__goal_0",
1795+
display_group="AUDIT_SAFE_MARKER",
1796+
technique_name="ignore_print",
1797+
technique_eval_hash="eval",
1798+
seed_group_ids=["seed-1"],
1799+
).model_dump(mode="json")
1800+
record = replace(
1801+
record,
1802+
plan_atomic_groups=[group],
1803+
plan_seed_id_map=[{"id": "seed-1", "objective_sha256": "hash-1"}],
1804+
)
1805+
mock_memory.get_scenario_run_history_page.return_value = ([record], {}, False)
1806+
1807+
summary = ScenarioRunService().list_runs().items[0]
1808+
1809+
assert summary.techniques_used == ["ignore_print"]
1810+
17551811
def test_history_falls_back_for_duplicate_objective_hashes_within_one_group(self, mock_memory) -> None:
17561812
record = _make_history_record(result_id="sr-ambiguous-objective", run_state=ScenarioRunState.COMPLETED)
17571813
group = ScenarioRunPlanAtomicGroup(
@@ -3188,6 +3244,42 @@ def test_get_progress_exposes_persisted_started_at(mock_memory) -> None:
31883244
assert progress.run.started_at == started_at
31893245

31903246

3247+
def test_get_progress_techniques_used_prefers_technique_name_over_display_group(mock_memory) -> None:
3248+
"""A goal-based display_group must not stand in for the technique identity."""
3249+
header = make_scenario_result(
3250+
scenario_name="garak.prompt_inject",
3251+
attack_results={},
3252+
metadata={
3253+
SCENARIO_RUN_PLAN_METADATA_KEY: ScenarioRunPlan(
3254+
scenario_registry_name="garak.prompt_inject",
3255+
atomic_groups=[
3256+
ScenarioRunPlanAtomicGroup(
3257+
id="group-1",
3258+
atomic_attack_name="ignore_print__goal_0",
3259+
display_group="AUDIT_SAFE_MARKER",
3260+
technique_name="ignore_print",
3261+
technique_eval_hash="eval",
3262+
seed_group_ids=[],
3263+
)
3264+
],
3265+
seed_groups=[],
3266+
).model_dump(mode="json"),
3267+
},
3268+
)
3269+
mock_memory.get_scenario_result_header.return_value = header
3270+
mock_memory.get_scenario_attack_result_deltas.return_value = ([], False)
3271+
3272+
progress = ScenarioRunService().get_run_progress_from_storage(
3273+
scenario_result_id=str(header.id),
3274+
since=None,
3275+
limit=25,
3276+
active_group_ids=[],
3277+
)
3278+
3279+
assert progress is not None
3280+
assert progress.run.techniques_used == ["ignore_print"]
3281+
3282+
31913283
@pytest.mark.parametrize("started_at", ["not-a-timestamp", "2026-08-08T12:30:00"])
31923284
def test_load_started_at_rejects_invalid_or_naive_timestamp(started_at: str) -> None:
31933285
scenario_result = make_scenario_result(

‎tests/unit/scenario/garak/test_prompt_inject.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,7 @@ async def test_technique_and_goal_select_independent_axes(self, mock_objective_t
125125
attack = scenario._atomic_attacks[0]
126126
assert attack.atomic_attack_name == "ignore_print__goal_0"
127127
assert attack.display_group == "custom goal"
128+
assert attack.technique_name == "ignore_print"
128129
assert len(attack.seed_groups) == 12
129130
converter = attack.attack_technique.attack.get_request_converters()[0].converters[0]
130131
for group in attack.seed_groups:

0 commit comments

Comments
 (0)