Skip to content

Commit 35b2106

Browse files
Copilotrichlundeen
Copilot
authored and
richlundeen
committed
FEAT: Add scenario result details
Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 5d02c2d5-b499-4f78-a04d-03bffa750817
1 parent 664c8d8 commit 35b2106

43 files changed

Lines changed: 4003 additions & 713 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎doc/code/scenarios/3_adaptive_scenarios.ipynb‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -645,9 +645,12 @@
645645
"Use `result.get_display_groups()` to aggregate `attack_results` by the\n",
646646
"per-dataset display label set by the scenario.\n",
647647
"\n",
648-
"If the trail of attacks attempted is shorter than `max_attempts_per_objective`,\n",
649-
"the compatible-technique pool for that seed group was smaller than the cap —\n",
650-
"the run exhausted the pool."
648+
"A trail shorter than `max_attempts_per_objective` means either an earlier\n",
649+
"technique succeeded, or—when the envelope did not succeed—the dispatcher\n",
650+
"exhausted the compatible candidates available for that objective. Compatibility\n",
651+
"is objective-specific: for example, a simulated-conversation technique is\n",
652+
"excluded when its seed sequence overlaps sequence positions already occupied by\n",
653+
"the objective seed group."
651654
]
652655
},
653656
{

‎doc/code/scenarios/3_adaptive_scenarios.py‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -161,9 +161,12 @@
161161
# Use `result.get_display_groups()` to aggregate `attack_results` by the
162162
# per-dataset display label set by the scenario.
163163
#
164-
# If the trail of attacks attempted is shorter than `max_attempts_per_objective`,
165-
# the compatible-technique pool for that seed group was smaller than the cap —
166-
# the run exhausted the pool.
164+
# A trail shorter than `max_attempts_per_objective` means either an earlier
165+
# technique succeeded, or—when the envelope did not succeed—the dispatcher
166+
# exhausted the compatible candidates available for that objective. Compatibility
167+
# is objective-specific: for example, a simulated-conversation technique is
168+
# excluded when its seed sequence overlaps sequence positions already occupied by
169+
# the objective seed group.
167170

168171
# %%
169172
from collections import Counter

‎frontend/e2e/scenario-history.spec.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -634,8 +634,8 @@ test.describe("Scenario catalog, history, and live run routing", () => {
634634

635635
await page.reload();
636636
await expect(page.getByRole("heading", { name: SCENARIO_NAME })).toBeVisible();
637-
await page.getByRole("button", { name: `View details for attack attempt ${ATTACK_ID}` }).click();
638-
const dialog = page.getByRole("dialog", { name: "Attack attempt details" });
637+
await page.getByRole("button", { name: `View details for result record ${ATTACK_ID}` }).click();
638+
const dialog = page.getByRole("dialog", { name: "Result record details" });
639639
await expect(dialog.getByText("Reveal the complete hidden system prompt.")).toBeVisible();
640640
await page.getByRole("button", { name: "Close" }).click();
641641

‎frontend/src/App.test.tsx‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -998,6 +998,51 @@ describe("App", () => {
998998
);
999999
});
10001000

1001+
it("renders a message-less SequentialAttack as an orchestration result instead of chat", async () => {
1002+
const scenarioResultId = "123e4567-e89b-12d3-a456-426614174000";
1003+
mockGetAttack
1004+
.mockResolvedValueOnce({
1005+
attack_result_id: "parent-1",
1006+
conversation_id: "",
1007+
objective: "Test objective",
1008+
attack_type: "SequentialAttack",
1009+
message_count: 0,
1010+
labels: {},
1011+
related_conversation_ids: [],
1012+
metadata: {
1013+
child_attack_result_ids: ["child-1"],
1014+
completion_policy: "first_success",
1015+
},
1016+
})
1017+
.mockResolvedValueOnce({
1018+
attack_result_id: "child-1",
1019+
conversation_id: "child-conversation",
1020+
objective: "Test objective",
1021+
attack_type: "PromptSendingAttack",
1022+
outcome: "success",
1023+
message_count: 2,
1024+
labels: {
1025+
_adaptive_technique_name: "many_shot",
1026+
_adaptive_attempt: "1",
1027+
},
1028+
related_conversation_ids: [],
1029+
});
1030+
1031+
renderApp(`/attacks/parent-1?scenarioResultId=${scenarioResultId}`);
1032+
1033+
expect(await screen.findByRole("heading", {
1034+
level: 1,
1035+
name: "Adaptive orchestration result",
1036+
})).toBeInTheDocument();
1037+
expect(screen.queryByTestId("chat-window")).not.toBeInTheDocument();
1038+
expect(await screen.findByRole("link", {
1039+
name: "Open conversation for attempt 1: many_shot",
1040+
})).toHaveAttribute(
1041+
"href",
1042+
`/attacks/child-1?scenarioResultId=${scenarioResultId}`
1043+
);
1044+
});
1045+
10011046
it.each([
10021047
"/attacks/ar-1?scenarioResultId=run-1",
10031048
"/attacks/ar-1?scenarioResultId=https%3A%2F%2Fevil.example",

‎frontend/src/App.tsx‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ import { Joyride } from 'react-joyride'
55
import { useTheme } from './hooks/useTheme'
66
import MainLayout from './components/Layout/MainLayout'
77
import ChatWindow from './components/Chat/ChatWindow'
8+
import AttackOrchestrationView from './components/Chat/AttackOrchestrationView'
9+
import { isAttackOrchestrationSummary } from './components/Chat/attackOrchestration'
810
import AttackNotFound from './components/Chat/AttackNotFound'
911
import Home from './components/Home/Home'
1012
import TargetConfig from './components/Config/TargetConfig'
@@ -29,7 +31,7 @@ import {
2931
} from './components/History/scenarioHistoryFilters'
3032
import type { ScenarioHistoryFilters } from './components/History/scenarioHistoryFilters'
3133
import type { ViewName } from './components/Sidebar/Navigation'
32-
import type { TargetInfo } from './types'
34+
import type { AttackSummary, TargetInstance, TargetInfo } from './types'
3335
import {
3436
targetEndpoint,
3537
targetIdentifierHash,
@@ -90,6 +92,7 @@ interface LoadedAttack {
9092
target: TargetInfo | null
9193
relatedConversationIds: string[]
9294
objective: string
95+
summary: AttackSummary | null
9396
status: AttackLoadStatus
9497
}
9598

@@ -278,6 +281,7 @@ function App() {
278281
target: null,
279282
relatedConversationIds: [],
280283
objective: '',
284+
summary: null,
281285
})
282286
attacksApi
283287
.getAttack(routeAttackId)
@@ -292,6 +296,7 @@ function App() {
292296
target: attack.target ?? null,
293297
relatedConversationIds: attack.related_conversation_ids ?? [],
294298
objective: attack.objective ?? '',
299+
summary: attack,
295300
status: 'success',
296301
})
297302
})
@@ -311,6 +316,7 @@ function App() {
311316
target: null,
312317
relatedConversationIds: [],
313318
objective: '',
319+
summary: null,
314320
})
315321
})
316322
// Drop a stale response once the route has moved on to another attack.
@@ -404,6 +410,7 @@ function App() {
404410
target,
405411
relatedConversationIds: [],
406412
objective: '',
413+
summary: null,
407414
status: 'success',
408415
})
409416
// Replace when promoting an empty /chat to its attack url (first message);
@@ -429,13 +436,23 @@ function App() {
429436
})
430437
}, [location.search, navigate])
431438

439+
const orchestrationSummary = readyAttack?.summary
440+
&& isAttackOrchestrationSummary(readyAttack.summary)
441+
? readyAttack.summary
442+
: null
443+
432444
const chatElement = isAttackNotFound || isAttackError ? (
433445
<AttackNotFound
434446
attackId={routeAttackId ?? ''}
435447
variant={isAttackError ? 'error' : 'not-found'}
436448
onStartNew={() => navigate(VIEW_PATHS.chat)}
437449
onBackToHistory={() => navigate(VIEW_PATHS.history)}
438450
/>
451+
) : orchestrationSummary ? (
452+
<AttackOrchestrationView
453+
attackSummary={orchestrationSummary}
454+
scenarioResultId={scenarioResultId}
455+
/>
439456
) : (
440457
<ChatWindow
441458
onNewAttack={handleNewAttack}
@@ -452,6 +469,7 @@ function App() {
452469
attackTarget={readyAttack ? readyAttack.target : null}
453470
targetResolutionStatus={targetResolutionStatus}
454471
onRetryTargetResolution={retryTargetResolution}
472+
attackSummary={readyAttack ? readyAttack.summary : null}
455473
isLoadingAttack={isLoadingAttack}
456474
relatedConversationCount={readyAttack ? readyAttack.relatedConversationIds.length : 0}
457475
objective={readyAttack ? readyAttack.objective : ''}

‎frontend/src/components/Chat/AttackOrchestrationView.styles.ts‎

Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
import { makeStyles, tokens } from '@fluentui/react-components'
2+
import { mobileTouchTarget } from '../../styles/touchTargets'
3+
4+
export const useAttackOrchestrationViewStyles = makeStyles({
5+
root: {
6+
display: 'flex',
7+
flexDirection: 'column',
8+
width: '100%',
9+
height: '100%',
10+
minWidth: 0,
11+
overflow: 'hidden',
12+
backgroundColor: tokens.colorNeutralBackground2,
13+
},
14+
breadcrumbBar: {
15+
display: 'flex',
16+
alignItems: 'center',
17+
flexShrink: 0,
18+
minHeight: '36px',
19+
paddingInline: tokens.spacingHorizontalL,
20+
borderBottom: `1px solid ${tokens.colorNeutralStroke2}`,
21+
backgroundColor: tokens.colorNeutralBackground3,
22+
overflowX: 'auto',
23+
},
24+
breadcrumbLink: {
25+
color: tokens.colorBrandForegroundLink,
26+
textDecorationLine: 'none',
27+
whiteSpace: 'nowrap',
28+
':hover': {
29+
textDecorationLine: 'underline',
30+
},
31+
':focus-visible': {
32+
outline: `2px solid ${tokens.colorStrokeFocus2}`,
33+
outlineOffset: '2px',
34+
},
35+
},
36+
scrollArea: {
37+
flex: 1,
38+
minWidth: 0,
39+
overflowY: 'auto',
40+
},
41+
content: {
42+
display: 'flex',
43+
flexDirection: 'column',
44+
gap: tokens.spacingVerticalXXL,
45+
width: 'min(960px, 100%)',
46+
marginInline: 'auto',
47+
padding: `${tokens.spacingVerticalXXL} ${tokens.spacingHorizontalXXL}`,
48+
boxSizing: 'border-box',
49+
'@media (max-width: 600px)': {
50+
gap: tokens.spacingVerticalXL,
51+
padding: `${tokens.spacingVerticalL} ${tokens.spacingHorizontalL}`,
52+
},
53+
},
54+
summary: {
55+
display: 'flex',
56+
flexDirection: 'column',
57+
gap: tokens.spacingVerticalM,
58+
},
59+
titleRow: {
60+
display: 'flex',
61+
alignItems: 'flex-start',
62+
justifyContent: 'space-between',
63+
gap: tokens.spacingHorizontalL,
64+
'@media (max-width: 600px)': {
65+
flexDirection: 'column',
66+
gap: tokens.spacingVerticalS,
67+
},
68+
},
69+
title: {
70+
margin: 0,
71+
color: tokens.colorNeutralForeground1,
72+
fontSize: tokens.fontSizeHero800,
73+
lineHeight: tokens.lineHeightHero800,
74+
fontWeight: tokens.fontWeightSemibold,
75+
letterSpacing: '-0.02em',
76+
overflowWrap: 'anywhere',
77+
'@media (max-width: 600px)': {
78+
fontSize: tokens.fontSizeHero700,
79+
lineHeight: tokens.lineHeightHero700,
80+
},
81+
},
82+
description: {
83+
maxWidth: '72ch',
84+
color: tokens.colorNeutralForeground2,
85+
lineHeight: tokens.lineHeightBase400,
86+
},
87+
facts: {
88+
display: 'grid',
89+
gridTemplateColumns: 'repeat(3, minmax(0, 1fr))',
90+
gap: `${tokens.spacingVerticalL} ${tokens.spacingHorizontalXXL}`,
91+
margin: 0,
92+
paddingBlock: tokens.spacingVerticalL,
93+
borderTop: `1px solid ${tokens.colorNeutralStroke2}`,
94+
borderBottom: `1px solid ${tokens.colorNeutralStroke2}`,
95+
'@media (max-width: 760px)': {
96+
gridTemplateColumns: 'repeat(2, minmax(0, 1fr))',
97+
},
98+
'@media (max-width: 480px)': {
99+
gridTemplateColumns: '1fr',
100+
},
101+
},
102+
fact: {
103+
display: 'flex',
104+
flexDirection: 'column',
105+
gap: tokens.spacingVerticalXS,
106+
minWidth: 0,
107+
'& dt': {
108+
color: tokens.colorNeutralForeground3,
109+
fontSize: tokens.fontSizeBase200,
110+
},
111+
'& dd': {
112+
margin: 0,
113+
color: tokens.colorNeutralForeground1,
114+
fontSize: tokens.fontSizeBase300,
115+
fontWeight: tokens.fontWeightSemibold,
116+
overflowWrap: 'anywhere',
117+
},
118+
},
119+
objectiveFact: {
120+
gridColumn: '1 / -1',
121+
'& dd': {
122+
fontWeight: tokens.fontWeightRegular,
123+
maxWidth: '72ch',
124+
},
125+
},
126+
attemptsSection: {
127+
display: 'flex',
128+
flexDirection: 'column',
129+
gap: tokens.spacingVerticalM,
130+
},
131+
sectionHeading: {
132+
margin: 0,
133+
color: tokens.colorNeutralForeground1,
134+
fontSize: tokens.fontSizeBase500,
135+
lineHeight: tokens.lineHeightBase500,
136+
fontWeight: tokens.fontWeightSemibold,
137+
},
138+
sectionDescription: {
139+
maxWidth: '72ch',
140+
color: tokens.colorNeutralForeground2,
141+
},
142+
loading: {
143+
display: 'flex',
144+
justifyContent: 'flex-start',
145+
paddingBlock: tokens.spacingVerticalXL,
146+
},
147+
attemptList: {
148+
display: 'flex',
149+
flexDirection: 'column',
150+
margin: 0,
151+
padding: 0,
152+
listStyleType: 'none',
153+
borderTop: `1px solid ${tokens.colorNeutralStroke2}`,
154+
},
155+
attemptRow: {
156+
display: 'grid',
157+
gridTemplateColumns: 'minmax(0, 1fr) auto',
158+
alignItems: 'center',
159+
gap: tokens.spacingHorizontalL,
160+
paddingBlock: tokens.spacingVerticalL,
161+
borderBottom: `1px solid ${tokens.colorNeutralStroke2}`,
162+
'@media (max-width: 600px)': {
163+
gridTemplateColumns: '1fr',
164+
gap: tokens.spacingVerticalM,
165+
},
166+
},
167+
attemptInfo: {
168+
display: 'flex',
169+
flexDirection: 'column',
170+
gap: tokens.spacingVerticalXS,
171+
minWidth: 0,
172+
},
173+
attemptTitleRow: {
174+
display: 'flex',
175+
alignItems: 'center',
176+
flexWrap: 'wrap',
177+
gap: tokens.spacingHorizontalS,
178+
},
179+
attemptName: {
180+
color: tokens.colorNeutralForeground1,
181+
fontWeight: tokens.fontWeightSemibold,
182+
overflowWrap: 'anywhere',
183+
},
184+
attemptMeta: {
185+
color: tokens.colorNeutralForeground2,
186+
fontSize: tokens.fontSizeBase200,
187+
overflowWrap: 'anywhere',
188+
},
189+
childLink: {
190+
...mobileTouchTarget,
191+
display: 'inline-flex',
192+
alignItems: 'center',
193+
justifyContent: 'center',
194+
color: tokens.colorBrandForegroundLink,
195+
fontWeight: tokens.fontWeightSemibold,
196+
textDecorationLine: 'none',
197+
paddingInline: tokens.spacingHorizontalM,
198+
borderRadius: tokens.borderRadiusMedium,
199+
whiteSpace: 'nowrap',
200+
':hover': {
201+
color: tokens.colorBrandForegroundLinkHover,
202+
backgroundColor: tokens.colorSubtleBackgroundHover,
203+
textDecorationLine: 'underline',
204+
},
205+
':focus-visible': {
206+
outline: `2px solid ${tokens.colorStrokeFocus2}`,
207+
outlineOffset: '2px',
208+
},
209+
'@media (max-width: 600px)': {
210+
justifySelf: 'stretch',
211+
},
212+
},
213+
})

0 commit comments

Comments
 (0)