Repository navigation
331 lines (311 loc) · 14.4 KB
/
Copy pathbrowser-benchmarks.yml
File metadata and controls
331 lines (311 loc) · 14.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
name: Browser Benchmark
# computesdk:secrets=NSC_TOKEN
# computesdk:secrets-env=declared
on:
push:
branches: [master]
paths:
- 'benchmarks/browser/**'
schedule:
- cron: '0 10 * * 5' # Weekly on Friday at 10am UTC (5am CDT)
workflow_dispatch:
inputs:
slug:
description: 'Optional platform benchmark slug override'
required: false
default: ''
name:
description: 'Optional platform benchmark name override'
required: false
default: ''
iterations:
description: 'Iterations per provider'
required: false
default: '100'
dry_run:
description: 'Run without ingesting or committing results'
required: false
default: false
type: boolean
trigger_source:
description: 'Recorded on platform runs as their trigger (e.g. platform-retrigger); leave empty for a plain manual run'
required: false
default: ''
trigger_requested_by:
description: 'Who asked for this run (e.g. the platform org slug); recorded alongside trigger_source'
required: false
default: ''
trigger_request_id:
description: 'Platform re-trigger request id; recorded so the platform can link this run back to the request'
required: false
default: ''
concurrency:
group: browser-benchmarks
cancel-in-progress: true
permissions:
contents: write
pull-requests: write
id-token: write
env:
BENCHMARK_SLUG: ${{ github.event.inputs.slug || '' }}
BENCH_TRIGGER_SOURCE: ${{ github.event.inputs.trigger_source || '' }}
BENCH_TRIGGER_REQUESTED_BY: ${{ github.event.inputs.trigger_requested_by || '' }}
BENCH_TRIGGER_REQUEST_ID: ${{ github.event.inputs.trigger_request_id || '' }}
BENCHMARK_NAME: ${{ github.event.inputs.name || '' }}
jobs:
bench:
name: Bench ${{ matrix.provider }}
# Browser Use's leg runs next to its API (house EC2, us-east-2); the rest on Namespace.
runs-on: ${{ matrix.provider == 'browseruse' && 'computesdk:computesdk:aws-us-east-2' || 'namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe;permissions.additional_grant=vault/object:*:reveal' }}
# runs-on: self-hosted
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
provider:
- browserbase
- hyperbrowser
- kernel
- notte
- steel
- tilion
- browseruse
# - anchorbrowser
steps:
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v4
- uses: actions/setup-node@v4
with:
node-version: 24
cache: 'pnpm'
# ComputeSDK runners get no GitHub OIDC token, so nscloud-setup's
# `nsc auth exchange-github-token` cannot work there. Pre-seed the
# workload token file it checks for when NSC_TOKEN is available;
# accepts either the raw bearer token or a full token JSON document.
# It also exports NSC_DOCKER_LOGIN/NSC_CONTAINER_REGISTRY so
# nscloud-setup skips `nsc docker login`, which this token is not
# granted for — the benchmarks only need vault access.
- name: Seed Namespace token file
run: |
if [ -n "${NSC_TOKEN:-}" ] && [ ! -f /var/run/nsc/token.json ]; then
mkdir -p /var/run/nsc
case "$NSC_TOKEN" in
\{*) printf '%s\n' "$NSC_TOKEN" > /var/run/nsc/token.json ;;
*) printf '{"bearer_token":"%s"}\n' "$NSC_TOKEN" > /var/run/nsc/token.json ;;
esac
chmod 600 /var/run/nsc/token.json
# The seeded token has no container-registry grant; tell
# nscloud-setup to skip `nsc docker login` for later steps.
echo "NSC_DOCKER_LOGIN=1" >> "$GITHUB_ENV"
echo "NSC_CONTAINER_REGISTRY=nscr.io" >> "$GITHUB_ENV"
fi
- uses: namespacelabs/nscloud-setup@v0
- name: Install dependencies
run: |
if [ "${{ github.event_name }}" = "schedule" ]; then
pnpm -r update --ignore-scripts
else
pnpm install --frozen-lockfile --ignore-scripts
fi
- name: Build workspace packages
# The @benchsdk/* packages are built from source and dist/ is not
# committed. Build them in topological order so dependents like
# @benchsdk/client can resolve @benchsdk/worker during their own build.
run: pnpm -r --filter './packages/**' run build
- name: Clear stale results from checkout
run: rm -rf results/browser/
- name: Run browser benchmark
run: |
. benchmarks/scripts/load-vault-secrets.sh '^(BROWSERBASE_API_KEY|BROWSERBASE_PROJECT_ID|BROWSER_USE_API_KEY|HYPERBROWSER_API_KEY|KERNEL_API_KEY|NOTTE_API_KEY|STEEL_API_KEY|TILION_API_KEY|TILION_BASE_URL|BENCHMARKS_PLATFORM_API_KEY)'
# All matrix jobs for this workflow run share a single platform run. The
# attempt number is included so re-runs do not rejoin completed runs.
RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
npx tsx packages/benchsdk-runner/dist/bin.js run benchmarks/browser/browser.bench.ts \
--provider ${{ matrix.provider }} \
--iterations ${{ github.event_name == 'push' && '10' || github.event.inputs.iterations || '100' }} \
--run-key "$RUN_KEY" \
${{ (github.event_name == 'push' || github.event.inputs.dry_run == 'true') && '--no-ingest' || '' }}
- name: Upload results
if: always()
uses: actions/upload-artifact@v4
with:
name: browser-results-${{ matrix.provider }}
path: results/browser/
if-no-files-found: ignore
retention-days: 7
collect:
name: Collect Results
runs-on: namespace-profile-default;permissions.additional_grant=vault/object:*:list;permissions.additional_grant=vault/object:*:describe;permissions.additional_grant=vault/object:*:reveal
# runs-on: self-hosted
needs: [bench]
if: always()
steps:
- uses: actions/checkout@v4
with:
# Full history so the rebase-on-push retry below has a merge base.
# A shallow (depth-1) clone can make `git rebase origin/<branch>`
# fail when the remote has advanced past the shallow boundary.
fetch-depth: 0
- uses: pnpm/action-setup@v4
- uses: actions/setup-node@v4
with:
node-version: 24
cache: 'pnpm'
# ComputeSDK runners get no GitHub OIDC token, so nscloud-setup's
# `nsc auth exchange-github-token` cannot work there. Pre-seed the
# workload token file it checks for when NSC_TOKEN is available;
# accepts either the raw bearer token or a full token JSON document.
# It also exports NSC_DOCKER_LOGIN/NSC_CONTAINER_REGISTRY so
# nscloud-setup skips `nsc docker login`, which this token is not
# granted for — the benchmarks only need vault access.
- name: Seed Namespace token file
run: |
if [ -n "${NSC_TOKEN:-}" ] && [ ! -f /var/run/nsc/token.json ]; then
mkdir -p /var/run/nsc
case "$NSC_TOKEN" in
\{*) printf '%s\n' "$NSC_TOKEN" > /var/run/nsc/token.json ;;
*) printf '{"bearer_token":"%s"}\n' "$NSC_TOKEN" > /var/run/nsc/token.json ;;
esac
chmod 600 /var/run/nsc/token.json
# The seeded token has no container-registry grant; tell
# nscloud-setup to skip `nsc docker login` for later steps.
echo "NSC_DOCKER_LOGIN=1" >> "$GITHUB_ENV"
echo "NSC_CONTAINER_REGISTRY=nscr.io" >> "$GITHUB_ENV"
fi
- uses: namespacelabs/nscloud-setup@v0
- name: Install dependencies
run: |
if [ "${{ github.event_name }}" = "schedule" ]; then
pnpm -r update --ignore-scripts
else
pnpm install --frozen-lockfile --ignore-scripts
fi
- name: Build workspace packages
# The @benchsdk/* packages are built from source and dist/ is not
# committed. Build them in topological order so dependents like
# @benchsdk/client can resolve @benchsdk/worker during their own build.
run: pnpm -r --filter './packages/**' run build
- name: Download all artifacts
uses: actions/download-artifact@v4
with:
path: artifacts/
pattern: browser-results-*
- name: Merge results
run: npx tsx benchmarks/src/merge-results.ts --input artifacts --mode browser
- run: pnpm run generate-browser-svg
- name: Upload SVG as artifact
if: github.event_name == 'push'
uses: actions/upload-artifact@v4
with:
name: browser-benchmark-svg
path: browser.svg
if-no-files-found: ignore
retention-days: 7
- name: Post results to merged PR
if: github.event_name == 'push'
continue-on-error: true
uses: actions/github-script@v7
with:
script: |
const fs = require('fs');
const path = require('path');
const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
const latestPath = path.join('results', 'browser', 'latest.json');
let body = '## Browser Benchmark Results\n\n';
if (!fs.existsSync(latestPath)) {
body += '> No browser benchmark results were generated.\n\n';
} else {
const data = JSON.parse(fs.readFileSync(latestPath, 'utf-8'));
const results = data.results
.filter(r => !r.skipped)
.sort((a, b) => (b.compositeScore || 0) - (a.compositeScore || 0));
if (results.length === 0) {
body += '> No browser benchmark results were generated.\n\n';
} else {
body += '| # | Provider | Score | Create | Connect | Navigate | Release | Total | Status |\n';
body += '|---|----------|-------|--------|---------|----------|---------|-------|--------|\n';
results.forEach((r, i) => {
const name = r.provider.charAt(0).toUpperCase() + r.provider.slice(1);
const score = r.compositeScore !== undefined ? r.compositeScore.toFixed(1) : '--';
const create = (r.summary.createMs.median / 1000).toFixed(2) + 's';
const connect = (r.summary.connectMs.median / 1000).toFixed(2) + 's';
const navigate = (r.summary.navigateMs.median / 1000).toFixed(2) + 's';
const release = (r.summary.releaseMs.median / 1000).toFixed(2) + 's';
const total = (r.summary.totalMs.median / 1000).toFixed(2) + 's';
const ok = r.iterations.filter(it => !it.error).length;
const count = r.iterations.length;
body += `| ${i + 1} | ${name} | ${score} | ${create} | ${connect} | ${navigate} | ${release} | ${total} | ${ok}/${count} |\n`;
});
body += '\n';
}
}
body += `---\n*[View full run](${runUrl}) · SVG available as [build artifact](${runUrl}#artifacts)*`;
// This push is a merge to master — post results to the PR it closed.
// A commit can be associated with several PRs (backports, merge
// queue), so pick the one merged into the branch we pushed to.
const target = context.ref.replace('refs/heads/', '');
const { data: prs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
owner: context.repo.owner,
repo: context.repo.repo,
commit_sha: context.sha,
});
const pr = prs.find(p => p.merged_at && p.base.ref === target);
if (!pr) {
// Direct push with no associated PR — fall back to a commit comment.
await github.rest.repos.createCommitComment({
owner: context.repo.owner,
repo: context.repo.repo,
commit_sha: context.sha,
body,
});
return;
}
const marker = '## Browser Benchmark Results';
const comments = await github.paginate(github.rest.issues.listComments, {
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: pr.number,
per_page: 100,
});
const existing = comments.find(c => c.body.startsWith(marker));
if (existing) {
await github.rest.issues.updateComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: existing.id,
body,
});
} else {
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: pr.number,
body,
});
}
- name: Commit and push
if: github.event_name != 'push' && github.event.inputs.dry_run != 'true'
run: |
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
git add package.json packages/*/package.json pnpm-lock.yaml browser.svg results/browser/
git diff --cached --quiet && echo "No changes to commit" && exit 0
git commit -m "chore: update browser benchmark results [skip ci]"
# Remote master can advance during the run (concurrent benchmark
# workflows push to master too), so a plain push fails non-fast-forward.
# Rebase onto the latest remote and retry a few times before giving up.
branch="${GITHUB_REF#refs/heads/}"
for attempt in 1 2 3 4 5; do
# Rebase before each attempt (including the last) so every rebase
# is followed by a push — otherwise the final iteration's rebase
# would be wasted and a resolvable non-fast-forward could still fail.
git fetch origin "${branch}"
git rebase --autostash "origin/${branch}" || { git rebase --abort; exit 1; }
if git push origin "HEAD:${branch}"; then
echo "Pushed on attempt ${attempt}"
exit 0
fi
echo "Push rejected (attempt ${attempt}); will rebase and retry"
done
echo "Failed to push after multiple attempts" >&2
exit 1