Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/ai-gateway-benchmarks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ jobs:
INPUT_ITERATIONS: ${{ github.event.inputs.iterations }}
FAMILY_ITERATIONS: ${{ matrix.family.iterations }}
run: |
. benchmarks/scripts/load-vault-secrets.sh '^(ANTHROPIC_API_KEY|OPENAI_API_KEY|GEMINI_API_KEY|KIMI_API_KEY|NOVITA_API_KEY|NEON_AI_GATEWAY_BASE_URL|NEON_AI_GATEWAY_TOKEN|NGROK_AI_GATEWAY_API_KEY|LLMAPI_API_KEY|CLOUDFLARE_AI_GATEWAY_ACCOUNT_ID|CLOUDFLARE_AI_GATEWAY_GATEWAY_ID|CLOUDFLARE_AI_GATEWAY_TOKEN|LLM_GATEWAY_API_KEY|OPENROUTER_API_KEY|PYDANTIC_AI_GATEWAY_API_KEY|VERCEL_AI_GATEWAY_API_KEY|CONCENTRATE_AI_GATEWAY_API_KEY|BLAZERAIL_API_KEY|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)'
. benchmarks/scripts/load-vault-secrets.sh '^(ANTHROPIC_API_KEY|OPENAI_API_KEY|GEMINI_API_KEY|KIMI_API_KEY|NOVITA_API_KEY|NEON_AI_GATEWAY_BASE_URL|NEON_AI_GATEWAY_TOKEN|NGROK_AI_GATEWAY_API_KEY|LLMAPI_API_KEY|CLOUDFLARE_AI_GATEWAY_ACCOUNT_ID|CLOUDFLARE_AI_GATEWAY_GATEWAY_ID|CLOUDFLARE_AI_GATEWAY_TOKEN|LLM_GATEWAY_API_KEY|OPENROUTER_API_KEY|PYDANTIC_AI_GATEWAY_API_KEY|VERCEL_AI_GATEWAY_API_KEY|CONCENTRATE_AI_GATEWAY_API_KEY|BLAZERAIL_API_KEY|BENCHMARKS_PLATFORM_API_KEY)'

if [ "${{ github.event_name }}" = "push" ] || [ "${{ github.event_name }}" = "schedule" ]; then
ITERATIONS="10"
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/browser-benchmarks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ jobs:
run: rm -rf results/browser/
- name: Run browser benchmark
run: |
. benchmarks/scripts/load-vault-secrets.sh '^(BROWSERBASE_API_KEY|BROWSERBASE_PROJECT_ID|BROWSER_USE_API_KEY|HYPERBROWSER_API_KEY|KERNEL_API_KEY|NOTTE_API_KEY|STEEL_API_KEY|TILION_API_KEY|TILION_BASE_URL|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)'
. benchmarks/scripts/load-vault-secrets.sh '^(BROWSERBASE_API_KEY|BROWSERBASE_PROJECT_ID|BROWSER_USE_API_KEY|HYPERBROWSER_API_KEY|KERNEL_API_KEY|NOTTE_API_KEY|STEEL_API_KEY|TILION_API_KEY|TILION_BASE_URL|BENCHMARKS_PLATFORM_API_KEY)'

# All matrix jobs for this workflow run share a single platform run. The
# attempt number is included so re-runs do not rejoin completed runs.
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/browser-concurrent-benchmarks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ jobs:
env:
THROUGHPUT_URLS: ${{ needs.setup.outputs.urls }}
run: |
. benchmarks/scripts/load-vault-secrets.sh '^(BROWSERBASE_API_KEY|BROWSERBASE_PROJECT_ID|BROWSER_USE_API_KEY|HYPERBROWSER_API_KEY|KERNEL_API_KEY|NOTTE_API_KEY|STEEL_API_KEY|TILION_API_KEY|TILION_BASE_URL|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)'
. benchmarks/scripts/load-vault-secrets.sh '^(BROWSERBASE_API_KEY|BROWSERBASE_PROJECT_ID|BROWSER_USE_API_KEY|HYPERBROWSER_API_KEY|KERNEL_API_KEY|NOTTE_API_KEY|STEEL_API_KEY|TILION_API_KEY|TILION_BASE_URL|BENCHMARKS_PLATFORM_API_KEY)'

# One invocation covers the whole sweep: the benchmark runs one task
# per concurrency level, and --levels picks which levels run (the
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/browser-throughput-benchmarks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ jobs:
env:
THROUGHPUT_URLS: ${{ needs.setup.outputs.urls }}
run: |
. benchmarks/scripts/load-vault-secrets.sh '^(BROWSERBASE_API_KEY|BROWSERBASE_PROJECT_ID|BROWSER_USE_API_KEY|HYPERBROWSER_API_KEY|KERNEL_API_KEY|NOTTE_API_KEY|STEEL_API_KEY|TILION_API_KEY|TILION_BASE_URL|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)'
. benchmarks/scripts/load-vault-secrets.sh '^(BROWSERBASE_API_KEY|BROWSERBASE_PROJECT_ID|BROWSER_USE_API_KEY|HYPERBROWSER_API_KEY|KERNEL_API_KEY|NOTTE_API_KEY|STEEL_API_KEY|TILION_API_KEY|TILION_BASE_URL|BENCHMARKS_PLATFORM_API_KEY)'

# All matrix jobs for this workflow run share a single platform run. The
# attempt number is included so re-runs do not rejoin completed runs.
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/reliability-tensorlake-opencode.yml
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ jobs:
STAGGER_DELAY_MS: ${{ github.event.inputs.stagger_delay_ms || '60000' }}
DRY_RUN: ${{ github.event.inputs.dry_run || false }}
run: |
. benchmarks/scripts/load-vault-secrets.sh '^(TENSORLAKE_API_KEY|BENCHMARKS_PLATFORM_API_KEY|COMPUTESDK_ADMIN_API_KEY)='
. benchmarks/scripts/load-vault-secrets.sh '^(TENSORLAKE_API_KEY|BENCHMARKS_PLATFORM_API_KEY)='
RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
ITERATIONS="${ITERATIONS:-60}"
STAGGER_DELAY_MS="${STAGGER_DELAY_MS:-60000}"
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/sandbox-capabilities.yml
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ jobs:
CONCURRENCY: ${{ github.event.inputs.concurrency || '1' }}
PROVIDER: ${{ matrix.provider }}
run: |
COMMON_KEYS='BENCHMARKS_PLATFORM_API_KEY|COMPUTESDK_ADMIN_API_KEY'
COMMON_KEYS='BENCHMARKS_PLATFORM_API_KEY'
case "$PROVIDER" in
archil) PROVIDER_KEYS='ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID' ;;
arker) PROVIDER_KEYS='ARKER_API_KEY' ;;
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/sandbox-dax-benchmarks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ jobs:
LIGHTNING_INSTANCE_TYPE: cpu-8 # 8 vCPU / 16 GiB, matching the standardized dax profile
ARKER_PLATFORMS: graviton4 # dax only; elsewhere Arker picks the platform automatically
run: |
. benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|ARKER_API_KEY|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_API_KEY|BL_WORKSPACE|BL_DAX_API_KEY|BL_DAX_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MIOSA_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|MSB_API_KEY|MOSAIC_API_URL|MOSAIC_API_TOKEN|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUN_CLOUD_API_KEY|RUNLOOP_API_KEY|SAIL_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)'
. benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|ARKER_API_KEY|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_API_KEY|BL_WORKSPACE|BL_DAX_API_KEY|BL_DAX_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MIOSA_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|MSB_API_KEY|MOSAIC_API_URL|MOSAIC_API_TOKEN|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUN_CLOUD_API_KEY|RUNLOOP_API_KEY|SAIL_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|BENCHMARKS_PLATFORM_API_KEY)'

# If dedicated DAX Blaxel credentials are provisioned, route them to the
# standard names that providers.ts expects, so no provider fork is needed.
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/sandbox-tti-benchmarks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ jobs:
- name: Run benchmark
if: env.SHOULD_RUN == 'true'
run: |
. benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|ARKER_API_KEY|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_API_KEY|BL_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MIOSA_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|MSB_API_KEY|MOSAIC_API_URL|MOSAIC_API_TOKEN|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUN_CLOUD_API_KEY|RUNLOOP_API_KEY|SAIL_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)'
. benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|ARKER_API_KEY|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_API_KEY|BL_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MIOSA_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|MSB_API_KEY|MOSAIC_API_URL|MOSAIC_API_TOKEN|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUN_CLOUD_API_KEY|RUNLOOP_API_KEY|SAIL_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|BENCHMARKS_PLATFORM_API_KEY)'

# Burst shape only; every provider passes the same --run-key, so they
# get-or-create one shared run and rank against each other (each
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/snapshot-fork-benchmarks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ jobs:
run: rm -rf results/snapshot-fork/
- name: Run snapshot/fork test
run: |
. benchmarks/scripts/load-vault-secrets.sh '^(AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|AWS_REGION|S3_BUCKET|S3_SNAPSHOT_ACCESS_KEY_ID|S3_SNAPSHOT_SECRET_ACCESS_KEY|S3_SNAPSHOT_BUCKET|R2_ACCESS_KEY_ID|R2_SECRET_ACCESS_KEY|R2_BUCKET|R2_ACCOUNT_ID|R2_SNAPSHOT_ACCESS_KEY_ID|R2_SNAPSHOT_SECRET_ACCESS_KEY|R2_SNAPSHOT_BUCKET|TIGRIS_STORAGE_ACCESS_KEY_ID|TIGRIS_STORAGE_SECRET_ACCESS_KEY|TIGRIS_STORAGE_BUCKET|TIGRIS_SNAPSHOT_ACCESS_KEY|TIGRIS_SNAPSHOT_SECRET_KEY|TIGRIS_SNAPSHOT_STORAGE_BUCKET|BLOB_READ_WRITE_TOKEN|VERCEL_BLOB_BUCKET|GCS_PROJECT_ID|GCS_BUCKET|GCS_CLIENT_EMAIL|GCS_PRIVATE_KEY|AZURE_ACCOUNT_NAME|AZURE_ACCOUNT_KEY|AZURE_CONTAINER|AZURE_SNAPSHOT_ACCOUNT_NAME|AZURE_SNAPSHOT_ACCOUNT_KEY|AZURE_SNAPSHOT_CONTAINER|TENSORLAKE_API_KEY|TENSORLAKE_API_URL|TENSORLAKE_ORGANIZATION_ID|TENSORLAKE_PROJECT_ID|TENSORLAKE_FILESYSTEM|ARCHIL_S3_ACCESS_KEY_ID|ARCHIL_S3_SECRET_ACCESS_KEY|ARCHIL_BUCKET|ARCHIL_REGION|ARCHIL_BRANCH|NEON_BUCKET|NEON_ENDPOINT|NEON_ACCESS_KEY_ID|NEON_SECRET_ACCESS_KEY|NEON_REGION|NEON_SNAPSHOT_BUCKET|NEON_SNAPSHOT_ENDPOINT|NEON_SNAPSHOT_ACCESS_KEY_ID|NEON_SNAPSHOT_SECRET_ACCESS_KEY|NEON_SNAPSHOT_REGION|MOSAIC_BUCKET|MOSAIC_ACCESS_KEY_ID|MOSAIC_SECRET_ACCESS_KEY|MOSAIC_REGION|MOSAIC_ENDPOINT|MOSAIC_SNAPSHOT_BUCKET|MOSAIC_SNAPSHOT_ACCESS_KEY_ID|MOSAIC_SNAPSHOT_SECRET_ACCESS_KEY|MOSAIC_SNAPSHOT_REGION|MOSAIC_SNAPSHOT_ENDPOINT|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)'
. benchmarks/scripts/load-vault-secrets.sh '^(AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|AWS_REGION|S3_BUCKET|S3_SNAPSHOT_ACCESS_KEY_ID|S3_SNAPSHOT_SECRET_ACCESS_KEY|S3_SNAPSHOT_BUCKET|R2_ACCESS_KEY_ID|R2_SECRET_ACCESS_KEY|R2_BUCKET|R2_ACCOUNT_ID|R2_SNAPSHOT_ACCESS_KEY_ID|R2_SNAPSHOT_SECRET_ACCESS_KEY|R2_SNAPSHOT_BUCKET|TIGRIS_STORAGE_ACCESS_KEY_ID|TIGRIS_STORAGE_SECRET_ACCESS_KEY|TIGRIS_STORAGE_BUCKET|TIGRIS_SNAPSHOT_ACCESS_KEY|TIGRIS_SNAPSHOT_SECRET_KEY|TIGRIS_SNAPSHOT_STORAGE_BUCKET|BLOB_READ_WRITE_TOKEN|VERCEL_BLOB_BUCKET|GCS_PROJECT_ID|GCS_BUCKET|GCS_CLIENT_EMAIL|GCS_PRIVATE_KEY|AZURE_ACCOUNT_NAME|AZURE_ACCOUNT_KEY|AZURE_CONTAINER|AZURE_SNAPSHOT_ACCOUNT_NAME|AZURE_SNAPSHOT_ACCOUNT_KEY|AZURE_SNAPSHOT_CONTAINER|TENSORLAKE_API_KEY|TENSORLAKE_API_URL|TENSORLAKE_ORGANIZATION_ID|TENSORLAKE_PROJECT_ID|TENSORLAKE_FILESYSTEM|ARCHIL_S3_ACCESS_KEY_ID|ARCHIL_S3_SECRET_ACCESS_KEY|ARCHIL_BUCKET|ARCHIL_REGION|ARCHIL_BRANCH|NEON_BUCKET|NEON_ENDPOINT|NEON_ACCESS_KEY_ID|NEON_SECRET_ACCESS_KEY|NEON_REGION|NEON_SNAPSHOT_BUCKET|NEON_SNAPSHOT_ENDPOINT|NEON_SNAPSHOT_ACCESS_KEY_ID|NEON_SNAPSHOT_SECRET_ACCESS_KEY|NEON_SNAPSHOT_REGION|MOSAIC_BUCKET|MOSAIC_ACCESS_KEY_ID|MOSAIC_SECRET_ACCESS_KEY|MOSAIC_REGION|MOSAIC_ENDPOINT|MOSAIC_SNAPSHOT_BUCKET|MOSAIC_SNAPSHOT_ACCESS_KEY_ID|MOSAIC_SNAPSHOT_SECRET_ACCESS_KEY|MOSAIC_SNAPSHOT_REGION|MOSAIC_SNAPSHOT_ENDPOINT|BENCHMARKS_PLATFORM_API_KEY)'

# All matrix jobs for this workflow run share a single platform run. The
# attempt number is included so re-runs do not rejoin completed runs.
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/storage-benchmarks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ jobs:
run: rm -rf results/storage/
- name: Run storage benchmark
run: |
. benchmarks/scripts/load-vault-secrets.sh '^(AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|AWS_REGION|S3_BUCKET|R2_ACCESS_KEY_ID|R2_SECRET_ACCESS_KEY|R2_BUCKET|R2_ACCOUNT_ID|TIGRIS_STORAGE_ACCESS_KEY_ID|TIGRIS_STORAGE_SECRET_ACCESS_KEY|TIGRIS_STORAGE_BUCKET|BLOB_READ_WRITE_TOKEN|VERCEL_BLOB_BUCKET|GCS_PROJECT_ID|GCS_BUCKET|GCS_CLIENT_EMAIL|GCS_PRIVATE_KEY|AZURE_ACCOUNT_NAME|AZURE_ACCOUNT_KEY|AZURE_CONTAINER|TENSORLAKE_API_KEY|TENSORLAKE_API_URL|TENSORLAKE_ORGANIZATION_ID|TENSORLAKE_PROJECT_ID|TENSORLAKE_FILESYSTEM|ARCHIL_S3_ACCESS_KEY_ID|ARCHIL_S3_SECRET_ACCESS_KEY|ARCHIL_BUCKET|ARCHIL_REGION|ARCHIL_BRANCH|NEON_BUCKET|NEON_ENDPOINT|NEON_ACCESS_KEY_ID|NEON_SECRET_ACCESS_KEY|NEON_REGION|MOSAIC_BUCKET|MOSAIC_ACCESS_KEY_ID|MOSAIC_SECRET_ACCESS_KEY|MOSAIC_REGION|MOSAIC_ENDPOINT|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)'
. benchmarks/scripts/load-vault-secrets.sh '^(AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|AWS_REGION|S3_BUCKET|R2_ACCESS_KEY_ID|R2_SECRET_ACCESS_KEY|R2_BUCKET|R2_ACCOUNT_ID|TIGRIS_STORAGE_ACCESS_KEY_ID|TIGRIS_STORAGE_SECRET_ACCESS_KEY|TIGRIS_STORAGE_BUCKET|BLOB_READ_WRITE_TOKEN|VERCEL_BLOB_BUCKET|GCS_PROJECT_ID|GCS_BUCKET|GCS_CLIENT_EMAIL|GCS_PRIVATE_KEY|AZURE_ACCOUNT_NAME|AZURE_ACCOUNT_KEY|AZURE_CONTAINER|TENSORLAKE_API_KEY|TENSORLAKE_API_URL|TENSORLAKE_ORGANIZATION_ID|TENSORLAKE_PROJECT_ID|TENSORLAKE_FILESYSTEM|ARCHIL_S3_ACCESS_KEY_ID|ARCHIL_S3_SECRET_ACCESS_KEY|ARCHIL_BUCKET|ARCHIL_REGION|ARCHIL_BRANCH|NEON_BUCKET|NEON_ENDPOINT|NEON_ACCESS_KEY_ID|NEON_SECRET_ACCESS_KEY|NEON_REGION|MOSAIC_BUCKET|MOSAIC_ACCESS_KEY_ID|MOSAIC_SECRET_ACCESS_KEY|MOSAIC_REGION|MOSAIC_ENDPOINT|BENCHMARKS_PLATFORM_API_KEY)'

# Every size runs in one job, so a run covers all of them and the
# dashboard can compare sizes within it. The attempt number is included
Expand Down
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,5 @@ node_modules/
.DS_Store
dist/
scale-run.log
scale.cjs
scale.mjs
fetch-raw-json.ts
1 change: 1 addition & 0 deletions benchmarks/ai-gateway/ai-gateway-gemini.bench.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ export const config = defineBenchmarkConfig({
phases,
groupBy: 'round',
participants: providers,
customCliFlags: ['--ai-gateway-iterations-cold', '--ai-gateway-iterations-warm'],
scoring: {
metrics: [
{ key: 'coldE2eMs', unit: 'ms', ceiling: 20000, weights: { median: 0.30, p95: 0.15, p99: 0 } },
Expand Down
1 change: 1 addition & 0 deletions benchmarks/ai-gateway/ai-gateway-kimi.bench.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ export const config = defineBenchmarkConfig({
phases,
groupBy: 'round',
participants: providers,
customCliFlags: ['--ai-gateway-iterations-cold', '--ai-gateway-iterations-warm'],
scoring: {
metrics: [
{ key: 'coldE2eMs', unit: 'ms', ceiling: 20000, weights: { median: 0.30, p95: 0.15, p99: 0 } },
Expand Down
1 change: 1 addition & 0 deletions benchmarks/ai-gateway/ai-gateway-openai.bench.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ export const config = defineBenchmarkConfig({
phases,
groupBy: 'round',
participants: providers,
customCliFlags: ['--ai-gateway-iterations-cold', '--ai-gateway-iterations-warm'],
scoring: {
metrics: [
{ key: 'coldE2eMs', unit: 'ms', ceiling: 20000, weights: { median: 0.30, p95: 0.15, p99: 0 } },
Expand Down
1 change: 1 addition & 0 deletions benchmarks/ai-gateway/ai-gateway.bench.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ export const config = defineBenchmarkConfig({
phases,
groupBy: 'round',
participants: providers,
customCliFlags: ['--ai-gateway-iterations-cold', '--ai-gateway-iterations-warm'],
scoring: {
metrics: [
{ key: 'coldE2eMs', unit: 'ms', ceiling: 20000, weights: { median: 0.30, p95: 0.15, p99: 0 } },
Expand Down
4 changes: 2 additions & 2 deletions benchmarks/scale/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,6 @@ RUN pnpm run --silent bundle:scale
FROM node:24-alpine

WORKDIR /app
COPY --from=builder /app/dist/scale.cjs /app/coordinator.cjs
COPY --from=builder /app/dist/scale.mjs /app/coordinator.mjs

CMD ["node", "/app/coordinator.cjs"]
CMD ["node", "/app/coordinator.mjs"]
2 changes: 1 addition & 1 deletion benchmarks/scale/coordinator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ async function main() {
})();

const provider = getProvider(PROVIDER);
const benchApiKey = process.env.COMPUTESDK_API_KEY ?? process.env.COMPUTESDK_ADMIN_API_KEY;
const benchApiKey = process.env.BENCHMARKS_PLATFORM_API_KEY ?? process.env.COMPUTESDK_API_KEY;
const LABEL = process.env.LABEL ?? `scale.${PROVIDER}`;

// Platform orchestration target. start.ts creates the run + plans the workers,
Expand Down
4 changes: 4 additions & 0 deletions benchmarks/scale/inject-require.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
import { createRequire } from 'node:module';
if (typeof globalThis.require === 'undefined') {
globalThis.require = createRequire(import.meta.url);
}
4 changes: 2 additions & 2 deletions benchmarks/scale/scripts/aggregate.ts
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ function usage(): string {
' --help, -h Print this help',
'',
'Required env:',
' COMPUTESDK_API_KEY Bench API token',
' BENCHMARKS_PLATFORM_API_KEY or COMPUTESDK_API_KEY Bench API token',
].join('\n');
}

Expand Down Expand Up @@ -105,7 +105,7 @@ const args = parseArgs();

const client = createBenchmarkClient({
baseUrl: 'https://platform.computesdk.com/api/v1',
apiKey: process.env.COMPUTESDK_ADMIN_API_KEY ?? process.env.COMPUTESDK_API_KEY,
apiKey: process.env.BENCHMARKS_PLATFORM_API_KEY ?? process.env.COMPUTESDK_API_KEY,
});

// Some analytics endpoints lag the run (events still importing) — degrade to a
Expand Down
Loading
Loading