From 23933e2daee2ab3652eeb677c73bad9e33d4006b Mon Sep 17 00:00:00 2001 From: david Date: Mon, 24 Aug 2026 16:16:41 +0000 Subject: [PATCH 1/4] benchsdk: clean up SDK surface, auth fallbacks, CLI validation, and scaffolding Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- package.json | 4 +- packages/benchsdk-api/package.json | 4 +- packages/benchsdk-api/src/client.ts | 4 +- packages/benchsdk-api/src/types.ts | 2 +- .../src/__tests__/runner.test.ts | 4 +- packages/benchsdk-runner/src/runner.ts | 20 +- packages/benchsdk-worker/package.json | 4 +- packages/benchsdk-worker/src/participants.ts | 13 +- .../benchsdk/src/__tests__/client.test.ts | 7 +- .../src/__tests__/public-api.contract.test.ts | 20 +- packages/benchsdk/src/client.ts | 18 +- packages/benchsdk/src/index.ts | 31 +- packages/benchsdk/src/metrics.ts | 5 - packages/benchsdk/src/participants.ts | 2 - packages/benchsdk/src/reporter.ts | 9 - packages/benchsdk/src/types.ts | 746 ------------------ .../create-bench/src/__tests__/cli.test.ts | 13 +- packages/create-bench/src/index.ts | 13 +- pnpm-lock.yaml | 99 +-- 19 files changed, 134 insertions(+), 884 deletions(-) delete mode 100644 packages/benchsdk/src/metrics.ts delete mode 100644 packages/benchsdk/src/participants.ts delete mode 100644 packages/benchsdk/src/reporter.ts delete mode 100644 packages/benchsdk/src/types.ts diff --git a/package.json b/package.json index 810888171..2762b2cdd 100644 --- a/package.json +++ b/package.json @@ -205,11 +205,11 @@ }, "devDependencies": { "@changesets/cli": "^2.31.1", - "@types/node": "^26.2.0", + "@types/node": "^20.0.0", "@types/pg": "^8.23.1", "esbuild": "^0.28.2", "tsx": "^4.23.12", - "typescript": "^6.0.3" + "typescript": "^5.0.0" }, "packageManager": "pnpm@10.33.0" } diff --git a/packages/benchsdk-api/package.json b/packages/benchsdk-api/package.json index de24cdeff..783c66991 100644 --- a/packages/benchsdk-api/package.json +++ b/packages/benchsdk-api/package.json @@ -27,8 +27,8 @@ "lint": "eslint", "prepare": "tsup", "pretest": "tsup", - "test": "vitest run", - "test:coverage": "vitest run --coverage", + "test": "vitest run --passWithNoTests", + "test:coverage": "vitest run --coverage --passWithNoTests", "test:watch": "vitest watch", "typecheck": "tsc --noEmit" }, diff --git a/packages/benchsdk-api/src/client.ts b/packages/benchsdk-api/src/client.ts index f82b5e96a..1780ac9b2 100644 --- a/packages/benchsdk-api/src/client.ts +++ b/packages/benchsdk-api/src/client.ts @@ -72,7 +72,9 @@ function queryString(input: Record): string { } function getApiKey(input?: string): string | undefined { - return input ?? (typeof process !== 'undefined' ? process.env.COMPUTESDK_ADMIN_API_KEY ?? process.env.COMPUTESDK_API_KEY : undefined); + if (input) return input; + if (typeof process === 'undefined') return undefined; + return process.env.COMPUTESDK_ADMIN_API_KEY ?? process.env.COMPUTESDK_API_KEY ?? process.env.BENCHMARKS_PLATFORM_API_KEY; } function getErrorCode(error: unknown): string { diff --git a/packages/benchsdk-api/src/types.ts b/packages/benchsdk-api/src/types.ts index 7fd0ad67b..0c748b777 100644 --- a/packages/benchsdk-api/src/types.ts +++ b/packages/benchsdk-api/src/types.ts @@ -4,7 +4,7 @@ export type JsonObject = { [key: string]: JsonValue }; export interface BenchmarkClientConfig { /** API base URL. Defaults to https://platform.computesdk.com/api/v1. */ baseUrl?: string; - /** Bearer token. Defaults to process.env.COMPUTESDK_ADMIN_API_KEY, then process.env.COMPUTESDK_API_KEY. */ + /** Bearer token. Defaults to COMPUTESDK_ADMIN_API_KEY, then COMPUTESDK_API_KEY, then BENCHMARKS_PLATFORM_API_KEY. */ apiKey?: string; /** Custom fetch implementation, mostly useful for tests. */ fetch?: typeof fetch; diff --git a/packages/benchsdk-runner/src/__tests__/runner.test.ts b/packages/benchsdk-runner/src/__tests__/runner.test.ts index e51f2baa9..ab59656b7 100644 --- a/packages/benchsdk-runner/src/__tests__/runner.test.ts +++ b/packages/benchsdk-runner/src/__tests__/runner.test.ts @@ -78,8 +78,8 @@ describe('parseCliArgs', () => { expect(() => parseCliArgs(['--group-by', 'nope'])).toThrow('--group-by'); }); - it('ignores unknown flags', () => { - expect(parseCliArgs(['--unknown', 'x', '--iterations', '3'])).toEqual({ iterations: 3 }); + it('throws on unknown flags', () => { + expect(() => parseCliArgs(['--unknown', 'x', '--iterations', '3'])).toThrow('Unknown flag'); }); it('throws on non-numeric numeric flags', () => { diff --git a/packages/benchsdk-runner/src/runner.ts b/packages/benchsdk-runner/src/runner.ts index bce0d06c3..3856f3f6b 100644 --- a/packages/benchsdk-runner/src/runner.ts +++ b/packages/benchsdk-runner/src/runner.ts @@ -162,12 +162,13 @@ async function runStepWithClient( } /** - * Parses the orchestration flags this runner understands, ignoring anything - * else. Supports both `--flag value` and `--flag=value`; `--provider` accepts + * Parses the orchestration flags this runner understands, rejecting unknown + * flags. Supports both `--flag value` and `--flag=value`; `--provider` accepts * a comma-separated list and may be repeated. */ export function parseCliArgs(argv: string[]): CliArgs { const args: CliArgs = {}; + const unknown: string[] = []; const readValue = (raw: string, i: number): { value: string; nextIndex: number } => { const eq = raw.indexOf('='); @@ -264,10 +265,15 @@ export function parseCliArgs(argv: string[]): CliArgs { args.noIngest = true; break; default: + unknown.push(name); break; } } + if (unknown.length > 0) { + throw new Error(`Unknown flag(s): ${unknown.join(', ')}`); + } + if (!args.noIngest && isEnvNoIngest()) { args.noIngest = true; } @@ -355,11 +361,15 @@ function defaultOnResult(record: TaskResultRecord, meta: { iterations: number; p function resolvePlatform(): { baseUrl: string; apiKey: string } { const root = (process.env.BENCHMARKS_PLATFORM_URL || DEFAULT_PLATFORM_URL).replace(/\/+$/, ''); - const apiKey = process.env.BENCHMARKS_PLATFORM_API_KEY; + const apiKey = + process.env.BENCHMARKS_PLATFORM_API_KEY ?? + process.env.COMPUTESDK_ADMIN_API_KEY ?? + process.env.COMPUTESDK_API_KEY; if (!apiKey) { throw new Error( - 'BENCHMARKS_PLATFORM_API_KEY is required. Create an org-scoped API key ' + - 'in your organization settings on the platform and set it in your .env.' + 'An API key is required. Set one of BENCHMARKS_PLATFORM_API_KEY, COMPUTESDK_ADMIN_API_KEY, ' + + 'or COMPUTESDK_API_KEY in your environment. Create an org-scoped API key in your ' + + 'organization settings on the platform.' ); } return { diff --git a/packages/benchsdk-worker/package.json b/packages/benchsdk-worker/package.json index 72493e6cf..1c54451f4 100644 --- a/packages/benchsdk-worker/package.json +++ b/packages/benchsdk-worker/package.json @@ -27,8 +27,8 @@ "lint": "eslint", "prepare": "tsup", "pretest": "tsup", - "test": "vitest run", - "test:coverage": "vitest run --coverage", + "test": "vitest run --passWithNoTests", + "test:coverage": "vitest run --coverage --passWithNoTests", "test:watch": "vitest watch", "typecheck": "tsc --noEmit" }, diff --git a/packages/benchsdk-worker/src/participants.ts b/packages/benchsdk-worker/src/participants.ts index 806acfa8f..01d098444 100644 --- a/packages/benchsdk-worker/src/participants.ts +++ b/packages/benchsdk-worker/src/participants.ts @@ -10,7 +10,6 @@ export interface BaseParticipant { requiredEnvVars: string[]; } -/** /** * Filters `participants` down to those whose `requiredEnvVars` are all set * in `process.env`. Returns an object `{ available, skipped }` where `skipped` @@ -35,9 +34,9 @@ export function filterParticipantsByEnv( } /** - * Filters `all` down to the requested `names`, exiting with a clear error - * if any name is unrecognized. Returns `all` unchanged when `names` is - * undefined (no filter specified). + * Filters `all` down to the requested `names`, throwing a clear error if any + * name is unrecognized. Returns `all` unchanged when `names` is undefined + * (no filter specified). */ export function selectParticipants( all: T[], @@ -46,9 +45,9 @@ export function selectParticipants( if (!names) return all; const unknown = names.filter((n) => !all.some((p) => p.name === n)); if (unknown.length > 0) { - console.error(`Unknown participant(s): ${unknown.join(', ')}`); - console.error(`Available: ${all.map((p) => p.name).join(', ')}`); - process.exit(1); + throw new Error( + `Unknown participant(s): ${unknown.join(', ')}. Available: ${all.map((p) => p.name).join(', ')}`, + ); } return all.filter((p) => names.includes(p.name)); } diff --git a/packages/benchsdk/src/__tests__/client.test.ts b/packages/benchsdk/src/__tests__/client.test.ts index 78f3473b5..39af8141d 100644 --- a/packages/benchsdk/src/__tests__/client.test.ts +++ b/packages/benchsdk/src/__tests__/client.test.ts @@ -1,8 +1,7 @@ import { describe, expect, it, vi } from 'vitest'; -import { BenchmarkApiError, createBenchmarkClient } from '../client'; -import { createSystemMetricsCollector } from '../metrics'; -import { BenchmarkReporter } from '../reporter'; -import type { BenchmarkAssignment } from '../types'; +import { createBenchmarkClient } from '../client'; +import { BenchmarkApiError, BenchmarkReporter, createSystemMetricsCollector } from '../index'; +import type { BenchmarkAssignment } from '../index'; function jsonResponse(body: unknown, status = 200): Response { return new Response(JSON.stringify(body), { diff --git a/packages/benchsdk/src/__tests__/public-api.contract.test.ts b/packages/benchsdk/src/__tests__/public-api.contract.test.ts index 155f503fb..5badb70ac 100644 --- a/packages/benchsdk/src/__tests__/public-api.contract.test.ts +++ b/packages/benchsdk/src/__tests__/public-api.contract.test.ts @@ -3,14 +3,15 @@ import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { afterEach, describe, expect, it, vi } from 'vitest'; +import * as barrel from '../index'; import { BenchmarkApiError, + BenchmarkReporter, + claimBenchmarkReporter, createBenchmarkClient, -} from '../client'; -import { BenchmarkReporter, claimBenchmarkReporter } from '../reporter'; -import { createSystemMetricsCollector } from '../metrics'; -import * as barrel from '../index'; -import type { BenchmarkAssignment, BenchmarkClient } from '../types'; + createSystemMetricsCollector, +} from '../index'; +import type { BenchmarkAssignment, BenchmarkClient } from '../index'; // The public type surface is imported by name here purely to prove every exported // type resolves against the package barrel (the consumer-compile contract). @@ -1391,8 +1392,8 @@ describe('createSystemMetricsCollector', () => { // --------------------------------------------------------------------------- describe('public API surface and type exports', () => { const here = dirname(fileURLToPath(import.meta.url)); - // The barrel re-exports the full public type surface from types/reporter/metrics. - // Internal helper types must never appear here. + // The barrel re-exports the full public type surface from @benchsdk/api and + // @benchsdk/worker. Internal helper types must never appear here. const TYPES_EXPORTS = [ 'BenchmarkAssignment', 'BenchmarkArtifact', 'BenchmarkClient', 'BenchmarkClientConfig', 'BenchmarkConcurrencyPoint', 'BenchmarkAnalyticsReadiness', 'BenchmarkEventRateBucket', 'BenchmarkFailurePoint', @@ -1473,11 +1474,6 @@ describe('public API surface and type exports', () => { expect(uniqueExportedTypeNames).not.toContain(internal); } - // The internal types do exist in types.ts, they are just not surfaced. - const typesSource = readFileSync(join(here, '..', 'types.ts'), 'utf8'); - for (const internal of INTERNAL_TYPES) { - expect(typesSource).toMatch(new RegExp(`export (interface|type) ${internal}\\b`)); - } }); it('VAL-SDK-091: every value export is present on the package barrel', () => { diff --git a/packages/benchsdk/src/client.ts b/packages/benchsdk/src/client.ts index b988ad69d..b2bbe8442 100644 --- a/packages/benchsdk/src/client.ts +++ b/packages/benchsdk/src/client.ts @@ -1,17 +1,19 @@ import { createBenchmarkClient as createApiClient, BenchmarkApiError } from '@benchsdk/api'; import { runWorker } from '@benchsdk/worker'; -import type { BenchmarkClient, BenchmarkClientConfig, RunWorkerOptions, RunWorkerResult } from './types'; +import type { BenchmarkClient as BenchmarkApiClient, BenchmarkClientConfig, RunWorkerOptions, RunWorkerResult } from '@benchsdk/api'; export { BenchmarkApiError }; +export type BenchmarkClient = BenchmarkApiClient & { + runWorker(options: RunWorkerOptions): Promise; +}; + +export type { BenchmarkClientConfig } from '@benchsdk/api'; + export function createBenchmarkClient(config: BenchmarkClientConfig = {}): BenchmarkClient { const apiClient = createApiClient(config); - function runWorkerWrapped(options: RunWorkerOptions): Promise { - return runWorker(client as any, options as unknown as any) as Promise; - } - const client = { + return { ...apiClient, - runWorker: runWorkerWrapped, - } as BenchmarkClient; - return client; + runWorker: (options: RunWorkerOptions) => runWorker(apiClient, options), + }; } diff --git a/packages/benchsdk/src/index.ts b/packages/benchsdk/src/index.ts index be7af4e06..256d11832 100644 --- a/packages/benchsdk/src/index.ts +++ b/packages/benchsdk/src/index.ts @@ -1,14 +1,9 @@ -export { - BenchmarkApiError, - createBenchmarkClient, -} from './client'; -export { BenchmarkReporter, claimBenchmarkReporter } from './reporter'; -export { createSystemMetricsCollector } from './metrics'; +export { BenchmarkApiError, createBenchmarkClient } from './client'; +export type { BenchmarkClient, BenchmarkClientConfig } from './client'; + export type { BenchmarkAssignment, BenchmarkArtifact, - BenchmarkClient, - BenchmarkClientConfig, BenchmarkConcurrencyPoint, BenchmarkAnalyticsReadiness, BenchmarkEventRateBucket, @@ -70,31 +65,31 @@ export type { UpdateParticipantInput, UpdateRunInput, UpdateWorkerInput, - WorkerConcurrencySample, - WorkerFinishContext, - WorkerHeartbeatInput, UpsertBenchmarkInput, UpsertParticipantInput, UploadWorkerArtifactInput, -} from './types'; + WorkerConcurrencySample, + WorkerFinishContext, + WorkerHeartbeatInput, +} from '@benchsdk/api'; export { + BenchmarkReporter, + claimBenchmarkReporter, + createSystemMetricsCollector, filterParticipantsByEnv, + runWorker, selectParticipants, -} from './participants'; +} from '@benchsdk/worker'; export type { BaseParticipant, -} from './participants'; -export type { BenchmarkReporterArtifactInput, BenchmarkReporterBarrierInput, BenchmarkReporterBarrierResult, BenchmarkReporterConfig, BenchmarkReporterHeartbeatInput, BenchmarkReporterProgress, -} from './reporter'; -export type { BenchmarkSystemMetricsCollector, BenchmarkSystemMetricsSample, -} from './metrics'; +} from '@benchsdk/worker'; diff --git a/packages/benchsdk/src/metrics.ts b/packages/benchsdk/src/metrics.ts deleted file mode 100644 index da2f84c5f..000000000 --- a/packages/benchsdk/src/metrics.ts +++ /dev/null @@ -1,5 +0,0 @@ -export { createSystemMetricsCollector } from '@benchsdk/worker'; -export type { - BenchmarkSystemMetricsCollector, - BenchmarkSystemMetricsSample, -} from '@benchsdk/worker'; diff --git a/packages/benchsdk/src/participants.ts b/packages/benchsdk/src/participants.ts deleted file mode 100644 index 54fadd41b..000000000 --- a/packages/benchsdk/src/participants.ts +++ /dev/null @@ -1,2 +0,0 @@ -export { filterParticipantsByEnv, selectParticipants } from '@benchsdk/worker'; -export type { BaseParticipant } from '@benchsdk/worker'; diff --git a/packages/benchsdk/src/reporter.ts b/packages/benchsdk/src/reporter.ts deleted file mode 100644 index f1f5c68f6..000000000 --- a/packages/benchsdk/src/reporter.ts +++ /dev/null @@ -1,9 +0,0 @@ -export { BenchmarkReporter, claimBenchmarkReporter } from '@benchsdk/worker'; -export type { - BenchmarkReporterArtifactInput, - BenchmarkReporterBarrierInput, - BenchmarkReporterBarrierResult, - BenchmarkReporterConfig, - BenchmarkReporterHeartbeatInput, - BenchmarkReporterProgress, -} from '@benchsdk/worker'; diff --git a/packages/benchsdk/src/types.ts b/packages/benchsdk/src/types.ts deleted file mode 100644 index 595f32c18..000000000 --- a/packages/benchsdk/src/types.ts +++ /dev/null @@ -1,746 +0,0 @@ -export type JsonValue = string | number | boolean | null | JsonObject | JsonValue[]; -export type JsonObject = { [key: string]: JsonValue }; - -export interface BenchmarkClientConfig { - /** API base URL. Defaults to https://platform.computesdk.com/api/v1. */ - baseUrl?: string; - /** Bearer token. Defaults to process.env.COMPUTESDK_ADMIN_API_KEY, then process.env.COMPUTESDK_API_KEY. */ - apiKey?: string; - /** Custom fetch implementation, mostly useful for tests. */ - fetch?: typeof fetch; -} - -export interface BenchmarkResource { - id: string; - slug: string; - name: string; - status?: string; - config?: JsonObject; - defaultRunConfig?: JsonObject; -} - -export type BenchmarkRunStatus = 'planned' | 'in_progress' | 'completed' | 'failed'; -export type BenchmarkWorkerStatus = 'pending' | 'running' | 'completed' | 'failed'; - -export interface BenchmarkRun { - id: string; - benchmarkId: string; - name?: string | null; - status: BenchmarkRunStatus | string; - /** Idempotency key: runs created with the same key (per org + benchmark) are the same run. */ - runKey?: string | null; - totalTasks: number; - /** The run declared no size: `totalTasks` is the sum of what its participants declare. */ - participantSized?: boolean; - workerCount: number; - config?: JsonObject; - createdAt?: string; - updatedAt?: string; -} - -export interface BenchmarkParticipant { - id: string; - benchmarkId: string; - runId: string; - slug: string; - label?: string | null; - provider?: string | null; - status: BenchmarkRunStatus | string; - totalTasks: number; - workerCount: number; - config?: JsonObject; -} - -export interface BenchmarkRunWorker { - id: string; - benchmarkId: string; - runId: string; - participantId: string; - workerIndex: number; - workerCount: number; - taskIndexStart: number; - taskIndexEnd: number; - targetConcurrency: number; - status: BenchmarkWorkerStatus | string; - progressDone?: number; - progressInFlight?: number; - progressErrors?: number; - progressTotal?: number; - currentStep?: string | null; - concurrency?: WorkerConcurrencySample[]; -} - -export interface BenchmarkWorkerAttempt { - id: string; - benchmarkId: string; - runId: string; - participantId: string; - workerId: string; - attemptNumber: number; - status: string; -} - -export interface BenchmarkAssignment { - benchmarkId: string; - benchmarkSlug: string; - runId: string; - participantId: string; - participantSlug: string; - provider?: string | null; - workerId: string; - workerIndex: number; - workerCount: number; - attemptId: string; - attemptNumber: number; - taskRange: { - start: number; - end: number; - count: number; - }; - targetConcurrency: number; - config?: JsonObject; -} - -export interface UpsertBenchmarkInput { - name: string; - status?: string; - config?: JsonObject; - defaultRunConfig?: JsonObject; -} - -export interface UpdateBenchmarkInput { - name?: string; - status?: string; - config?: JsonObject; - defaultRunConfig?: JsonObject; -} - -export interface CreateRunInput { - /** - * Idempotency key for get-or-create: sibling callers passing the same key - * (per org + benchmark) converge on one run instead of each opening its own. - */ - runKey?: string; - /** Omit to open a participant-sized run: each participant declares its own size when it registers. */ - totalTasks?: number; - workerCount?: number; - participants?: string[]; - config?: JsonObject; -} - -export interface UpdateRunInput { - status?: BenchmarkRunStatus; - config?: JsonObject; -} - -export interface UpsertParticipantInput { - label?: string; - provider?: string; - status?: string; - totalTasks?: number; - workerCount?: number; - config?: JsonObject; -} - -export type UpdateParticipantInput = UpsertParticipantInput; - -export interface UpdateWorkerInput { - status?: BenchmarkWorkerStatus; - progressDone?: number; - progressInFlight?: number; - progressErrors?: number; - progressTotal?: number; -} - -export interface ClaimWorkerInput { - processKind?: string; - processKey?: string; -} - -export interface PlanWorkersInput { - workerCount?: number; - targetConcurrency?: number; - config?: JsonObject; -} - -export interface TaskResultRecord { - taskIndex: number; - status: string; - startedAt?: string; - completedAt?: string; - latencyMs?: number; - firstCommandMs?: number | null; - errorCode?: string | null; - steps?: TaskStepRecord[]; - data?: JsonObject; -} - -export interface TaskStepRecord { - name: string; - status: 'success' | 'error'; - startedAt?: string; - completedAt?: string; - latencyMs?: number; - errorCode?: string | null; - data?: JsonObject; - /** Number of parallel invocations requested for this step. */ - concurrency?: number; - /** Per-iteration timeout in milliseconds applied to this step. */ - timeoutMs?: number; -} - -export interface SendTaskResultsInput { - benchmarkSlug: string; - runId: string; - workerId: string; - attemptId: string; - sequenceNumber: number; - isFinal: boolean; - records: TaskResultRecord[]; -} - -export interface TaskResultsResponse { - accepted?: number; - eventBatchId?: string; - queued?: boolean; - eventBatch?: unknown; - duplicate?: boolean; - queueMessageId?: string; -} - -export interface CreateWorkerArtifactInput { - attemptId: string; - kind: string; - contentType?: string; - name?: string; - metadata?: JsonObject; -} - -export interface UploadWorkerArtifactInput extends CreateWorkerArtifactInput { - body: BodyInit; -} - -export interface BenchmarkArtifact { - id?: string; - artifactId?: string; - benchmarkId?: string; - runId?: string; - participantId?: string; - participantSlug?: string; - workerId?: string; - attemptId?: string; - kind: string; - name?: string | null; - contentType?: string | null; - objectKey?: string; - uploadUrl?: string; - uploadUrlExpiresAt?: string; - metadata?: JsonObject; - createdAt?: string; -} - -export interface CreateWorkerArtifactResponse { - artifact?: BenchmarkArtifact; - artifactId?: string; - uploadUrl?: string; - uploadUrlExpiresAt?: string; - objectKey?: string; -} - -export interface BenchmarkResultLatencySummary { - min: number | null; - avg: number | null; - p50: number | null; - p95: number | null; - p99: number | null; - max: number | null; -} - -export interface BenchmarkResultSummary { - taskCount: number; - successCount: number; - errorCount: number; - otherCount: number; - latencyCount: number; - successRate: number; - latencyMs: BenchmarkResultLatencySummary; - firstStartedAt: string | null; - lastCompletedAt: string | null; -} - -export interface BenchmarkParticipantResultSummary extends BenchmarkResultSummary { - participantSlug: string; - provider: string | null; -} - -export interface BenchmarkStepResultSummary { - participantSlug: string; - provider: string | null; - stepName: string; - stepCount: number; - successCount: number; - errorCount: number; - otherCount: number; - latencyCount: number; - successRate: number; - latencyMs: BenchmarkResultLatencySummary; -} - -export interface BenchmarkResultsOverviewInput { - limit?: number; -} - -export type BenchmarkAnalyticsReadiness = 'ready' | 'complete' | 'partial' | 'pending' | 'unavailable' | 'failed'; - -export interface BenchmarkRunAnalyticsSummary { - status: BenchmarkAnalyticsReadiness; - eventBatches: number; - persisted: number; - queued: number; - failed: number; - imports: { - pending: number; - importing: number; - imported: number; - failed: number; - missing: number; - }; -} - -export interface BenchmarkResultsOverviewAnalytics { - status: BenchmarkAnalyticsReadiness; - query: 'available' | 'unavailable'; - error?: string; -} - -export interface BenchmarkResultsOverviewRun { - run: BenchmarkRun; - analytics: BenchmarkRunAnalyticsSummary; - participants: Array; -} - -export interface BenchmarkResultsOverview { - benchmark: Pick; - generatedAt: string; - analytics: BenchmarkResultsOverviewAnalytics; - items: BenchmarkResultsOverviewRun[]; -} - -export interface BenchmarkRunResults { - benchmark: Pick; - run: Pick; - generatedAt: string; - overall: BenchmarkResultSummary; - participants: BenchmarkParticipantResultSummary[]; - steps: BenchmarkStepResultSummary[]; -} - -export interface BenchmarkRunTaskResultsInput { - bucketSize?: number; - failureLimit?: number; -} - -export interface BenchmarkTaskBucket { - participantSlug: string; - provider: string | null; - bucketStart: number; - bucketEnd: number; - taskIndexMidpoint: number; - taskCount: number; - successCount: number; - errorCount: number; - latencyMs: Pick; -} - -export interface BenchmarkFailurePoint { - participantSlug: string; - provider: string | null; - taskIndex: number; - errorCode: string | null; -} - -export interface BenchmarkRunTaskResults { - run: { id: string }; - generatedAt: string; - bucketSize: number; - buckets: BenchmarkTaskBucket[]; - failures: BenchmarkFailurePoint[]; -} - -export interface BenchmarkRunTimelineInput { - bucketMs?: number; -} - -export interface BenchmarkEventRateBucket { - participantSlug: string; - provider: string | null; - tMs: number; - completed: number; - succeeded: number; - failed: number; -} - -export interface BenchmarkConcurrencyPoint { - participantSlug: string; - provider: string | null; - workerId: string; - recordedAt: string; - tMs: number; - step: string; - active: number; - target: number; -} - -export interface BenchmarkRunTimeline { - run: { id: string }; - generatedAt: string; - eventRate: { - bucketMs: number; - buckets: BenchmarkEventRateBucket[]; - }; - concurrency: { - firstRecordedAt: string | null; - heartbeatCount: number; - points: BenchmarkConcurrencyPoint[]; - }; -} - -export interface BenchmarkRunImportsSummary { - eventBatches: number; - persisted: number; - queued: number; - failed: number; - imports: { - pending: number; - importing: number; - imported: number; - failed: number; - missing: number; - }; -} - -export interface BenchmarkRunImportItem { - eventBatchId: string; - batchType: string; - sequenceNumber: number; - batchStatus: string; - eventCount: number; - objectKey: string | null; - batchErrorMessage: string | null; - createdAt: string; - persistedAt: string | null; - sink: string | null; - importStatus: string | null; - importAttempts: number | null; - importedAt: string | null; - failedAt: string | null; - importErrorMessage: string | null; -} - -export interface BenchmarkRunImports { - run: { id: string }; - generatedAt: string; - summary: BenchmarkRunImportsSummary; - items: BenchmarkRunImportItem[]; -} - -export interface WorkerConcurrencySample { - step: string; - active: number; - target: number; -} - -export interface WorkerHeartbeatInput { - attemptId: string; - progressDone?: number; - progressInFlight?: number; - progressErrors?: number; - progressTotal?: number; - currentStep?: string | null; - concurrency?: WorkerConcurrencySample[]; -} - -export interface RunProgressConcurrency { - step: string; - active: number; - target: number; - ready: boolean; - freshWorkerCount: number; -} - -export type RunProgressStatus = 'planned' | 'in_progress' | 'completed' | 'failed'; - -export interface RunProgressWorkerCounts { - pending: number; - running: number; - completed: number; - failed: number; - stale: number; - total: number; -} - -export interface RunProgressTaskCounts { - done: number; - inFlight: number; - errors: number; - total: number; - completionRatio: number; -} - -export interface RunProgressParticipantCounts { - planned: number; - inProgress: number; - completed: number; - failed: number; - total: number; -} - -export interface RunProgressSummary { - status: RunProgressStatus; - started: boolean; - completed: boolean; - participants: RunProgressParticipantCounts; -} - -export interface RunProgressParticipant { - id: string; - slug: string; - provider?: string | null; - status: RunProgressStatus; - totalTasks: number; - workerCount: number; - workers: RunProgressWorkerCounts; - tasks: RunProgressTaskCounts; - concurrency: RunProgressConcurrency[]; -} - -export interface RunProgress { - run: { - id: string; - status: string; - totalTasks: number; - workerCount: number; - }; - summary: RunProgressSummary; - freshnessWindowSeconds: number; - generatedAt: string; - participants: RunProgressParticipant[]; -} - -export interface RunWorkerContext { - assignment: BenchmarkAssignment; - taskIndex: number; - step(name: string, fn: () => Promise | T, options?: DefineStepOptions): Promise; - /** - * Attaches a JSON measurement to the platform. Called inside a `step`, it - * lands on that step's `data`; called at task top-level, on the task record's - * `data`. Repeated calls merge (shallow). Use this for anything you want on - * the platform — step return values are control flow and are never recorded. - */ - measure(data: JsonObject): void; - /** Appends a line to the worker log, uploaded as an artifact when the worker finishes. */ - log(message: string, meta?: JsonObject): void; -} - -export interface WorkerFinishContext { - assignment: BenchmarkAssignment; - records: TaskResultRecord[]; - status: 'success' | 'error'; - client: BenchmarkClient; - uploadArtifact(input: Omit): Promise; -} - -export interface DefineStepOptions { - /** Report this step as active in heartbeat concurrency samples. Defaults to true. */ - reportConcurrency?: boolean; - /** Per-worker target for this step. Defaults to worker concurrency/assignment target. */ - concurrency?: number; - /** Number of parallel invocations the step function should run internally. Used by the runner to record step-level concurrency. */ - stepConcurrency?: number; - /** Per-invocation timeout in milliseconds for this step. Used by the runner to record step-level timeout metadata. */ - timeoutMs?: number; - /** Readiness coordination mode. Defaults to internal. */ - readiness?: 'poll' | 'internal'; - /** Poll interval while waiting for readiness. Defaults to 1000ms. */ - readyPollIntervalMs?: number; - /** Maximum time to wait for readiness. Defaults to no timeout. */ - readyTimeoutMs?: number; -} - -/** - * The unit of work a worker runs, once per task index. Steps are declared - * imperatively via `context.step(...)`; this is the sole task shape the worker - * engine accepts. Higher-level authoring (`defineTask`) lives in - * `@benchsdk/runner`, which compiles down to a function of this shape. - */ -export type TaskFunction = (context: RunWorkerContext) => Promise | JsonObject | void; - -export interface RunWorkerResult { - assignment: BenchmarkAssignment | null; - records: TaskResultRecord[]; -} - -export interface RunWorkerOptions { - benchmarkSlug: string; - runId: string; - participantSlug: string; - processKind?: string; - processKey?: string; - concurrency?: number; - batchSize?: number; - flushIntervalMs?: number; - heartbeatIntervalMs?: number; - readyPollIntervalMs?: number; - onResult?: (record: TaskResultRecord) => void; - /** Runs once after final result flush and before worker completion/failure is reported. */ - onFinish?: (context: WorkerFinishContext) => Promise | void; - task: TaskFunction; -} - -export interface BenchmarkRunSummaryMetric { - name: string; - unit: string; - median: number; - p95: number; - p99: number; -} - -export interface BenchmarkRunSummaryScalar { - name: string; - value: number; - unit: string; -} - -export interface BenchmarkRunSummaryResult { - provider: string; - dimensions?: Record; - metrics: BenchmarkRunSummaryMetric[]; - scalars?: BenchmarkRunSummaryScalar[]; - compositeScore: number; - successRate: number; - scoringVersion?: string | null; - skipped: boolean; - skipReason?: string | null; -} - -export interface BenchmarkRunSummaryRunMetadata { - gitSha?: string; - gitRef?: string; - triggeredBy?: string; - nodeVersion?: string; - platform?: string; - arch?: string; -} - -export interface BenchmarkRunSummaryInput { - run: BenchmarkRunSummaryRunMetadata; - results: BenchmarkRunSummaryResult[]; -} - -export interface BenchmarkClient { - upsertBenchmark(slug: string, input: UpsertBenchmarkInput): Promise; - updateBenchmark(slug: string, input: UpdateBenchmarkInput): Promise; - getBenchmark(slug: string): Promise; - listBenchmarks(): Promise; - createRun(benchmarkSlug: string, input: CreateRunInput): Promise<{ - run: BenchmarkRun; - participants: BenchmarkParticipant[]; - /** The slug of the org the run was attributed to, resolved server-side from the caller's API key. */ - organizationSlug: string; - }>; - listRuns(benchmarkSlug: string): Promise; - getRun(benchmarkSlug: string, runId: string): Promise; - updateRun(benchmarkSlug: string, runId: string, input: UpdateRunInput): Promise; - upsertParticipant( - benchmarkSlug: string, - runId: string, - participantSlug: string, - input?: UpsertParticipantInput, - ): Promise; - updateParticipant( - benchmarkSlug: string, - runId: string, - participantSlug: string, - input: UpdateParticipantInput, - ): Promise; - listParticipants(benchmarkSlug: string, runId: string): Promise; - getParticipant( - benchmarkSlug: string, - runId: string, - participantSlug: string, - ): Promise; - listWorkers( - benchmarkSlug: string, - runId: string, - participantSlug: string, - ): Promise; - planWorkers( - benchmarkSlug: string, - runId: string, - participantSlug: string, - input?: PlanWorkersInput, - ): Promise; - getWorker(benchmarkSlug: string, runId: string, workerId: string): Promise; - updateWorker( - benchmarkSlug: string, - runId: string, - workerId: string, - input: UpdateWorkerInput, - ): Promise; - getRunProgress(benchmarkSlug: string, runId: string): Promise; - claimWorker( - benchmarkSlug: string, - runId: string, - participantSlug: string, - input?: ClaimWorkerInput, - ): Promise; - releaseWorker(benchmarkSlug: string, runId: string, workerId: string, attemptId: string): Promise<{ - worker: BenchmarkRunWorker; - attempt: BenchmarkWorkerAttempt; - }>; - sendTaskResults(input: SendTaskResultsInput): Promise; - heartbeatWorker(benchmarkSlug: string, runId: string, workerId: string, input: WorkerHeartbeatInput): Promise<{ - worker: BenchmarkRunWorker; - attempt: BenchmarkWorkerAttempt; - }>; - completeWorker(benchmarkSlug: string, runId: string, workerId: string, attemptId: string): Promise<{ - worker: BenchmarkRunWorker; - attempt: BenchmarkWorkerAttempt; - }>; - failWorker( - benchmarkSlug: string, - runId: string, - workerId: string, - attemptId: string, - error?: unknown, - ): Promise<{ worker: BenchmarkRunWorker; attempt: BenchmarkWorkerAttempt }>; - createWorkerArtifact( - benchmarkSlug: string, - runId: string, - workerId: string, - input: CreateWorkerArtifactInput, - ): Promise; - uploadWorkerArtifact( - benchmarkSlug: string, - runId: string, - workerId: string, - input: UploadWorkerArtifactInput, - ): Promise; - listRunArtifacts(benchmarkSlug: string, runId: string): Promise; - listWorkerArtifacts(benchmarkSlug: string, runId: string, workerId: string): Promise; - getBenchmarkResults(benchmarkSlug: string, input?: BenchmarkResultsOverviewInput): Promise; - getRunResults(benchmarkSlug: string, runId: string): Promise; - getRunTaskResults( - benchmarkSlug: string, - runId: string, - input?: BenchmarkRunTaskResultsInput, - ): Promise; - getRunTimeline( - benchmarkSlug: string, - runId: string, - input?: BenchmarkRunTimelineInput, - ): Promise; - getRunImports(benchmarkSlug: string, runId: string): Promise; - submitRunSummary(benchmarkSlug: string, runId: string, input: BenchmarkRunSummaryInput): Promise; - runWorker(options: RunWorkerOptions): Promise; -} diff --git a/packages/create-bench/src/__tests__/cli.test.ts b/packages/create-bench/src/__tests__/cli.test.ts index 9b4a57c7c..1f5057b91 100644 --- a/packages/create-bench/src/__tests__/cli.test.ts +++ b/packages/create-bench/src/__tests__/cli.test.ts @@ -34,7 +34,18 @@ describe('create-bench CLI', () => { fs.readFileSync(path.join(tempDir, 'package.json'), 'utf8'), ); - expect(pkg.name).toBe(path.basename(tempDir)); + expect(pkg.name).toBe(path.basename(tempDir).toLowerCase()); expect(pkg.dependencies['@benchsdk/runner']).toBe('^0.2.0'); }); + + it('sanitizes project names into valid package.json names', async () => { + const target = path.join(tempDir, 'My Project!'); + await createBench(target); + + const pkg = JSON.parse( + fs.readFileSync(path.join(target, 'package.json'), 'utf8'), + ); + + expect(pkg.name).toBe('my-project'); + }); }); diff --git a/packages/create-bench/src/index.ts b/packages/create-bench/src/index.ts index d1138fb26..9ff9c670b 100644 --- a/packages/create-bench/src/index.ts +++ b/packages/create-bench/src/index.ts @@ -4,11 +4,22 @@ import path from 'node:path'; import readline from 'node:readline/promises'; import { fileURLToPath } from 'node:url'; +function toValidPackageName(input: string): string { + const name = input + .toLowerCase() + .replace(/\s+/g, '-') + .replace(/[^a-z0-9-]/g, '') + .replace(/^[^a-z]+/, '') + .replace(/-+/g, '-') + .replace(/^-+/, ''); + return name || 'bench'; +} + function scaffold(targetDir: string, projectName: string): void { fs.mkdirSync(targetDir, { recursive: true }); const packageJson = { - name: projectName, + name: toValidPackageName(projectName), version: '0.0.0', private: true, type: 'module', diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e976a69b9..92cc74900 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -34,7 +34,7 @@ importers: version: link:packages/benchsdk-runner '@codesandbox/sdk': specifier: ^2.4.2 - version: 2.4.2(@hey-api/openapi-ts@0.99.0(magicast@0.3.5)(typescript@6.0.3))(@xterm/xterm@5.5.0)(ink@7.1.1(bufferutil@4.1.0)(react@19.2.8)(utf-8-validate@6.0.6)) + version: 2.4.2(@hey-api/openapi-ts@0.99.0(magicast@0.3.5)(typescript@5.9.3))(@xterm/xterm@5.5.0)(ink@7.1.1(bufferutil@4.1.0)(react@19.2.8)(utf-8-validate@6.0.6)) '@computesdk/anchorbrowser': specifier: ^0.2.7 version: 0.2.7(bufferutil@4.1.0)(utf-8-validate@6.0.6) @@ -46,10 +46,10 @@ importers: version: 0.1.3(@computesdk/daytona@1.7.33(ws@8.21.3(bufferutil@4.1.0)(utf-8-validate@6.0.6)))(@computesdk/e2b@1.7.53)(@computesdk/modal@1.9.5)(bufferutil@4.1.0)(utf-8-validate@6.0.6) '@computesdk/beam': specifier: ^0.3.2 - version: 0.3.2(bufferutil@4.1.0)(typescript@6.0.3)(utf-8-validate@6.0.6) + version: 0.3.2(bufferutil@4.1.0)(typescript@5.9.3)(utf-8-validate@6.0.6) '@computesdk/blaxel': specifier: ^1.6.19 - version: 1.6.19(@hey-api/openapi-ts@0.99.0(magicast@0.3.5)(typescript@6.0.3))(bufferutil@4.1.0)(utf-8-validate@6.0.6) + version: 1.6.19(@hey-api/openapi-ts@0.99.0(magicast@0.3.5)(typescript@5.9.3))(bufferutil@4.1.0)(utf-8-validate@6.0.6) '@computesdk/browserbase': specifier: ^0.3.11 version: 0.3.11 @@ -64,7 +64,7 @@ importers: version: 1.6.15(@xterm/xterm@5.5.0) '@computesdk/codesandbox': specifier: ^1.5.52 - version: 1.5.52(@hey-api/openapi-ts@0.99.0(magicast@0.3.5)(typescript@6.0.3))(@xterm/xterm@5.5.0)(ink@7.1.1(bufferutil@4.1.0)(react@19.2.8)(utf-8-validate@6.0.6)) + version: 1.5.52(@hey-api/openapi-ts@0.99.0(magicast@0.3.5)(typescript@5.9.3))(@xterm/xterm@5.5.0)(ink@7.1.1(bufferutil@4.1.0)(react@19.2.8)(utf-8-validate@6.0.6)) '@computesdk/collimate': specifier: ^0.1.2 version: 0.1.2 @@ -215,10 +215,10 @@ importers: devDependencies: '@changesets/cli': specifier: ^2.31.1 - version: 2.31.1(@types/node@26.2.0) + version: 2.31.1(@types/node@20.19.43) '@types/node': - specifier: ^26.2.0 - version: 26.2.0 + specifier: ^20.0.0 + version: 20.19.43 '@types/pg': specifier: ^8.23.1 version: 8.23.1 @@ -229,8 +229,8 @@ importers: specifier: ^4.23.12 version: 4.23.12 typescript: - specifier: ^6.0.3 - version: 6.0.3 + specifier: ^5.0.0 + version: 5.9.3 packages/benchsdk: dependencies: @@ -2419,9 +2419,6 @@ packages: '@types/node@26.1.1': resolution: {integrity: sha512-nxAkRSVkN1Y0JC1W8ky/fTfkGsMmcrRsbx+3XoZE+rMOX71kLYTV7fLXpqud1GpbpP5TuffXFqfX7fH2GgZREw==} - '@types/node@26.2.0': - resolution: {integrity: sha512-5IviulTZeRNp2vAJ514cc/HUlY5nZ9fCbq9DMyC52BrhFZACo3nI0R7qBxhQmo/d27NFe96ur/b7Wwxklda+kg==} - '@types/pg-pool@2.0.6': resolution: {integrity: sha512-TaAUE5rq2VQYxab5Ts7WZhKNmuN78Q6PiFonTDdpbx8a1H0M1vhy3rhiMjl+e2iHmogyMw7jZF4FrE6eJUy5HQ==} @@ -5295,11 +5292,6 @@ packages: engines: {node: '>=14.17'} hasBin: true - typescript@6.0.3: - resolution: {integrity: sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==} - engines: {node: '>=14.17'} - hasBin: true - ufo@1.6.4: resolution: {integrity: sha512-JFNbkD1Svwe0KvGi8GOeLcP4kAWQ609twvCdcHxq1oSL8svv39ZuSvajcD8B+5D0eL4+s1Is2D/O6KN3qcTeRA==} @@ -5973,13 +5965,13 @@ snapshots: '@bcoe/v8-coverage@0.2.3': {} - '@beamcloud/beam-js@1.0.17(bufferutil@4.1.0)(typescript@6.0.3)(utf-8-validate@6.0.6)': + '@beamcloud/beam-js@1.0.17(bufferutil@4.1.0)(typescript@5.9.3)(utf-8-validate@6.0.6)': dependencies: axios: 1.19.0 commander: 12.1.0 ignore: 7.0.6 strip-ansi: 7.2.0 - typescript: 6.0.3 + typescript: 5.9.3 ws: 8.21.3(bufferutil@4.1.0)(utf-8-validate@6.0.6) yazl: 3.3.1 transitivePeerDependencies: @@ -5988,9 +5980,9 @@ snapshots: - supports-color - utf-8-validate - '@blaxel/core@0.3.12(@hey-api/openapi-ts@0.99.0(magicast@0.3.5)(typescript@6.0.3))(bufferutil@4.1.0)(utf-8-validate@6.0.6)': + '@blaxel/core@0.3.12(@hey-api/openapi-ts@0.99.0(magicast@0.3.5)(typescript@5.9.3))(bufferutil@4.1.0)(utf-8-validate@6.0.6)': dependencies: - '@hey-api/client-fetch': 0.10.2(@hey-api/openapi-ts@0.99.0(magicast@0.3.5)(typescript@6.0.3)) + '@hey-api/client-fetch': 0.10.2(@hey-api/openapi-ts@0.99.0(magicast@0.3.5)(typescript@5.9.3)) '@modelcontextprotocol/sdk': 1.30.0(zod@4.4.3) archiver: 7.0.1 axios: 1.19.0 @@ -6079,7 +6071,7 @@ snapshots: dependencies: '@changesets/types': 6.1.0 - '@changesets/cli@2.31.1(@types/node@26.2.0)': + '@changesets/cli@2.31.1(@types/node@20.19.43)': dependencies: '@changesets/apply-release-plan': 7.1.1 '@changesets/assemble-release-plan': 6.0.10 @@ -6095,7 +6087,7 @@ snapshots: '@changesets/should-skip-package': 0.1.2 '@changesets/types': 6.1.0 '@changesets/write': 0.4.0 - '@inquirer/external-editor': 1.0.3(@types/node@26.2.0) + '@inquirer/external-editor': 1.0.3(@types/node@20.19.43) '@manypkg/get-packages': 1.1.3 ansi-colors: 4.1.3 enquirer: 2.4.1 @@ -6202,9 +6194,9 @@ snapshots: optionalDependencies: '@xterm/xterm': 5.5.0 - '@codesandbox/sdk@2.4.2(@hey-api/openapi-ts@0.99.0(magicast@0.3.5)(typescript@6.0.3))(@xterm/xterm@5.5.0)(ink@7.1.1(bufferutil@4.1.0)(react@19.2.8)(utf-8-validate@6.0.6))': + '@codesandbox/sdk@2.4.2(@hey-api/openapi-ts@0.99.0(magicast@0.3.5)(typescript@5.9.3))(@xterm/xterm@5.5.0)(ink@7.1.1(bufferutil@4.1.0)(react@19.2.8)(utf-8-validate@6.0.6))': dependencies: - '@hey-api/client-fetch': 0.13.1(@hey-api/openapi-ts@0.99.0(magicast@0.3.5)(typescript@6.0.3)) + '@hey-api/client-fetch': 0.13.1(@hey-api/openapi-ts@0.99.0(magicast@0.3.5)(typescript@5.9.3)) '@inkjs/ui': 2.0.0(ink@7.1.1(bufferutil@4.1.0)(react@19.2.8)(utf-8-validate@6.0.6)) '@opentelemetry/api': 1.9.1 '@xterm/addon-serialize': 0.13.0(@xterm/xterm@5.5.0) @@ -6262,9 +6254,9 @@ snapshots: - bufferutil - utf-8-validate - '@computesdk/beam@0.3.2(bufferutil@4.1.0)(typescript@6.0.3)(utf-8-validate@6.0.6)': + '@computesdk/beam@0.3.2(bufferutil@4.1.0)(typescript@5.9.3)(utf-8-validate@6.0.6)': dependencies: - '@beamcloud/beam-js': 1.0.17(bufferutil@4.1.0)(typescript@6.0.3)(utf-8-validate@6.0.6) + '@beamcloud/beam-js': 1.0.17(bufferutil@4.1.0)(typescript@5.9.3)(utf-8-validate@6.0.6) '@computesdk/provider': 2.1.5 computesdk: 4.1.4 transitivePeerDependencies: @@ -6274,9 +6266,9 @@ snapshots: - typescript - utf-8-validate - '@computesdk/blaxel@1.6.19(@hey-api/openapi-ts@0.99.0(magicast@0.3.5)(typescript@6.0.3))(bufferutil@4.1.0)(utf-8-validate@6.0.6)': + '@computesdk/blaxel@1.6.19(@hey-api/openapi-ts@0.99.0(magicast@0.3.5)(typescript@5.9.3))(bufferutil@4.1.0)(utf-8-validate@6.0.6)': dependencies: - '@blaxel/core': 0.3.12(@hey-api/openapi-ts@0.99.0(magicast@0.3.5)(typescript@6.0.3))(bufferutil@4.1.0)(utf-8-validate@6.0.6) + '@blaxel/core': 0.3.12(@hey-api/openapi-ts@0.99.0(magicast@0.3.5)(typescript@5.9.3))(bufferutil@4.1.0)(utf-8-validate@6.0.6) '@computesdk/provider': 2.1.5 computesdk: 4.1.4 transitivePeerDependencies: @@ -6329,9 +6321,9 @@ snapshots: '@computesdk/cmd@0.4.1': {} - '@computesdk/codesandbox@1.5.52(@hey-api/openapi-ts@0.99.0(magicast@0.3.5)(typescript@6.0.3))(@xterm/xterm@5.5.0)(ink@7.1.1(bufferutil@4.1.0)(react@19.2.8)(utf-8-validate@6.0.6))': + '@computesdk/codesandbox@1.5.52(@hey-api/openapi-ts@0.99.0(magicast@0.3.5)(typescript@5.9.3))(@xterm/xterm@5.5.0)(ink@7.1.1(bufferutil@4.1.0)(react@19.2.8)(utf-8-validate@6.0.6))': dependencies: - '@codesandbox/sdk': 2.4.2(@hey-api/openapi-ts@0.99.0(magicast@0.3.5)(typescript@6.0.3))(@xterm/xterm@5.5.0)(ink@7.1.1(bufferutil@4.1.0)(react@19.2.8)(utf-8-validate@6.0.6)) + '@codesandbox/sdk': 2.4.2(@hey-api/openapi-ts@0.99.0(magicast@0.3.5)(typescript@5.9.3))(@xterm/xterm@5.5.0)(ink@7.1.1(bufferutil@4.1.0)(react@19.2.8)(utf-8-validate@6.0.6)) '@computesdk/provider': 2.1.5 computesdk: 4.1.4 transitivePeerDependencies: @@ -6967,13 +6959,13 @@ snapshots: protobufjs: 7.6.5 yargs: 17.7.3 - '@hey-api/client-fetch@0.10.2(@hey-api/openapi-ts@0.99.0(magicast@0.3.5)(typescript@6.0.3))': + '@hey-api/client-fetch@0.10.2(@hey-api/openapi-ts@0.99.0(magicast@0.3.5)(typescript@5.9.3))': dependencies: - '@hey-api/openapi-ts': 0.99.0(magicast@0.3.5)(typescript@6.0.3) + '@hey-api/openapi-ts': 0.99.0(magicast@0.3.5)(typescript@5.9.3) - '@hey-api/client-fetch@0.13.1(@hey-api/openapi-ts@0.99.0(magicast@0.3.5)(typescript@6.0.3))': + '@hey-api/client-fetch@0.13.1(@hey-api/openapi-ts@0.99.0(magicast@0.3.5)(typescript@5.9.3))': dependencies: - '@hey-api/openapi-ts': 0.99.0(magicast@0.3.5)(typescript@6.0.3) + '@hey-api/openapi-ts': 0.99.0(magicast@0.3.5)(typescript@5.9.3) '@hey-api/codegen-core@0.9.1(magicast@0.3.5)': dependencies: @@ -6990,7 +6982,7 @@ snapshots: '@types/json-schema': 7.0.15 js-yaml: 4.2.0 - '@hey-api/openapi-ts@0.99.0(magicast@0.3.5)(typescript@6.0.3)': + '@hey-api/openapi-ts@0.99.0(magicast@0.3.5)(typescript@5.9.3)': dependencies: '@hey-api/codegen-core': 0.9.1(magicast@0.3.5) '@hey-api/json-schema-ref-parser': 1.4.4 @@ -7002,7 +6994,7 @@ snapshots: color-support: 1.1.3 commander: 15.0.0 get-tsconfig: 4.14.0 - typescript: 6.0.3 + typescript: 5.9.3 transitivePeerDependencies: - magicast @@ -7076,12 +7068,12 @@ snapshots: figures: 6.1.0 ink: 7.1.1(bufferutil@4.1.0)(react@19.2.8)(utf-8-validate@6.0.6) - '@inquirer/external-editor@1.0.3(@types/node@26.2.0)': + '@inquirer/external-editor@1.0.3(@types/node@20.19.43)': dependencies: chardet: 2.2.0 iconv-lite: 0.7.3 optionalDependencies: - '@types/node': 26.2.0 + '@types/node': 20.19.43 '@isaacs/cliui@8.0.2': dependencies: @@ -8141,7 +8133,7 @@ snapshots: '@types/connect@3.4.38': dependencies: - '@types/node': 26.2.0 + '@types/node': 20.19.43 optional: true '@types/estree@1.0.9': {} @@ -8150,12 +8142,12 @@ snapshots: '@types/mysql@2.15.26': dependencies: - '@types/node': 26.2.0 + '@types/node': 20.19.43 optional: true '@types/node-fetch@2.6.13': dependencies: - '@types/node': 26.2.0 + '@types/node': 20.19.43 form-data: 4.0.6 '@types/node@12.20.55': {} @@ -8177,10 +8169,6 @@ snapshots: undici-types: 8.3.0 optional: true - '@types/node@26.2.0': - dependencies: - undici-types: 8.3.0 - '@types/pg-pool@2.0.6': dependencies: '@types/pg': 8.23.1 @@ -8188,13 +8176,13 @@ snapshots: '@types/pg@8.23.1': dependencies: - '@types/node': 26.2.0 + '@types/node': 20.19.43 pg-protocol: 1.16.0 pg-types: 2.2.0 '@types/pg@8.6.1': dependencies: - '@types/node': 26.2.0 + '@types/node': 20.19.43 pg-protocol: 1.16.0 pg-types: 2.2.0 optional: true @@ -8202,7 +8190,7 @@ snapshots: '@types/request@2.48.13': dependencies: '@types/caseless': 0.12.5 - '@types/node': 26.2.0 + '@types/node': 20.19.43 '@types/tough-cookie': 4.0.5 form-data: 2.5.6 @@ -8211,14 +8199,14 @@ snapshots: '@types/tedious@4.0.14': dependencies: - '@types/node': 26.2.0 + '@types/node': 20.19.43 optional: true '@types/tough-cookie@4.0.5': {} '@types/ws@8.18.1': dependencies: - '@types/node': 26.2.0 + '@types/node': 20.19.43 '@typespec/ts-http-runtime@0.3.8': dependencies: @@ -10579,7 +10567,7 @@ snapshots: '@protobufjs/path': 1.1.2 '@protobufjs/pool': 1.1.0 '@protobufjs/utf8': 1.1.2 - '@types/node': 26.2.0 + '@types/node': 20.19.43 long: 5.3.2 protobufjs@8.0.1: @@ -10594,7 +10582,7 @@ snapshots: '@protobufjs/path': 1.1.2 '@protobufjs/pool': 1.1.0 '@protobufjs/utf8': 1.1.2 - '@types/node': 26.2.0 + '@types/node': 20.19.43 long: 5.3.2 proxy-addr@2.0.7: @@ -11346,8 +11334,6 @@ snapshots: typescript@5.9.3: {} - typescript@6.0.3: {} - ufo@1.6.4: {} uint8array-extras@1.5.0: {} @@ -11356,7 +11342,8 @@ snapshots: undici-types@6.21.0: {} - undici-types@8.3.0: {} + undici-types@8.3.0: + optional: true undici@6.28.0: {} From 2124f48047611084e11af9ec0fd1ffce19563528 Mon Sep 17 00:00:00 2001 From: david Date: Mon, 24 Aug 2026 16:23:32 +0000 Subject: [PATCH 2/4] runner: support customCliFlags in benchmark configs and keep unknown-flag rejection Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../ai-gateway/ai-gateway-gemini.bench.ts | 1 + .../ai-gateway/ai-gateway-kimi.bench.ts | 1 + .../ai-gateway/ai-gateway-openai.bench.ts | 1 + benchmarks/ai-gateway/ai-gateway.bench.ts | 1 + benchmarks/storage/snapshot-fork.bench.ts | 1 + benchmarks/storage/storage.bench.ts | 1 + .../src/__tests__/runner.test.ts | 13 ++++++++ packages/benchsdk-runner/src/bench-config.ts | 11 +++++++ packages/benchsdk-runner/src/runner.ts | 33 ++++++++++++++++--- 9 files changed, 59 insertions(+), 4 deletions(-) diff --git a/benchmarks/ai-gateway/ai-gateway-gemini.bench.ts b/benchmarks/ai-gateway/ai-gateway-gemini.bench.ts index 5f2e7014f..847d10822 100644 --- a/benchmarks/ai-gateway/ai-gateway-gemini.bench.ts +++ b/benchmarks/ai-gateway/ai-gateway-gemini.bench.ts @@ -35,6 +35,7 @@ export const config = defineBenchmarkConfig({ phases, groupBy: 'round', participants: providers, + customCliFlags: ['--ai-gateway-iterations-cold', '--ai-gateway-iterations-warm'], scoring: { metrics: [ { key: 'coldE2eMs', unit: 'ms', ceiling: 20000, weights: { median: 0.30, p95: 0.15, p99: 0 } }, diff --git a/benchmarks/ai-gateway/ai-gateway-kimi.bench.ts b/benchmarks/ai-gateway/ai-gateway-kimi.bench.ts index 88eeace54..164af4223 100644 --- a/benchmarks/ai-gateway/ai-gateway-kimi.bench.ts +++ b/benchmarks/ai-gateway/ai-gateway-kimi.bench.ts @@ -52,6 +52,7 @@ export const config = defineBenchmarkConfig({ phases, groupBy: 'round', participants: providers, + customCliFlags: ['--ai-gateway-iterations-cold', '--ai-gateway-iterations-warm'], scoring: { metrics: [ { key: 'coldE2eMs', unit: 'ms', ceiling: 20000, weights: { median: 0.30, p95: 0.15, p99: 0 } }, diff --git a/benchmarks/ai-gateway/ai-gateway-openai.bench.ts b/benchmarks/ai-gateway/ai-gateway-openai.bench.ts index 6c4165e7f..0ede5441d 100644 --- a/benchmarks/ai-gateway/ai-gateway-openai.bench.ts +++ b/benchmarks/ai-gateway/ai-gateway-openai.bench.ts @@ -35,6 +35,7 @@ export const config = defineBenchmarkConfig({ phases, groupBy: 'round', participants: providers, + customCliFlags: ['--ai-gateway-iterations-cold', '--ai-gateway-iterations-warm'], scoring: { metrics: [ { key: 'coldE2eMs', unit: 'ms', ceiling: 20000, weights: { median: 0.30, p95: 0.15, p99: 0 } }, diff --git a/benchmarks/ai-gateway/ai-gateway.bench.ts b/benchmarks/ai-gateway/ai-gateway.bench.ts index 0e0fc306d..24e36aa18 100644 --- a/benchmarks/ai-gateway/ai-gateway.bench.ts +++ b/benchmarks/ai-gateway/ai-gateway.bench.ts @@ -52,6 +52,7 @@ export const config = defineBenchmarkConfig({ phases, groupBy: 'round', participants: providers, + customCliFlags: ['--ai-gateway-iterations-cold', '--ai-gateway-iterations-warm'], scoring: { metrics: [ { key: 'coldE2eMs', unit: 'ms', ceiling: 20000, weights: { median: 0.30, p95: 0.15, p99: 0 } }, diff --git a/benchmarks/storage/snapshot-fork.bench.ts b/benchmarks/storage/snapshot-fork.bench.ts index a571a898c..ff0a32773 100644 --- a/benchmarks/storage/snapshot-fork.bench.ts +++ b/benchmarks/storage/snapshot-fork.bench.ts @@ -53,6 +53,7 @@ export const config = defineBenchmarkConfig({ iterations: 2, concurrency: 1, participants, + customCliFlags: ['--dataset'], dimensions: { dataset }, scoring: { // A fork whose read-back didn't match is not a usable fork, so its timings diff --git a/benchmarks/storage/storage.bench.ts b/benchmarks/storage/storage.bench.ts index c7c96c9db..76883d261 100644 --- a/benchmarks/storage/storage.bench.ts +++ b/benchmarks/storage/storage.bench.ts @@ -53,6 +53,7 @@ const baseConfig = { benchmarkName: `Storage Lifecycle${process.env.DAILY_BENCH_NAME ? ` - ${process.env.DAILY_BENCH_NAME}` : ''}`, concurrency: 1, participants: storageProviders, + customCliFlags: ['--file-size'], scoring: { groupBy: 'file_size', metrics: [ diff --git a/packages/benchsdk-runner/src/__tests__/runner.test.ts b/packages/benchsdk-runner/src/__tests__/runner.test.ts index ab59656b7..36316fd2f 100644 --- a/packages/benchsdk-runner/src/__tests__/runner.test.ts +++ b/packages/benchsdk-runner/src/__tests__/runner.test.ts @@ -82,6 +82,19 @@ describe('parseCliArgs', () => { expect(() => parseCliArgs(['--unknown', 'x', '--iterations', '3'])).toThrow('Unknown flag'); }); + it('allows declared custom pass-through flags and their values', () => { + expect(parseCliArgs(['--file-size', '10MB', '--iterations', '3'], ['--file-size'])).toEqual({ + iterations: 3, + }); + expect(parseCliArgs(['--file-size=10MB', '--iterations', '3'], ['--file-size'])).toEqual({ + iterations: 3, + }); + }); + + it('still throws when a pass-through-style flag is not declared', () => { + expect(() => parseCliArgs(['--file-size', '10MB'])).toThrow('Unknown flag'); + }); + it('throws on non-numeric numeric flags', () => { expect(() => parseCliArgs(['--iterations', 'abc'])).toThrow('--iterations'); }); diff --git a/packages/benchsdk-runner/src/bench-config.ts b/packages/benchsdk-runner/src/bench-config.ts index 3bf0ca4f5..5550ffc0c 100644 --- a/packages/benchsdk-runner/src/bench-config.ts +++ b/packages/benchsdk-runner/src/bench-config.ts @@ -256,6 +256,12 @@ export interface BenchmarkConfig { * The platform can recompute `compositeScore` from the same spec at read time. */ scoring?: BenchmarkScoringConfig; + /** + * Custom CLI flags this benchmark reads from `process.argv` (e.g. `--file-size`). + * Declaring them lets the runner distinguish intentional pass-through flags + * from typos and report unknown flags accurately. + */ + customCliFlags?: readonly string[]; } function assertPositiveInt(value: number | undefined, field: string): void { @@ -392,6 +398,11 @@ export function defineBenchmarkConfig typeof f === 'string' && f.startsWith('--'))) { + throw new Error('customCliFlags must be an array of strings starting with "--"'); + } + } return config; } diff --git a/packages/benchsdk-runner/src/runner.ts b/packages/benchsdk-runner/src/runner.ts index 3856f3f6b..d14a7bc86 100644 --- a/packages/benchsdk-runner/src/runner.ts +++ b/packages/benchsdk-runner/src/runner.ts @@ -165,10 +165,15 @@ async function runStepWithClient( * Parses the orchestration flags this runner understands, rejecting unknown * flags. Supports both `--flag value` and `--flag=value`; `--provider` accepts * a comma-separated list and may be repeated. + * + * `allowedCustomFlags` lists pass-through flags the benchmark file reads from + * `process.argv` itself; the runner validates and skips them without choking on + * their values. */ -export function parseCliArgs(argv: string[]): CliArgs { +export function parseCliArgs(argv: string[], allowedCustomFlags?: readonly string[]): CliArgs { const args: CliArgs = {}; const unknown: string[] = []; + const allowed = new Set(allowedCustomFlags ?? []); const readValue = (raw: string, i: number): { value: string; nextIndex: number } => { const eq = raw.indexOf('='); @@ -264,9 +269,29 @@ export function parseCliArgs(argv: string[]): CliArgs { case '--dry-run': args.noIngest = true; break; - default: - unknown.push(name); + default: { + if (allowed.has(name)) { + // Skip the flag's value (if supplied as a separate token) so the + // benchmark file can read it from process.argv itself. + if (!arg.includes('=')) { + const next = argv[i + 1]; + if (next && !next.startsWith('-')) { + i++; + } + } + } else { + unknown.push(name); + // Unknown flags that take a value shouldn't report the value as a + // separate unknown flag. + if (!arg.includes('=')) { + const next = argv[i + 1]; + if (next && !next.startsWith('-')) { + i++; + } + } + } break; + } } } @@ -484,7 +509,7 @@ export async function runBenchmark( task: BenchmarkTask, argv: string[] = [], ): Promise { - const args = parseCliArgs(argv); + const args = parseCliArgs(argv, fileConfig.customCliFlags); const noIngest = args.noIngest ?? isEnvNoIngest(); const shaped = applyShape(fileConfig, resolveShape(fileConfig, args.shape)); const config = applyIdentityOverrides(shaped, args); From 1c1b2883c38ba3043642aa6103d8dcc0073fecb2 Mon Sep 17 00:00:00 2001 From: david Date: Mon, 24 Aug 2026 18:30:09 +0000 Subject: [PATCH 3/4] chore: remove COMPUTESDK_ADMIN_API_KEY from SDK, scripts, and workflows Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/ai-gateway-benchmarks.yml | 2 +- .github/workflows/browser-benchmarks.yml | 2 +- .../workflows/browser-concurrent-benchmarks.yml | 2 +- .../workflows/browser-throughput-benchmarks.yml | 2 +- .../workflows/reliability-tensorlake-opencode.yml | 2 +- .github/workflows/sandbox-capabilities.yml | 2 +- .github/workflows/sandbox-dax-benchmarks.yml | 2 +- .github/workflows/sandbox-tti-benchmarks.yml | 2 +- .github/workflows/snapshot-fork-benchmarks.yml | 2 +- .github/workflows/storage-benchmarks.yml | 2 +- benchmarks/scale/coordinator.ts | 2 +- benchmarks/scale/scripts/aggregate.ts | 4 ++-- benchmarks/scale/scripts/start.ts | 15 ++++++++------- benchmarks/scale/scripts/watch.ts | 2 +- packages/benchsdk-api/src/client.ts | 2 +- packages/benchsdk-api/src/types.ts | 2 +- packages/benchsdk-runner/src/runner.ts | 4 +--- packages/benchsdk/README.md | 4 ++-- .../benchsdk/src/__tests__/integration.test.ts | 4 ++-- .../src/__tests__/public-api.contract.test.ts | 14 +++++++------- 20 files changed, 36 insertions(+), 37 deletions(-) diff --git a/.github/workflows/ai-gateway-benchmarks.yml b/.github/workflows/ai-gateway-benchmarks.yml index 03adabea7..7f7cef2ea 100644 --- a/.github/workflows/ai-gateway-benchmarks.yml +++ b/.github/workflows/ai-gateway-benchmarks.yml @@ -141,7 +141,7 @@ jobs: INPUT_ITERATIONS: ${{ github.event.inputs.iterations }} FAMILY_ITERATIONS: ${{ matrix.family.iterations }} run: | - . benchmarks/scripts/load-vault-secrets.sh '^(ANTHROPIC_API_KEY|OPENAI_API_KEY|GEMINI_API_KEY|KIMI_API_KEY|NOVITA_API_KEY|NEON_AI_GATEWAY_BASE_URL|NEON_AI_GATEWAY_TOKEN|NGROK_AI_GATEWAY_API_KEY|LLMAPI_API_KEY|CLOUDFLARE_AI_GATEWAY_ACCOUNT_ID|CLOUDFLARE_AI_GATEWAY_GATEWAY_ID|CLOUDFLARE_AI_GATEWAY_TOKEN|LLM_GATEWAY_API_KEY|OPENROUTER_API_KEY|PYDANTIC_AI_GATEWAY_API_KEY|VERCEL_AI_GATEWAY_API_KEY|CONCENTRATE_AI_GATEWAY_API_KEY|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)' + . benchmarks/scripts/load-vault-secrets.sh '^(ANTHROPIC_API_KEY|OPENAI_API_KEY|GEMINI_API_KEY|KIMI_API_KEY|NOVITA_API_KEY|NEON_AI_GATEWAY_BASE_URL|NEON_AI_GATEWAY_TOKEN|NGROK_AI_GATEWAY_API_KEY|LLMAPI_API_KEY|CLOUDFLARE_AI_GATEWAY_ACCOUNT_ID|CLOUDFLARE_AI_GATEWAY_GATEWAY_ID|CLOUDFLARE_AI_GATEWAY_TOKEN|LLM_GATEWAY_API_KEY|OPENROUTER_API_KEY|PYDANTIC_AI_GATEWAY_API_KEY|VERCEL_AI_GATEWAY_API_KEY|CONCENTRATE_AI_GATEWAY_API_KEY|BENCHMARKS_PLATFORM_API_KEY)' if [ "${{ github.event_name }}" = "push" ] || [ "${{ github.event_name }}" = "schedule" ]; then ITERATIONS="10" diff --git a/.github/workflows/browser-benchmarks.yml b/.github/workflows/browser-benchmarks.yml index fd669a455..98116dbee 100644 --- a/.github/workflows/browser-benchmarks.yml +++ b/.github/workflows/browser-benchmarks.yml @@ -68,7 +68,7 @@ jobs: run: rm -rf results/browser/ - name: Run browser benchmark run: | - . benchmarks/scripts/load-vault-secrets.sh '^(BROWSERBASE_API_KEY|BROWSERBASE_PROJECT_ID|BROWSER_USE_API_KEY|HYPERBROWSER_API_KEY|KERNEL_API_KEY|NOTTE_API_KEY|STEEL_API_KEY|TILION_API_KEY|TILION_BASE_URL|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)' + . benchmarks/scripts/load-vault-secrets.sh '^(BROWSERBASE_API_KEY|BROWSERBASE_PROJECT_ID|BROWSER_USE_API_KEY|HYPERBROWSER_API_KEY|KERNEL_API_KEY|NOTTE_API_KEY|STEEL_API_KEY|TILION_API_KEY|TILION_BASE_URL|BENCHMARKS_PLATFORM_API_KEY)' # All matrix jobs for this workflow run share a single platform run. The # attempt number is included so re-runs do not rejoin completed runs. diff --git a/.github/workflows/browser-concurrent-benchmarks.yml b/.github/workflows/browser-concurrent-benchmarks.yml index 2c8754347..4d6ef1526 100644 --- a/.github/workflows/browser-concurrent-benchmarks.yml +++ b/.github/workflows/browser-concurrent-benchmarks.yml @@ -145,7 +145,7 @@ jobs: env: THROUGHPUT_URLS: ${{ needs.setup.outputs.urls }} run: | - . benchmarks/scripts/load-vault-secrets.sh '^(BROWSERBASE_API_KEY|BROWSERBASE_PROJECT_ID|BROWSER_USE_API_KEY|HYPERBROWSER_API_KEY|KERNEL_API_KEY|NOTTE_API_KEY|STEEL_API_KEY|TILION_API_KEY|TILION_BASE_URL|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)' + . benchmarks/scripts/load-vault-secrets.sh '^(BROWSERBASE_API_KEY|BROWSERBASE_PROJECT_ID|BROWSER_USE_API_KEY|HYPERBROWSER_API_KEY|KERNEL_API_KEY|NOTTE_API_KEY|STEEL_API_KEY|TILION_API_KEY|TILION_BASE_URL|BENCHMARKS_PLATFORM_API_KEY)' # One invocation covers the whole sweep: the benchmark runs one task # per concurrency level, and --levels picks which levels run (the diff --git a/.github/workflows/browser-throughput-benchmarks.yml b/.github/workflows/browser-throughput-benchmarks.yml index 36f871afa..b03323e3c 100644 --- a/.github/workflows/browser-throughput-benchmarks.yml +++ b/.github/workflows/browser-throughput-benchmarks.yml @@ -122,7 +122,7 @@ jobs: env: THROUGHPUT_URLS: ${{ needs.setup.outputs.urls }} run: | - . benchmarks/scripts/load-vault-secrets.sh '^(BROWSERBASE_API_KEY|BROWSERBASE_PROJECT_ID|BROWSER_USE_API_KEY|HYPERBROWSER_API_KEY|KERNEL_API_KEY|NOTTE_API_KEY|STEEL_API_KEY|TILION_API_KEY|TILION_BASE_URL|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)' + . benchmarks/scripts/load-vault-secrets.sh '^(BROWSERBASE_API_KEY|BROWSERBASE_PROJECT_ID|BROWSER_USE_API_KEY|HYPERBROWSER_API_KEY|KERNEL_API_KEY|NOTTE_API_KEY|STEEL_API_KEY|TILION_API_KEY|TILION_BASE_URL|BENCHMARKS_PLATFORM_API_KEY)' # All matrix jobs for this workflow run share a single platform run. The # attempt number is included so re-runs do not rejoin completed runs. diff --git a/.github/workflows/reliability-tensorlake-opencode.yml b/.github/workflows/reliability-tensorlake-opencode.yml index 6dde26eea..d24e73155 100644 --- a/.github/workflows/reliability-tensorlake-opencode.yml +++ b/.github/workflows/reliability-tensorlake-opencode.yml @@ -48,7 +48,7 @@ jobs: STAGGER_DELAY_MS: ${{ github.event.inputs.stagger_delay_ms || '60000' }} DRY_RUN: ${{ github.event.inputs.dry_run || false }} run: | - . benchmarks/scripts/load-vault-secrets.sh '^(TENSORLAKE_API_KEY|BENCHMARKS_PLATFORM_API_KEY|COMPUTESDK_ADMIN_API_KEY)=' + . benchmarks/scripts/load-vault-secrets.sh '^(TENSORLAKE_API_KEY|BENCHMARKS_PLATFORM_API_KEY)=' RUN_KEY="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" ITERATIONS="${ITERATIONS:-60}" STAGGER_DELAY_MS="${STAGGER_DELAY_MS:-60000}" diff --git a/.github/workflows/sandbox-capabilities.yml b/.github/workflows/sandbox-capabilities.yml index 2696eaa66..ce834ddc6 100644 --- a/.github/workflows/sandbox-capabilities.yml +++ b/.github/workflows/sandbox-capabilities.yml @@ -100,7 +100,7 @@ jobs: CONCURRENCY: ${{ github.event.inputs.concurrency || '1' }} PROVIDER: ${{ matrix.provider }} run: | - COMMON_KEYS='BENCHMARKS_PLATFORM_API_KEY|COMPUTESDK_ADMIN_API_KEY' + COMMON_KEYS='BENCHMARKS_PLATFORM_API_KEY' case "$PROVIDER" in archil) PROVIDER_KEYS='ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID' ;; arker) PROVIDER_KEYS='ARKER_API_KEY' ;; diff --git a/.github/workflows/sandbox-dax-benchmarks.yml b/.github/workflows/sandbox-dax-benchmarks.yml index c5dd4b153..0bf9cdaa4 100644 --- a/.github/workflows/sandbox-dax-benchmarks.yml +++ b/.github/workflows/sandbox-dax-benchmarks.yml @@ -116,7 +116,7 @@ jobs: LIGHTNING_INSTANCE_TYPE: cpu-8 # 8 vCPU / 16 GiB, matching the standardized dax profile ARKER_PLATFORMS: graviton4 # dax only; elsewhere Arker picks the platform automatically run: | - . benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|ARKER_API_KEY|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_API_KEY|BL_WORKSPACE|BL_DAX_API_KEY|BL_DAX_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MIOSA_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|MSB_API_KEY|MOSAIC_API_URL|MOSAIC_API_TOKEN|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUN_CLOUD_API_KEY|RUNLOOP_API_KEY|SAIL_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)' + . benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|ARKER_API_KEY|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_API_KEY|BL_WORKSPACE|BL_DAX_API_KEY|BL_DAX_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MIOSA_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|MSB_API_KEY|MOSAIC_API_URL|MOSAIC_API_TOKEN|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUN_CLOUD_API_KEY|RUNLOOP_API_KEY|SAIL_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|BENCHMARKS_PLATFORM_API_KEY)' # If dedicated DAX Blaxel credentials are provisioned, route them to the # standard names that providers.ts expects, so no provider fork is needed. diff --git a/.github/workflows/sandbox-tti-benchmarks.yml b/.github/workflows/sandbox-tti-benchmarks.yml index a424c620e..3845be85e 100644 --- a/.github/workflows/sandbox-tti-benchmarks.yml +++ b/.github/workflows/sandbox-tti-benchmarks.yml @@ -114,7 +114,7 @@ jobs: - name: Run benchmark if: env.SHOULD_RUN == 'true' run: | - . benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|ARKER_API_KEY|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_API_KEY|BL_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MIOSA_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|MSB_API_KEY|MOSAIC_API_URL|MOSAIC_API_TOKEN|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUN_CLOUD_API_KEY|RUNLOOP_API_KEY|SAIL_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)' + . benchmarks/scripts/load-vault-secrets.sh '^(ARCHIL_API_KEY|ARCHIL_REGION|ARCHIL_DISK_ID|ARKER_API_KEY|BEAM_TOKEN|BEAM_WORKSPACE_ID|BL_API_KEY|BL_WORKSPACE|CLOUD_RUN_SANDBOX_URL|CLOUD_RUN_SANDBOX_SECRET|CLOUDFLARE_SANDBOX_URL|CLOUDFLARE_SANDBOX_SECRET|CSB_API_KEY|CREATEOS_SANDBOX_API_KEY|DAYTONA_API_KEY|DECLAW_API_KEY|E2B_API_KEY|HOPX_API_KEY|ISORUN_API_KEY|LIGHTNING_API_KEY|MIOSA_API_KEY|MODAL_TOKEN_ID|MODAL_TOKEN_SECRET|MSB_API_KEY|MOSAIC_API_URL|MOSAIC_API_TOKEN|NSC_TOKEN|NORTHFLANK_TOKEN|NORTHFLANK_PROJECT_ID|OPENCOMPUTER_API_KEY|OPENCOMPUTER_API_URL|RUN_CLOUD_API_KEY|RUNLOOP_API_KEY|SAIL_API_KEY|SANDBOX0_TOKEN|SPRITES_TOKEN|SUPERSERVE_API_KEY|TENKI_API_KEY|TENSORLAKE_API_KEY|UPSTASH_BOX_API_KEY|VERCEL_TOKEN|VERCEL_TEAM_ID|VERCEL_PROJECT_ID|BENCHMARKS_PLATFORM_API_KEY)' # Burst shape only; every provider passes the same --run-key, so they # get-or-create one shared run and rank against each other (each diff --git a/.github/workflows/snapshot-fork-benchmarks.yml b/.github/workflows/snapshot-fork-benchmarks.yml index 10ec347dd..daf67ae83 100644 --- a/.github/workflows/snapshot-fork-benchmarks.yml +++ b/.github/workflows/snapshot-fork-benchmarks.yml @@ -87,7 +87,7 @@ jobs: run: rm -rf results/snapshot-fork/ - name: Run snapshot/fork test run: | - . benchmarks/scripts/load-vault-secrets.sh '^(AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|AWS_REGION|S3_BUCKET|S3_SNAPSHOT_ACCESS_KEY_ID|S3_SNAPSHOT_SECRET_ACCESS_KEY|S3_SNAPSHOT_BUCKET|R2_ACCESS_KEY_ID|R2_SECRET_ACCESS_KEY|R2_BUCKET|R2_ACCOUNT_ID|R2_SNAPSHOT_ACCESS_KEY_ID|R2_SNAPSHOT_SECRET_ACCESS_KEY|R2_SNAPSHOT_BUCKET|TIGRIS_STORAGE_ACCESS_KEY_ID|TIGRIS_STORAGE_SECRET_ACCESS_KEY|TIGRIS_STORAGE_BUCKET|TIGRIS_SNAPSHOT_ACCESS_KEY|TIGRIS_SNAPSHOT_SECRET_KEY|TIGRIS_SNAPSHOT_STORAGE_BUCKET|BLOB_READ_WRITE_TOKEN|VERCEL_BLOB_BUCKET|GCS_PROJECT_ID|GCS_BUCKET|GCS_CLIENT_EMAIL|GCS_PRIVATE_KEY|AZURE_ACCOUNT_NAME|AZURE_ACCOUNT_KEY|AZURE_CONTAINER|AZURE_SNAPSHOT_ACCOUNT_NAME|AZURE_SNAPSHOT_ACCOUNT_KEY|AZURE_SNAPSHOT_CONTAINER|TENSORLAKE_API_KEY|TENSORLAKE_API_URL|TENSORLAKE_ORGANIZATION_ID|TENSORLAKE_PROJECT_ID|TENSORLAKE_FILESYSTEM|ARCHIL_S3_ACCESS_KEY_ID|ARCHIL_S3_SECRET_ACCESS_KEY|ARCHIL_BUCKET|ARCHIL_REGION|ARCHIL_BRANCH|NEON_BUCKET|NEON_ENDPOINT|NEON_ACCESS_KEY_ID|NEON_SECRET_ACCESS_KEY|NEON_REGION|NEON_SNAPSHOT_BUCKET|NEON_SNAPSHOT_ENDPOINT|NEON_SNAPSHOT_ACCESS_KEY_ID|NEON_SNAPSHOT_SECRET_ACCESS_KEY|NEON_SNAPSHOT_REGION|MOSAIC_BUCKET|MOSAIC_ACCESS_KEY_ID|MOSAIC_SECRET_ACCESS_KEY|MOSAIC_REGION|MOSAIC_ENDPOINT|MOSAIC_SNAPSHOT_BUCKET|MOSAIC_SNAPSHOT_ACCESS_KEY_ID|MOSAIC_SNAPSHOT_SECRET_ACCESS_KEY|MOSAIC_SNAPSHOT_REGION|MOSAIC_SNAPSHOT_ENDPOINT|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)' + . benchmarks/scripts/load-vault-secrets.sh '^(AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|AWS_REGION|S3_BUCKET|S3_SNAPSHOT_ACCESS_KEY_ID|S3_SNAPSHOT_SECRET_ACCESS_KEY|S3_SNAPSHOT_BUCKET|R2_ACCESS_KEY_ID|R2_SECRET_ACCESS_KEY|R2_BUCKET|R2_ACCOUNT_ID|R2_SNAPSHOT_ACCESS_KEY_ID|R2_SNAPSHOT_SECRET_ACCESS_KEY|R2_SNAPSHOT_BUCKET|TIGRIS_STORAGE_ACCESS_KEY_ID|TIGRIS_STORAGE_SECRET_ACCESS_KEY|TIGRIS_STORAGE_BUCKET|TIGRIS_SNAPSHOT_ACCESS_KEY|TIGRIS_SNAPSHOT_SECRET_KEY|TIGRIS_SNAPSHOT_STORAGE_BUCKET|BLOB_READ_WRITE_TOKEN|VERCEL_BLOB_BUCKET|GCS_PROJECT_ID|GCS_BUCKET|GCS_CLIENT_EMAIL|GCS_PRIVATE_KEY|AZURE_ACCOUNT_NAME|AZURE_ACCOUNT_KEY|AZURE_CONTAINER|AZURE_SNAPSHOT_ACCOUNT_NAME|AZURE_SNAPSHOT_ACCOUNT_KEY|AZURE_SNAPSHOT_CONTAINER|TENSORLAKE_API_KEY|TENSORLAKE_API_URL|TENSORLAKE_ORGANIZATION_ID|TENSORLAKE_PROJECT_ID|TENSORLAKE_FILESYSTEM|ARCHIL_S3_ACCESS_KEY_ID|ARCHIL_S3_SECRET_ACCESS_KEY|ARCHIL_BUCKET|ARCHIL_REGION|ARCHIL_BRANCH|NEON_BUCKET|NEON_ENDPOINT|NEON_ACCESS_KEY_ID|NEON_SECRET_ACCESS_KEY|NEON_REGION|NEON_SNAPSHOT_BUCKET|NEON_SNAPSHOT_ENDPOINT|NEON_SNAPSHOT_ACCESS_KEY_ID|NEON_SNAPSHOT_SECRET_ACCESS_KEY|NEON_SNAPSHOT_REGION|MOSAIC_BUCKET|MOSAIC_ACCESS_KEY_ID|MOSAIC_SECRET_ACCESS_KEY|MOSAIC_REGION|MOSAIC_ENDPOINT|MOSAIC_SNAPSHOT_BUCKET|MOSAIC_SNAPSHOT_ACCESS_KEY_ID|MOSAIC_SNAPSHOT_SECRET_ACCESS_KEY|MOSAIC_SNAPSHOT_REGION|MOSAIC_SNAPSHOT_ENDPOINT|BENCHMARKS_PLATFORM_API_KEY)' # All matrix jobs for this workflow run share a single platform run. The # attempt number is included so re-runs do not rejoin completed runs. diff --git a/.github/workflows/storage-benchmarks.yml b/.github/workflows/storage-benchmarks.yml index c822d51c1..a7048e985 100644 --- a/.github/workflows/storage-benchmarks.yml +++ b/.github/workflows/storage-benchmarks.yml @@ -86,7 +86,7 @@ jobs: run: rm -rf results/storage/ - name: Run storage benchmark run: | - . benchmarks/scripts/load-vault-secrets.sh '^(AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|AWS_REGION|S3_BUCKET|R2_ACCESS_KEY_ID|R2_SECRET_ACCESS_KEY|R2_BUCKET|R2_ACCOUNT_ID|TIGRIS_STORAGE_ACCESS_KEY_ID|TIGRIS_STORAGE_SECRET_ACCESS_KEY|TIGRIS_STORAGE_BUCKET|BLOB_READ_WRITE_TOKEN|VERCEL_BLOB_BUCKET|GCS_PROJECT_ID|GCS_BUCKET|GCS_CLIENT_EMAIL|GCS_PRIVATE_KEY|AZURE_ACCOUNT_NAME|AZURE_ACCOUNT_KEY|AZURE_CONTAINER|TENSORLAKE_API_KEY|TENSORLAKE_API_URL|TENSORLAKE_ORGANIZATION_ID|TENSORLAKE_PROJECT_ID|TENSORLAKE_FILESYSTEM|ARCHIL_S3_ACCESS_KEY_ID|ARCHIL_S3_SECRET_ACCESS_KEY|ARCHIL_BUCKET|ARCHIL_REGION|ARCHIL_BRANCH|NEON_BUCKET|NEON_ENDPOINT|NEON_ACCESS_KEY_ID|NEON_SECRET_ACCESS_KEY|NEON_REGION|MOSAIC_BUCKET|MOSAIC_ACCESS_KEY_ID|MOSAIC_SECRET_ACCESS_KEY|MOSAIC_REGION|MOSAIC_ENDPOINT|COMPUTESDK_ADMIN_API_KEY|BENCHMARKS_PLATFORM_API_KEY)' + . benchmarks/scripts/load-vault-secrets.sh '^(AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|AWS_REGION|S3_BUCKET|R2_ACCESS_KEY_ID|R2_SECRET_ACCESS_KEY|R2_BUCKET|R2_ACCOUNT_ID|TIGRIS_STORAGE_ACCESS_KEY_ID|TIGRIS_STORAGE_SECRET_ACCESS_KEY|TIGRIS_STORAGE_BUCKET|BLOB_READ_WRITE_TOKEN|VERCEL_BLOB_BUCKET|GCS_PROJECT_ID|GCS_BUCKET|GCS_CLIENT_EMAIL|GCS_PRIVATE_KEY|AZURE_ACCOUNT_NAME|AZURE_ACCOUNT_KEY|AZURE_CONTAINER|TENSORLAKE_API_KEY|TENSORLAKE_API_URL|TENSORLAKE_ORGANIZATION_ID|TENSORLAKE_PROJECT_ID|TENSORLAKE_FILESYSTEM|ARCHIL_S3_ACCESS_KEY_ID|ARCHIL_S3_SECRET_ACCESS_KEY|ARCHIL_BUCKET|ARCHIL_REGION|ARCHIL_BRANCH|NEON_BUCKET|NEON_ENDPOINT|NEON_ACCESS_KEY_ID|NEON_SECRET_ACCESS_KEY|NEON_REGION|MOSAIC_BUCKET|MOSAIC_ACCESS_KEY_ID|MOSAIC_SECRET_ACCESS_KEY|MOSAIC_REGION|MOSAIC_ENDPOINT|BENCHMARKS_PLATFORM_API_KEY)' # Every size runs in one job, so a run covers all of them and the # dashboard can compare sizes within it. The attempt number is included diff --git a/benchmarks/scale/coordinator.ts b/benchmarks/scale/coordinator.ts index 92a79e5bc..9570df68a 100644 --- a/benchmarks/scale/coordinator.ts +++ b/benchmarks/scale/coordinator.ts @@ -35,7 +35,7 @@ async function main() { })(); const provider = getProvider(PROVIDER); - const benchApiKey = process.env.COMPUTESDK_API_KEY ?? process.env.COMPUTESDK_ADMIN_API_KEY; + const benchApiKey = process.env.BENCHMARKS_PLATFORM_API_KEY ?? process.env.COMPUTESDK_API_KEY; const LABEL = process.env.LABEL ?? `scale.${PROVIDER}`; // Platform orchestration target. start.ts creates the run + plans the workers, diff --git a/benchmarks/scale/scripts/aggregate.ts b/benchmarks/scale/scripts/aggregate.ts index 92e4c5446..958ee6fbd 100644 --- a/benchmarks/scale/scripts/aggregate.ts +++ b/benchmarks/scale/scripts/aggregate.ts @@ -75,7 +75,7 @@ function usage(): string { ' --help, -h Print this help', '', 'Required env:', - ' COMPUTESDK_API_KEY Bench API token', + ' BENCHMARKS_PLATFORM_API_KEY or COMPUTESDK_API_KEY Bench API token', ].join('\n'); } @@ -105,7 +105,7 @@ const args = parseArgs(); const client = createBenchmarkClient({ baseUrl: 'https://platform.computesdk.com/api/v1', - apiKey: process.env.COMPUTESDK_ADMIN_API_KEY ?? process.env.COMPUTESDK_API_KEY, + apiKey: process.env.BENCHMARKS_PLATFORM_API_KEY ?? process.env.COMPUTESDK_API_KEY, }); // Some analytics endpoints lag the run (events still importing) — degrade to a diff --git a/benchmarks/scale/scripts/start.ts b/benchmarks/scale/scripts/start.ts index a861fbe46..098758925 100644 --- a/benchmarks/scale/scripts/start.ts +++ b/benchmarks/scale/scripts/start.ts @@ -33,7 +33,8 @@ * * Optional env: * GITHUB_SHA Defaults to `git rev-parse HEAD` or "local" - * COMPUTESDK_API_KEY Bench ingest token (Bearer) + * BENCHMARKS_PLATFORM_API_KEY Bench ingest token (Bearer) + * COMPUTESDK_API_KEY Legacy bench ingest token (fallback) * SCALE_IMAGE_REPO Default image repository when --image is unset * (default: nscr.io/5enq753trme1k/scale) * SCALE_IMAGE_TAG Default image tag when --image is unset @@ -231,7 +232,7 @@ interface ShardOpts { shardIndex?: number; shardCount?: number; // Platform orchestration target; undefined when bench reporting is disabled - // (no COMPUTESDK_API_KEY, or run creation failed). The coordinator then runs + // (no BENCHMARKS_PLATFORM_API_KEY or COMPUTESDK_API_KEY, or run creation failed). The coordinator then runs // Tigris-only. benchmarkRunId?: string; participantSlug?: string; @@ -257,8 +258,8 @@ function buildEnv(opts: ShardOpts): Record { // in addition to the logger's in-memory buffer uploaded as coordinator.log. COORDINATOR_LOG_PATH: '/tmp/coordinator.log', }; + if (process.env.BENCHMARKS_PLATFORM_API_KEY) env.BENCHMARKS_PLATFORM_API_KEY = process.env.BENCHMARKS_PLATFORM_API_KEY; if (process.env.COMPUTESDK_API_KEY) env.COMPUTESDK_API_KEY = process.env.COMPUTESDK_API_KEY; - if (process.env.COMPUTESDK_ADMIN_API_KEY) env.COMPUTESDK_ADMIN_API_KEY = process.env.COMPUTESDK_ADMIN_API_KEY; if (opts.label !== undefined) env.LABEL = opts.label; if (opts.groupId !== undefined && opts.shardIndex !== undefined && opts.shardCount !== undefined) { env.GROUP_ID = opts.groupId; @@ -371,13 +372,13 @@ async function launchOne(shard: number, opts: ShardOpts, log: Logger): Promise { - const apiKey = process.env.COMPUTESDK_ADMIN_API_KEY ?? process.env.COMPUTESDK_API_KEY; + const apiKey = process.env.BENCHMARKS_PLATFORM_API_KEY ?? process.env.COMPUTESDK_API_KEY; if (!apiKey) { - console.log(' bench: disabled (no COMPUTESDK_API_KEY) — Tigris-only run\n'); + console.log(' bench: disabled (no BENCHMARKS_PLATFORM_API_KEY or COMPUTESDK_API_KEY) — Tigris-only run\n'); return null; } try { diff --git a/benchmarks/scale/scripts/watch.ts b/benchmarks/scale/scripts/watch.ts index 687379b97..8f9c8bd23 100644 --- a/benchmarks/scale/scripts/watch.ts +++ b/benchmarks/scale/scripts/watch.ts @@ -97,7 +97,7 @@ const args = parseArgs(); const client = createBenchmarkClient({ baseUrl: 'https://platform.computesdk.com/api/v1', - apiKey: process.env.COMPUTESDK_ADMIN_API_KEY ?? process.env.COMPUTESDK_API_KEY, + apiKey: process.env.BENCHMARKS_PLATFORM_API_KEY ?? process.env.COMPUTESDK_API_KEY, }); function classifyError(err: unknown): { status: string; detail: string } { diff --git a/packages/benchsdk-api/src/client.ts b/packages/benchsdk-api/src/client.ts index 1780ac9b2..5af824bed 100644 --- a/packages/benchsdk-api/src/client.ts +++ b/packages/benchsdk-api/src/client.ts @@ -74,7 +74,7 @@ function queryString(input: Record): string { function getApiKey(input?: string): string | undefined { if (input) return input; if (typeof process === 'undefined') return undefined; - return process.env.COMPUTESDK_ADMIN_API_KEY ?? process.env.COMPUTESDK_API_KEY ?? process.env.BENCHMARKS_PLATFORM_API_KEY; + return process.env.BENCHMARKS_PLATFORM_API_KEY ?? process.env.COMPUTESDK_API_KEY; } function getErrorCode(error: unknown): string { diff --git a/packages/benchsdk-api/src/types.ts b/packages/benchsdk-api/src/types.ts index 0c748b777..98f066e04 100644 --- a/packages/benchsdk-api/src/types.ts +++ b/packages/benchsdk-api/src/types.ts @@ -4,7 +4,7 @@ export type JsonObject = { [key: string]: JsonValue }; export interface BenchmarkClientConfig { /** API base URL. Defaults to https://platform.computesdk.com/api/v1. */ baseUrl?: string; - /** Bearer token. Defaults to COMPUTESDK_ADMIN_API_KEY, then COMPUTESDK_API_KEY, then BENCHMARKS_PLATFORM_API_KEY. */ + /** Bearer token. Defaults to BENCHMARKS_PLATFORM_API_KEY, then COMPUTESDK_API_KEY. */ apiKey?: string; /** Custom fetch implementation, mostly useful for tests. */ fetch?: typeof fetch; diff --git a/packages/benchsdk-runner/src/runner.ts b/packages/benchsdk-runner/src/runner.ts index d14a7bc86..320fb4801 100644 --- a/packages/benchsdk-runner/src/runner.ts +++ b/packages/benchsdk-runner/src/runner.ts @@ -388,12 +388,10 @@ function resolvePlatform(): { baseUrl: string; apiKey: string } { const root = (process.env.BENCHMARKS_PLATFORM_URL || DEFAULT_PLATFORM_URL).replace(/\/+$/, ''); const apiKey = process.env.BENCHMARKS_PLATFORM_API_KEY ?? - process.env.COMPUTESDK_ADMIN_API_KEY ?? process.env.COMPUTESDK_API_KEY; if (!apiKey) { throw new Error( - 'An API key is required. Set one of BENCHMARKS_PLATFORM_API_KEY, COMPUTESDK_ADMIN_API_KEY, ' + - 'or COMPUTESDK_API_KEY in your environment. Create an org-scoped API key in your ' + + 'An API key is required. Set BENCHMARKS_PLATFORM_API_KEY in your environment. Create an org-scoped API key in your ' + 'organization settings on the platform.' ); } diff --git a/packages/benchsdk/README.md b/packages/benchsdk/README.md index 3d3712298..5f3e64a15 100644 --- a/packages/benchsdk/README.md +++ b/packages/benchsdk/README.md @@ -22,7 +22,7 @@ import { createBenchmarkClient } from '@benchsdk/client'; import { compute } from 'computesdk'; const client = createBenchmarkClient({ - apiKey: process.env.COMPUTESDK_ADMIN_API_KEY, + apiKey: process.env.BENCHMARKS_PLATFORM_API_KEY, }); // `task` is a raw function; declare named steps imperatively via `step(...)`, @@ -62,7 +62,7 @@ Task results are flushed to the platform in batches of 1,000 records by default. import { createBenchmarkClient } from '@benchsdk/client'; const client = createBenchmarkClient({ - apiKey: process.env.COMPUTESDK_ADMIN_API_KEY, + apiKey: process.env.BENCHMARKS_PLATFORM_API_KEY, }); await client.upsertBenchmark('scale', { diff --git a/packages/benchsdk/src/__tests__/integration.test.ts b/packages/benchsdk/src/__tests__/integration.test.ts index d56bf0c77..a1c872e84 100644 --- a/packages/benchsdk/src/__tests__/integration.test.ts +++ b/packages/benchsdk/src/__tests__/integration.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'vitest'; import { BenchmarkApiError, createBenchmarkClient } from '../client'; -const shouldRun = !!process.env.COMPUTESDK_ADMIN_API_KEY || !!process.env.COMPUTESDK_API_KEY; +const shouldRun = !!process.env.BENCHMARKS_PLATFORM_API_KEY || !!process.env.COMPUTESDK_API_KEY; const describeIntegration = shouldRun ? describe : describe.skip; function isInvalidApiKeyError(error: unknown): boolean { @@ -167,7 +167,7 @@ describeIntegration('benchmark orchestrator integration', () => { expect(Array.isArray(imports.items)).toBe(true); } catch (error) { if (isInvalidApiKeyError(error)) { - console.warn('Skipping bench orchestrator smoke - COMPUTESDK_ADMIN_API_KEY is invalid for the platform API.'); + console.warn('Skipping bench orchestrator smoke - the bench API key is invalid for the platform API.'); return; } throw error; diff --git a/packages/benchsdk/src/__tests__/public-api.contract.test.ts b/packages/benchsdk/src/__tests__/public-api.contract.test.ts index 5badb70ac..fe6123f39 100644 --- a/packages/benchsdk/src/__tests__/public-api.contract.test.ts +++ b/packages/benchsdk/src/__tests__/public-api.contract.test.ts @@ -393,23 +393,23 @@ describe('transport layer', () => { } }); - it('VAL-SDK-005: falls back to COMPUTESDK_ADMIN_API_KEY then COMPUTESDK_API_KEY', async () => { - const prevAdmin = process.env.COMPUTESDK_ADMIN_API_KEY; + it('VAL-SDK-005: falls back to BENCHMARKS_PLATFORM_API_KEY then COMPUTESDK_API_KEY', async () => { + const prevBenchmarks = process.env.BENCHMARKS_PLATFORM_API_KEY; const prevApi = process.env.COMPUTESDK_API_KEY; try { - process.env.COMPUTESDK_ADMIN_API_KEY = 'admin_key'; + process.env.BENCHMARKS_PLATFORM_API_KEY = 'benchmarks_key'; process.env.COMPUTESDK_API_KEY = 'api_key'; const first = recordingClient(() => jsonResponse({ benchmark: benchmarkResource() })); await createBenchmarkClient({ baseUrl: BASE, fetch: first.fetchMock }).getBenchmark('scale'); - expect(first.calls[0].headers.Authorization).toBe('Bearer admin_key'); + expect(first.calls[0].headers.Authorization).toBe('Bearer benchmarks_key'); - delete process.env.COMPUTESDK_ADMIN_API_KEY; + delete process.env.BENCHMARKS_PLATFORM_API_KEY; const second = recordingClient(() => jsonResponse({ benchmark: benchmarkResource() })); await createBenchmarkClient({ baseUrl: BASE, fetch: second.fetchMock }).getBenchmark('scale'); expect(second.calls[0].headers.Authorization).toBe('Bearer api_key'); } finally { - if (prevAdmin === undefined) delete process.env.COMPUTESDK_ADMIN_API_KEY; - else process.env.COMPUTESDK_ADMIN_API_KEY = prevAdmin; + if (prevBenchmarks === undefined) delete process.env.BENCHMARKS_PLATFORM_API_KEY; + else process.env.BENCHMARKS_PLATFORM_API_KEY = prevBenchmarks; if (prevApi === undefined) delete process.env.COMPUTESDK_API_KEY; else process.env.COMPUTESDK_API_KEY = prevApi; } From 64ca68ece03fc7aceda114c73d50582825606cba Mon Sep 17 00:00:00 2001 From: david Date: Mon, 24 Aug 2026 18:35:30 +0000 Subject: [PATCH 4/4] fix(scale): bundle coordinator as ESM to support top-level await dependencies Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .gitignore | 2 +- benchmarks/scale/Dockerfile | 4 ++-- benchmarks/scale/inject-require.mjs | 4 ++++ benchmarks/scale/scripts/start.ts | 4 ++-- package.json | 2 +- 5 files changed, 10 insertions(+), 6 deletions(-) create mode 100644 benchmarks/scale/inject-require.mjs diff --git a/.gitignore b/.gitignore index 5bd7d9254..f5d264c78 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,5 @@ node_modules/ .DS_Store dist/ scale-run.log -scale.cjs +scale.mjs fetch-raw-json.ts \ No newline at end of file diff --git a/benchmarks/scale/Dockerfile b/benchmarks/scale/Dockerfile index 759b4f0e3..5cd946f38 100644 --- a/benchmarks/scale/Dockerfile +++ b/benchmarks/scale/Dockerfile @@ -22,6 +22,6 @@ RUN pnpm run --silent bundle:scale FROM node:24-alpine WORKDIR /app -COPY --from=builder /app/dist/scale.cjs /app/coordinator.cjs +COPY --from=builder /app/dist/scale.mjs /app/coordinator.mjs -CMD ["node", "/app/coordinator.cjs"] +CMD ["node", "/app/coordinator.mjs"] diff --git a/benchmarks/scale/inject-require.mjs b/benchmarks/scale/inject-require.mjs new file mode 100644 index 000000000..5cea08ee4 --- /dev/null +++ b/benchmarks/scale/inject-require.mjs @@ -0,0 +1,4 @@ +import { createRequire } from 'node:module'; +if (typeof globalThis.require === 'undefined') { + globalThis.require = createRequire(import.meta.url); +} diff --git a/benchmarks/scale/scripts/start.ts b/benchmarks/scale/scripts/start.ts index 098758925..105efdd16 100644 --- a/benchmarks/scale/scripts/start.ts +++ b/benchmarks/scale/scripts/start.ts @@ -52,11 +52,11 @@ const BENCHMARK_SLUG = 'scale'; const BENCH_TIMEOUT_MS = 120_000; // The container command — the coordinator bundle baked into the scale image -// (see src/scale/Dockerfile: CMD ["node", "/app/coordinator.cjs"]). We run it +// (see src/scale/Dockerfile: CMD ["node", "/app/coordinator.mjs"]). We run it // as the container's PID 1 (not via runCommand), so its stdout is captured // natively by `nsc logs --kind containers` and the instance auto-reaps // when it exits — exactly like the old `nsc run` path. -const COORDINATOR_ARGS = ['node', '/app/coordinator.cjs']; +const COORDINATOR_ARGS = ['node', '/app/coordinator.mjs']; // Namespace Compute API endpoints. The package's own create/describe use these // via its exported fetchNamespace; we call CreateInstance directly so we can set diff --git a/package.json b/package.json index 67d1298fd..ad9d9acd1 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,7 @@ "bench:snapshot-fork:s3": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/storage/snapshot-fork.bench.ts --provider aws-s3", "bench:snapshot-fork:wide": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/storage/snapshot-fork.bench.ts --dataset wide", "bench:snapshot-fork:deep": "tsx packages/benchsdk-runner/dist/bin.js run benchmarks/storage/snapshot-fork.bench.ts --dataset deep", - "bundle:scale": "esbuild benchmarks/scale/coordinator.ts --bundle --platform=node --target=node20 --format=cjs --outfile=dist/scale.cjs", + "bundle:scale": "esbuild benchmarks/scale/coordinator.ts --bundle --platform=node --target=node20 --format=esm --outfile=dist/scale.mjs --inject:benchmarks/scale/inject-require.mjs", "bench:scale:image": "SCALE_IMAGE_REPO=${SCALE_IMAGE_REPO:-nscr.io/5enq753trme1k/scale}; SCALE_IMAGE_TAG=${SCALE_IMAGE_TAG:-latest}; docker buildx build --platform linux/amd64 --provenance=false --sbom=false -f benchmarks/scale/Dockerfile --push -t \"$SCALE_IMAGE_REPO:$SCALE_IMAGE_TAG\" .", "bench:scale:local": "tsx benchmarks/scale/coordinator.ts", "bench:scale:start": "tsx benchmarks/scale/scripts/start.ts",