diff --git a/.github/workflows/llm-benchmark-periodic.yml b/.github/workflows/llm-benchmark-periodic.yml index 8e46d9ebc2b..587cfdafd42 100644 --- a/.github/workflows/llm-benchmark-periodic.yml +++ b/.github/workflows/llm-benchmark-periodic.yml @@ -48,9 +48,29 @@ concurrency: cancel-in-progress: true jobs: + prepare_matrix: + name: Prepare benchmark matrix + runs-on: ubuntu-latest + outputs: + languages: ${{ steps.matrix.outputs.languages }} + steps: + - id: matrix + env: + INPUT_LANGUAGES: ${{ inputs.languages || 'rust,csharp,typescript' }} + run: | + languages="$(jq -cn --arg value "$INPUT_LANGUAGES" '$value | split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))')" + echo "languages=$languages" >> "$GITHUB_OUTPUT" + run-benchmarks: + name: Run benchmarks (${{ matrix.lang }}) + needs: prepare_matrix runs-on: spacetimedb-new-runner-2 + strategy: + fail-fast: false + matrix: + lang: ${{ fromJSON(needs.prepare_matrix.outputs.languages) }} + steps: - name: Checkout repository uses: actions/checkout@v4 @@ -61,11 +81,13 @@ jobs: - uses: Swatinem/rust-cache@v2 - name: Setup .NET SDK + if: matrix.lang == 'csharp' uses: actions/setup-dotnet@v4 with: global-json-file: global.json - name: Install WASI workload + if: matrix.lang == 'csharp' env: DOTNET_MULTILEVEL_LOOKUP: "0" DOTNET_CLI_HOME: ${{ runner.temp }}/dotnet-home @@ -74,25 +96,25 @@ jobs: dotnet workload install wasi-experimental --skip-manifest-update --disable-parallel - name: Pack C# runtime packages - if: ${{ github.event_name != 'workflow_dispatch' || contains(inputs.languages || 'rust,csharp,typescript', 'csharp') }} + if: matrix.lang == 'csharp' run: | dotnet pack -c Release crates/bindings-csharp/BSATN.Runtime dotnet pack -c Release crates/bindings-csharp/Runtime - name: Set up Node.js - if: ${{ github.event_name != 'workflow_dispatch' || contains(inputs.languages || 'rust,csharp,typescript', 'typescript') }} + if: matrix.lang == 'typescript' uses: actions/setup-node@v4 with: node-version: 22 - name: Install pnpm - if: ${{ github.event_name != 'workflow_dispatch' || contains(inputs.languages || 'rust,csharp,typescript', 'typescript') }} + if: matrix.lang == 'typescript' uses: ./.github/actions/setup-pnpm with: run_install: true - name: Build TypeScript SDK - if: ${{ github.event_name != 'workflow_dispatch' || contains(inputs.languages || 'rust,csharp,typescript', 'typescript') }} + if: matrix.lang == 'typescript' run: pnpm build working-directory: crates/bindings-typescript @@ -119,8 +141,11 @@ jobs: DOTNET_SKIP_FIRST_TIME_EXPERIENCE: "1" MSBUILDDISABLENODEREUSE: "1" DOTNET_CLI_USE_MSBUILD_SERVER: "0" - LLM_BENCH_CSHARP_CONCURRENCY: "1" - INPUT_LANGUAGES: ${{ inputs.languages || 'rust,csharp,typescript' }} + LLM_BENCH_CONCURRENCY: "8" + LLM_BENCH_RUST_CONCURRENCY: "4" + LLM_BENCH_CSHARP_CONCURRENCY: "2" + LLM_BENCH_ROUTE_CONCURRENCY: ${{ matrix.lang == 'typescript' && '4' || '2' }} + INPUT_LANGUAGE: ${{ matrix.lang }} INPUT_MODEL_SET: ${{ inputs.model_set || 'website_active' }} INPUT_MODELS: ${{ inputs.models || '' }} INPUT_MODES: ${{ inputs.modes || 'guidelines,no_context' }} @@ -128,7 +153,7 @@ jobs: INPUT_TASKS: ${{ inputs.tasks || '' }} INPUT_DRY_RUN: ${{ inputs.dry_run || 'false' }} run: | - LANGS="$INPUT_LANGUAGES" + LANG="$INPUT_LANGUAGE" MODEL_SET="$INPUT_MODEL_SET" MODELS="$INPUT_MODELS" MODES="$INPUT_MODES" @@ -162,45 +187,21 @@ jobs: ;; esac - SUCCEEDED=0 - FAILED=0 - for LANG in $(echo "$LANGS" | tr ',' ' '); do - EXTRA_ARGS=() - if [ -n "$CATEGORIES" ]; then - EXTRA_ARGS+=(--categories "$CATEGORIES") - fi - if [ -n "$TASKS" ]; then - EXTRA_ARGS+=(--tasks "$TASKS") - fi - if [ "$DRY_RUN" = "true" ]; then - EXTRA_ARGS+=(--dry-run) - fi - - if [ "$MODEL_SET" = "website_active" ]; then - if llm_benchmark run --lang "$LANG" --modes "$MODES" --model-source remote "${EXTRA_ARGS[@]}"; then - SUCCEEDED=$((SUCCEEDED + 1)) - else - echo "::warning::Benchmark run failed for lang=$LANG" - FAILED=$((FAILED + 1)) - fi - elif [ "$MODEL_SET" = "local_defaults" ]; then - if llm_benchmark run --lang "$LANG" --modes "$MODES" "${EXTRA_ARGS[@]}"; then - SUCCEEDED=$((SUCCEEDED + 1)) - else - echo "::warning::Benchmark run failed for lang=$LANG" - FAILED=$((FAILED + 1)) - fi - else - if llm_benchmark run --lang "$LANG" --modes "$MODES" --models "${MODEL_ARGS[@]}" "${EXTRA_ARGS[@]}"; then - SUCCEEDED=$((SUCCEEDED + 1)) - else - echo "::warning::Benchmark run failed for lang=$LANG models=$MODELS" - FAILED=$((FAILED + 1)) - fi - fi - done - echo "Benchmark runs: $SUCCEEDED succeeded, $FAILED failed" - if [ "$FAILED" -gt 0 ]; then - echo "::error::$FAILED benchmark run(s) failed" - exit 1 + EXTRA_ARGS=() + if [ -n "$CATEGORIES" ]; then + EXTRA_ARGS+=(--categories "$CATEGORIES") + fi + if [ -n "$TASKS" ]; then + EXTRA_ARGS+=(--tasks "$TASKS") + fi + if [ "$DRY_RUN" = "true" ]; then + EXTRA_ARGS+=(--dry-run) + fi + + if [ "$MODEL_SET" = "website_active" ]; then + llm_benchmark run --lang "$LANG" --modes "$MODES" --model-source remote "${EXTRA_ARGS[@]}" + elif [ "$MODEL_SET" = "local_defaults" ]; then + llm_benchmark run --lang "$LANG" --modes "$MODES" "${EXTRA_ARGS[@]}" + else + llm_benchmark run --lang "$LANG" --modes "$MODES" --models "${MODEL_ARGS[@]}" "${EXTRA_ARGS[@]}" fi diff --git a/skills/csharp-server/SKILL.md b/skills/csharp-server/SKILL.md index d88fd5ec0d2..1d08ba89fe7 100644 --- a/skills/csharp-server/SKILL.md +++ b/skills/csharp-server/SKILL.md @@ -17,6 +17,8 @@ metadata: Reducers are static methods in a `static partial class`; tables are `public partial struct`s. This reference keeps everything in one `public static partial class Module`, which needs only `using SpacetimeDB;`: +Methods exported through SpacetimeDB attributes, including reducers, procedures, views, HTTP handlers, and routers, must be `public static`; generated bindings invoke them from another class. + ```csharp using SpacetimeDB; @@ -51,8 +53,10 @@ public partial struct Entity [PrimaryKey] [AutoInc] public ulong Id; + [SpacetimeDB.Index.BTree] public Identity Owner; public string Name; + [SpacetimeDB.Index.BTree] public bool Active; } ``` @@ -90,8 +94,11 @@ The complete set of column attributes: [AutoInc] // auto-increment (use 0 as placeholder on insert) [Unique] // unique constraint; indexes the column, enables .Find() [SpacetimeDB.Index.BTree] // btree index (enables .Filter() on this column) +[Default(true)] // migration-safe default for a newly appended field ``` +Defaults support compatible addition of a newly appended field. Do not apply `[Default(...)]` to primary-key, unique, or auto-increment fields. + ## Indexes Write the index attribute fully qualified: `[SpacetimeDB.Index.BTree]`. Prefer inline for single-column; multi-column uses struct-level: @@ -155,6 +162,8 @@ public static void OnConnect(ReducerContext ctx) { ... } public static void OnDisconnect(ReducerContext ctx) { ... } ``` +`ctx.ConnectionId` is `ConnectionId?`, including in connection lifecycle reducers. Check or unwrap it before storing it in a non-nullable column or passing it to an index accessor. + ## Views ```csharp @@ -162,17 +171,44 @@ public static void OnDisconnect(ReducerContext ctx) { ... } [SpacetimeDB.View(Accessor = "ActiveUsers", Public = true)] public static List ActiveUsers(AnonymousViewContext ctx) { - return ctx.Db.Entity.Iter().Where(e => e.Active).ToList(); + return ctx.Db.Entity.Active.Filter(true).ToList(); } // Per-user view: -[SpacetimeDB.View(Accessor = "MyProfile", Public = true)] -public static Entity? MyProfile(ViewContext ctx) +[SpacetimeDB.View(Accessor = "MyEntities", Public = true)] +public static List MyEntities(ViewContext ctx) { - return ctx.Db.Entity.Identity.Find(ctx.Sender) as Entity?; + return ctx.Db.Entity.Owner.Filter(ctx.Sender).ToList(); } ``` +View contexts expose read-only table handles. These support `Count` plus `Find`/`Filter` on declared indexes, but not full-table `Iter()`. Start procedural view traversal from an indexed lookup or filter. + +Query-builder views use `ViewContext`, `ctx.From`, and return `IQuery` directly. Use `Where` for predicates and `RightSemijoin` when the result should contain right-side rows that have a matching left-side row: + +```csharp +ctx.From.Article().Where(article => article.Published.Eq(true)); +ctx.From.Subscription().RightSemijoin( + ctx.From.Account(), + (subscription, account) => subscription.AccountId.Eq(account.Id) +); +``` + +Declare a procedural view primary key in its attribute: `[SpacetimeDB.View(Accessor = "CatalogEntry", Public = true, PrimaryKey = nameof(CatalogRow.Sku))]`. + +Procedural-view primary keys are explicit schema metadata. Add one only when the view itself is required to expose a primary key; a source table's primary key is not inherited by the view. + +Inclusive btree ranges use tuples, for example `ctx.Db.Shipment.DeliverBy.Filter((new Timestamp(1_000), new Timestamp(2_000)))`. + +## Client Visibility Filters + +```csharp +[ClientVisibilityFilter] +public static readonly Filter PrivateNoteFilter = new Filter.Sql( + "SELECT * FROM OwnedRow WHERE Owner = :sender" +); +``` + ## Reducer Context API `ReducerContext` (`ctx`) is the only source of sender identity, time, and randomness; stdlib clocks and RNG are unavailable in modules. @@ -229,6 +265,71 @@ var at = new ScheduleAt.Interval(TimeSpan.FromSeconds(5)); ctx.Db.TickTimer.Insert(new TickTimer { ScheduledId = 0, ScheduledAt = at }); ``` +Scheduled reducer callbacks use the ordinary `[SpacetimeDB.Reducer]` attribute. There is no `ReducerKind.Scheduled`; the table's `Scheduled` option associates the callback. + +To construct a `ConnectionId` from a 128-bit numeric representation, encode it as exactly 16 little-endian bytes and call `ConnectionId.From(bytes)`. The result is nullable and must be checked before use. + +## Procedures and HTTP + +Procedures are unstable APIs, so modules using them should include `#pragma warning disable STDB_UNSTABLE`. They receive `ProcedureContext` and may return `[SpacetimeDB.Type]` values: + +```csharp +[SpacetimeDB.Type] +public partial struct ResultValue { public string Value; } + +[SpacetimeDB.Procedure] +public static ResultValue Inspect(ProcedureContext ctx, string input) => + new() { Value = input }; +``` + +Outbound HTTP is available through `ctx.Http`; handle its success/error result before using the response. Open short database transactions with `ctx.WithTx`. Perform network I/O before opening the transaction, and keep only database work inside its callback. + +```csharp +var result = ctx.Http.Get(uri); +var text = result.Match( + response => response.Body.ToStringUtf8Lossy(), + error => throw new Exception(error.Message) +); + +ctx.WithTx(tx => +{ + tx.Db.ScoreRecord.Insert(new ScoreRecord { Id = 0, Owner = ctx.Sender, Value = 1 }); + return 0; +}); +``` + +`WithTx` is generic: its callback must return a value (use `return 0;` when no result is needed). The callback's generated `tx.Db` exposes module table accessors; `ProcedureContext` and `HandlerContext` do not expose tables directly. + +For non-GET requests, construct and send an `HttpRequest` directly: + +```csharp +var result = ctx.Http.Send(new HttpRequest +{ + Uri = uri, + Method = SpacetimeDB.HttpMethod.Post, + Headers = new() { new HttpHeader("content-type", "text/plain") }, + Body = HttpBody.FromString(payload), +}); +``` + +`HttpResponse.StatusCode` is `ushort`. HTTP headers are `HttpHeader` values, not tuples, and each header's `Value` is `byte[]`; decode text values with `System.Text.Encoding.UTF8.GetString(header.Value)`. Treat bodies as bytes: use `HttpBody.FromString(...)` to create text, `new HttpBody(bytes)` to supply raw bytes, `ToBytes()` to read raw bytes, and `ToStringUtf8Lossy()` to read text. There is no `HttpBody.FromBytes`, and `ToString()` does not return body contents. Qualify `SpacetimeDB.HttpMethod` when .NET's implicit `System.Net.Http` imports could make the name ambiguous. + +Scheduled procedures use the ordinary scheduled-table shape. Its `Scheduled` name refers to a `[SpacetimeDB.Procedure]` method taking `ProcedureContext` plus the scheduled row, and database access inside that procedure goes through `ctx.WithTx`. + +Inbound HTTP uses handler attributes and one router. Handler database access also goes through `ctx.WithTx`: + +```csharp +[SpacetimeDB.HttpHandler] +public static HttpResponse Health(HandlerContext ctx, HttpRequest request) => new( + 200, HttpVersion.Http11, new(), HttpBody.FromString("ok") +); + +[SpacetimeDB.HttpRouter] +public static Router Routes() => SpacetimeDB.Router.New().Get("/health", Handlers.Health); +``` + +`Handlers` is generated from methods marked `[SpacetimeDB.HttpHandler]`; do not declare it yourself. Reference an attributed method in a router as `Handlers.MethodName`. + ## Custom Types ```csharp diff --git a/skills/rust-server/SKILL.md b/skills/rust-server/SKILL.md index f83a4387b1c..0e282242263 100644 --- a/skills/rust-server/SKILL.md +++ b/skills/rust-server/SKILL.md @@ -17,8 +17,9 @@ metadata: ```rust use spacetimedb::{ - reducer, table, Identity, ReducerContext, SpacetimeType, Table, - ConnectionId, ScheduleAt, TimeDuration, Timestamp, Uuid, + procedure, reducer, table, Filter, Identity, ProcedureContext, Query, + ReducerContext, SpacetimeType, Table, ConnectionId, ScheduleAt, + TimeDuration, Timestamp, Uuid, }; ``` @@ -36,6 +37,7 @@ pub struct Entity { pub id: u64, pub owner: Identity, pub name: String, + #[index(btree)] pub active: bool, } ``` @@ -50,6 +52,7 @@ Options: `accessor = snake_case` (required), `public`, `scheduled(reducer_fn)`, |-----------|-------| | `u8` / `u16` / `u32` / `u64` / `u128` | unsigned integers | | `i8` / `i16` / `i32` / `i64` / `i128` | signed integers | +| `spacetimedb::sats::u256` / `spacetimedb::sats::i256` | 256-bit integers | | `f32` / `f64` | floats | | `bool` | boolean | | `String` | text | @@ -68,8 +71,11 @@ Options: `accessor = snake_case` (required), `public`, `scheduled(reducer_fn)`, #[auto_inc] // auto-increment (use 0 as placeholder on insert) #[unique] // unique constraint #[index(btree)] // btree index (enables .filter() on this column) +#[default(true)] // migration-safe default for a newly appended column ``` +Defaults support compatible addition of a newly appended field. Do not place `#[default(...)]` on primary-key, unique, or auto-increment columns. + ## Indexes Prefer `#[index(btree)]` inline for single-column. Multi-column uses table-level: @@ -122,13 +128,15 @@ ctx.db.entity().iter(); // All rows ctx.db.entity().count(); // Count rows ctx.db.entity().id().update(Entity { name: new_name, ..existing }); // Update (override + spread) ctx.db.entity().id().delete(entity_id); // Delete by PK -ctx.db.entity().name().delete("Alice"); // Delete by indexed column +ctx.db.entity().name().delete("Alice".to_string()); // Delete by indexed String column ``` Note: `iter()` and `filter()` return iterators. Collect to Vec if you need `.sort()`, `.filter()`, `.map()`. Range queries on btree indexes: `filter(18..=65)`, `filter(18..)`, `filter(..18)`. +String column accessors operate on the column's owned `String` type, not `&str`. Pass a `String` or `&String` to `find` and `delete`; index filters borrow the key, as in `ctx.db.product().category().filter(&"hardware".to_string())`. + ## Lifecycle Hooks ```rust @@ -142,6 +150,8 @@ pub fn on_connect(ctx: &ReducerContext) { ... } pub fn on_disconnect(ctx: &ReducerContext) { ... } ``` +The current connection ID is available through `ctx.connection_id()` (not a public field) and may be absent outside connection-scoped calls. + ## Views ```rust @@ -150,7 +160,7 @@ use spacetimedb::{view, AnonymousViewContext}; #[view(accessor = active_users, public)] fn active_users(ctx: &AnonymousViewContext) -> Vec { - ctx.db.entity().iter().filter(|e| e.active).collect() + ctx.db.entity().active().filter(true).collect() } // Per-user view (result varies by sender): @@ -162,6 +172,39 @@ fn my_profile(ctx: &ViewContext) -> Option { } ``` +Procedural-view table handles support indexed `find` and `filter` access, but not full-table `iter()`. Start a procedural view from an appropriate table index. + +Declare a procedural view primary key in the view attribute: + +```rust +#[view(accessor = catalog_entry, public, primary_key = sku)] +fn catalog_entry(ctx: &AnonymousViewContext) -> Vec { ... } +``` + +Procedural-view primary keys are explicit schema metadata. Add one only when the view itself is required to expose a primary key; a source table's primary key is not inherited by the view. + +Query-builder views use `ViewContext`, `ctx.from`, and return `impl Query`. Use `filter` for predicates and `right_semijoin` when the result should contain right-side rows that have a matching left-side row: + +```rust +ctx.from.article().filter(|article| article.published.eq(true)) +ctx.from.subscription().right_semijoin( + ctx.from.account(), + |subscription, account| subscription.account_id.eq(account.id), +) +``` + +Query-builder comparisons against a `String` column take an owned `String`, for example `row.label.eq("active".to_string())`. + +## Client Visibility Filters + +```rust +use spacetimedb::Filter; + +#[spacetimedb::client_visibility_filter] +const PRIVATE_NOTE_FILTER: Filter = + Filter::Sql("SELECT * FROM owned_row WHERE owner = :sender"); +``` + ## Reducer Context API `ReducerContext` (`ctx`) is the only source of sender identity, time, and randomness; stdlib clocks and RNG are unavailable in modules. @@ -212,6 +255,57 @@ let at = ScheduleAt::Interval(std::time::Duration::from_secs(5).into()); ctx.db.tick_timer().insert(TickTimer { scheduled_id: 0, scheduled_at: at }); ``` +Construct a connection ID from a numeric representation with `ConnectionId::from_u128(value)`. + +## Procedures and HTTP + +Procedures use `&mut ProcedureContext` and may return typed values: + +```rust +use spacetimedb::{procedure, ProcedureContext, SpacetimeType}; + +#[derive(SpacetimeType)] +pub struct ResultValue { pub value: String } + +#[procedure] +pub fn inspect(_ctx: &mut ProcedureContext, input: String) -> ResultValue { + ResultValue { value: input } +} +``` + +Outbound HTTP is available through `ctx.http`. Convenience methods such as `get` return a response, and other methods use `Request::builder()` with `ctx.http.send(request)`. `response.status()` returns a `StatusCode`; use `is_success()` to test it or `as_u16()` when a numeric status is needed. Responses are not cloneable, so inspect status and headers before consuming the body with `response.into_body().into_string_lossy()`. Header values use the fallible `to_str()` conversion; they do not provide `as_str()`. + +Open short database transactions with `ctx.with_tx(|tx| ...)`. Access tables inside the callback through `tx.db`, not directly on `tx`. It returns the callback's value directly, so do not call `unwrap` or `expect` on the result unless the callback itself returns a `Result`. The callback implements `Fn`, so clone captured owned values when storing them rather than moving them out of the closure. Perform network I/O before opening the transaction. + +For an outbound request without a convenience method: + +```rust +let request = Request::builder() + .method("POST") + .uri(url) + .body(Body::from_bytes(data)) + .unwrap(); +let response = ctx.http.send(request).expect("request failed"); +``` + +Scheduled procedures use the ordinary scheduled-table shape, but the scheduled function is marked `#[procedure]` and receives `&mut ProcedureContext` plus the scheduled row. + +Inbound HTTP uses handler functions and one router: + +```rust +use spacetimedb::http::{handler, router, Body, HandlerContext, Request, Response, Router}; + +#[handler] +fn health(_ctx: &mut HandlerContext, _request: Request) -> Response { + Response::builder().status(200).body(Body::from_bytes("ok")).unwrap() +} + +#[router] +fn routes() -> Router { Router::new().get("/health", health) } +``` + +`HandlerContext` does not expose `db`; open database access with `ctx.with_tx(|tx| ...)`. HTTP response bodies must own their data or borrow `'static` data, so convert dynamic borrowed text to an owned `String` before passing it to `Body::from_bytes`. + ## Logging ```rust diff --git a/skills/typescript-server/SKILL.md b/skills/typescript-server/SKILL.md index 2e9326b3150..c9f2e7343fd 100644 --- a/skills/typescript-server/SKILL.md +++ b/skills/typescript-server/SKILL.md @@ -40,14 +40,21 @@ export const addRecord = spacetimedb.reducer( ); ``` +Only table definitions belong in `schema({...})`. Row and object builders used as reducer arguments or view return types are not schema entries. + +Named runtime exports are reserved for values registered with SpacetimeDB, such as reducers, lifecycle hooks, views, procedures, HTTP exports, and visibility filters. Keep ordinary helper functions and constants unexported. + ## Imports -`spacetimedb/server` is the only import path for server modules: +Schema builders and module exports come from `spacetimedb/server`. Runtime value classes such as `ScheduleAt`, `Timestamp`, and `ConnectionId` come from the root `spacetimedb` package; `Range` comes from `spacetimedb/server`: ```typescript -import { schema, table, t } from 'spacetimedb/server'; -import { SenderError } from 'spacetimedb/server'; -import { ScheduleAt } from 'spacetimedb'; // for scheduled tables only +import { + schema, table, t, SenderError, + type InferSchema, type ReducerCtx, +} from 'spacetimedb/server'; +import { ConnectionId, ScheduleAt, Timestamp } from 'spacetimedb'; +import { Range } from 'spacetimedb/server'; ``` ## Tables @@ -75,9 +82,11 @@ Every column is a `t` builder value: | Builder | JS type | Notes | |---------|---------|-------| +| `t.u8()` / `t.u16()` / `t.u32()` | number | | +| `t.i8()` / `t.i16()` / `t.i32()` | number | | | `t.u64()` | bigint | Use `0n` literals | | `t.i64()` | bigint | Use `0n` literals | -| `t.u32()` / `t.i32()` | number | | +| `t.u128()` / `t.i128()` / `t.u256()` / `t.i256()` | bigint | | | `t.f64()` / `t.f32()` | number | | | `t.bool()` | boolean | | | `t.string()` | string | | @@ -87,13 +96,17 @@ Every column is a `t` builder value: | `t.timeDuration()` | TimeDuration | | | `t.scheduleAt()` | ScheduleAt | | -Modifiers (complete set): `.primaryKey()`, `.autoInc()`, `.unique()`, `.index('btree')` +Modifiers: `.primaryKey()`, `.autoInc()`, `.unique()`, `.index('btree')`, `.default(value)`. + +Use `.default(value)` only for a newly appended migration-safe field. Do not put defaults on primary-key, unique, or auto-increment columns. Optional columns: `nickname: t.option(t.string())` +Schema builders describe the database's wire types; they are not TypeScript type names. For example, a `t.u16()` value is a TypeScript `number`, not a value cast to a type named `u16`. + ## Indexes -Prefer inline `.index('btree')` for single-column. Use named indexes only for multi-column: +Use inline `.index('btree')` when a single-column index does not need a named accessor. Use an `indexes` entry when the accessor is named explicitly or the index spans multiple columns. Every `indexes` entry requires `columns`; do not also add `.index('btree')` to the same column. ```typescript // Inline (preferred for single-column): @@ -145,6 +158,14 @@ ctx.db.score_record.id.update({ ...existing, value: 2 }); // Update (spread + o ctx.db.score_record.id.delete(recordId); // Delete by PK ``` +Insert through the table accessor (`ctx.db.score_record.insert(...)`). Primary-key, unique, and index accessors support lookup or mutation of existing rows, but do not have `insert(...)`. + +`insert(...)` returns the inserted row, including database-assigned auto-increment fields. + +The accessor for a primary key or index is the declared column name. For example, a primary key named `eventId` is accessed as `ctx.db.event.eventId`, not `ctx.db.event.id`. + +The schema value registers module exports but does not expose database rows. Pass a context into any helper that needs `ctx.db`. + Note: `iter()` and `filter()` return iterators. Spread to Array for `.sort()`, `.filter()`, `.map()`. ## Lifecycle Hooks @@ -157,9 +178,19 @@ export const onConnect = spacetimedb.clientConnected((ctx) => { ... }); export const onDisconnect = spacetimedb.clientDisconnected((ctx) => { ... }); ``` +`ctx.connectionId` is `ConnectionId | null`, including in lifecycle contexts. Guard it before passing it to a helper or using it as a table key. + ## Reducer Context API -`ctx` is the only source of sender identity, time, and randomness; stdlib clocks and RNG are unavailable in modules. In helpers, type it as `ReducerCtx>`. +`ctx` is the only source of sender identity, time, and randomness; stdlib clocks and RNG are unavailable in modules. Let exported callbacks infer their context type. In helpers, use `ReducerCtx>`; do not annotate a context as `any`, because that erases table row types and can make `bigint` expressions infer as `number`. + +```typescript +type Ctx = ReducerCtx>; + +function findRecord(ctx: Ctx, id: bigint) { + return ctx.db.score_record.id.find(id); +} +``` ```typescript // Auth: ctx.sender is the caller's Identity @@ -174,7 +205,8 @@ ctx.db.item.insert({ id: 0n, createdAt: ctx.timestamp }); // Deterministic RNG const f: number = ctx.random(); // [0.0, 1.0) -const roll: number = ctx.random.integerInRange(1, 6); // inclusive +const roll: number = ctx.random.integerInRange(1, 6); // safe JS number bounds/result, inclusive +const storedRoll: bigint = BigInt(roll); // convert the result for an i64/u64 column const bytes: Uint8Array = ctx.random.fill(new Uint8Array(16)); // Client: Timestamp → Date @@ -183,8 +215,24 @@ new Date(Number(row.createdAt.microsSinceUnixEpoch / 1000n)); Do not construct `Identity` values from strings (e.g. `'hex' as Identity`): serialization fails and kills the module. Identities come from `ctx.sender` or `t.identity()` columns. +Construct a `ConnectionId` from its numeric representation with `new ConnectionId(value)` after importing `ConnectionId` from `spacetimedb`. + +Construct exact timestamps with `new Timestamp(micros)` after importing `Timestamp` from `spacetimedb`. Inclusive index ranges use `Range` from `spacetimedb/server`: + +```typescript +import { Timestamp } from 'spacetimedb'; +import { Range } from 'spacetimedb/server'; + +ctx.db.shipment.deliverBy.filter(new Range( + { tag: 'included', value: new Timestamp(1_000n) }, + { tag: 'included', value: new Timestamp(2_000n) }, +)); +``` + ## Scheduled Tables +The reducer or procedure referenced by a table's `scheduled` option must be exported. + ```typescript import { ScheduleAt } from 'spacetimedb'; // ScheduleAt comes from the root package @@ -231,6 +279,10 @@ A client subscribing to a view receives only the rows it returns. Use a per-user (keyed on `ctx.sender`) for per-viewer access control: deleting a row it depends on (e.g. a membership row) automatically drops the rows it was exposing from that client. +`t.row(...)` and `t.object(...)` return schema builders, not TypeScript runtime row types. Let a view callback infer its result, or annotate a separately declared structural type such as `Array<{ sku: bigint; label: string }>`. A named output type must not reuse the generated PascalCase name of its view accessor (for example, reserve `DiscountedProduct` for a `discounted_product` view). + +Both `spacetimedb.view(...)` and `spacetimedb.anonymousView(...)` take three arguments: view options, the declared return schema, and the callback. + ```typescript // Anonymous view (same for all clients): export const activeUsers = spacetimedb.anonymousView( @@ -246,3 +298,78 @@ export const myProfile = spacetimedb.view( (ctx) => ctx.db.entity.identity.find(ctx.sender) ?? undefined ); ``` + +For a procedural view primary key, define the output with `t.row` and mark its field `.primaryKey()`: + +```typescript +const CatalogKey = t.row('CatalogKey', { + sku: t.u64().primaryKey(), + label: t.string(), +}); +``` + +Query-builder views use `ctx.from` and return the query directly. Because a query returns a row set, declare its return schema as `t.array(tableName.rowType)`. Use `where` for predicates and `rightSemijoin` when the result should contain right-side rows that have a matching left-side row: + +```typescript +ctx => ctx.from.article.where(article => article.published.eq(true)) +ctx => ctx.from.subscription.rightSemijoin( + ctx.from.account, + (subscription, account) => subscription.accountId.eq(account.id) +) +``` + +The method name identifies which side is returned: `A.leftSemijoin(B, ...)` returns rows from `A`, while `A.rightSemijoin(B, ...)` returns rows from `B`. To return one table's rows when another table has a match, put the returned table on the corresponding side. Semijoins do not project combined columns from both tables. + +Procedural views read through `ctx.db` and return materialized values such as arrays. Query-builder values from `ctx.from` are returned directly; they are not iterators and cannot be spread, looped over, or mixed with array methods. Use a procedural view when the result is a custom row assembled from multiple tables. + +## Client Visibility Filters + +```typescript +export const privateNoteFilter = spacetimedb.clientVisibilityFilter.sql( + 'SELECT * FROM owned_row WHERE owner = :sender' +); +``` + +## Procedures and HTTP + +`spacetimedb` is the local schema value returned by `schema({...})`; it is not a named export to import from `spacetimedb/server`. + +Procedures declare argument and return types: + +```typescript +const Result = t.object('Result', { value: t.string() }); + +export const inspect = spacetimedb.procedure( + { input: t.string() }, + Result, + (_ctx, { input }) => ({ value: input }) +); +``` + +Procedure callbacks are synchronous. Do not mark them `async` or use `await`; return the declared value directly. A procedure with return type `t.unit()` returns `{}`. + +TypeScript outbound HTTP uses `ctx.http.fetch(url, options)`, including for non-GET requests; it does not provide convenience methods such as `get()` or `post()`. Responses expose the numeric `status`, `headers.get(name)`, and `text()` APIs. + +`t.array(t.u8())` values are `number[]`. Convert one to `new Uint8Array(value)` before using it as a binary request body. + +Procedures and handlers open short database transactions with `ctx.withTx(tx => ...)`. Perform network I/O before opening the transaction; only database work belongs inside its callback. + +Scheduled procedures use the ordinary scheduled-table shape. Its `scheduled` option references an exported `spacetimedb.procedure(...)` value instead of a reducer, and the procedure accepts the scheduled row as its argument. + +Inbound HTTP uses `httpHandler`, `httpRouter`, `Router`, and `SyncResponse`: + +`httpHandler` and `httpRouter` are methods on the local `spacetimedb` schema value, not named imports. + +```typescript +import { Router, SyncResponse } from 'spacetimedb/server'; + +export const health = spacetimedb.httpHandler((_ctx, _request) => + new SyncResponse('ok', { + status: 200, + headers: { 'content-type': 'text/plain' }, + }) +); +export const routes = spacetimedb.httpRouter(new Router().get('/health', health)); +``` + +Handlers are synchronous: return `SyncResponse` directly rather than marking the callback `async`. Pass the exported `httpHandler(...)` value to the router, not its raw callback. The router selects the path, while a handler reads request data with APIs such as `request.text()`; `Request` has no `path` property. A handler context does not expose `ctx.db`; use `ctx.withTx(tx => ...)` when a handler needs transactional database access. diff --git a/tools/xtask-llm-benchmark/Cargo.toml b/tools/xtask-llm-benchmark/Cargo.toml index bce7d73e717..6565fa9723e 100644 --- a/tools/xtask-llm-benchmark/Cargo.toml +++ b/tools/xtask-llm-benchmark/Cargo.toml @@ -24,12 +24,15 @@ dotenvy = "0.15" async-trait = "0.1.89" tokio = { version = "1", features = ["rt-multi-thread", "macros"] } urlencoding = "2.1.3" -reqwest = { version = "0.12", features = ["json"] } +reqwest = { version = "0.12", features = ["blocking", "json"] } futures = "0.3.31" regex = "1" heck = "0.5.0" thiserror = "2.0.17" +[dev-dependencies] +spacetimedb-lib = { workspace = true, features = ["serde"] } + [lib] path = "src/lib.rs" diff --git a/tools/xtask-llm-benchmark/src/bench/mod.rs b/tools/xtask-llm-benchmark/src/bench/mod.rs index 6b1ebb1cf98..7d900be053a 100644 --- a/tools/xtask-llm-benchmark/src/bench/mod.rs +++ b/tools/xtask-llm-benchmark/src/bench/mod.rs @@ -6,7 +6,7 @@ mod templates; pub mod types; pub(crate) mod utils; -pub use publishers::{DotnetPublisher, Publisher, SpacetimeRustPublisher, TypeScriptPublisher}; +pub use publishers::{DotnetPublisher, PublishedDatabase, Publisher, SpacetimeRustPublisher, TypeScriptPublisher}; pub use runner::TaskRunner; pub use types::{RunOutcome, TaskPaths}; -pub use utils::bench_route_concurrency; +pub use utils::{bench_route_concurrency, run_scope_tag}; diff --git a/tools/xtask-llm-benchmark/src/bench/publishers.rs b/tools/xtask-llm-benchmark/src/bench/publishers.rs index b7fb74c6936..e6e05bbefec 100644 --- a/tools/xtask-llm-benchmark/src/bench/publishers.rs +++ b/tools/xtask-llm-benchmark/src/bench/publishers.rs @@ -1,4 +1,5 @@ use crate::bench::utils::sanitize_db_name; +use crate::eval::spacetime_command; use anyhow::{bail, Context, Result}; use regex::Regex; use std::borrow::Cow; @@ -6,12 +7,17 @@ use std::env; use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; -use std::sync::{ - atomic::{AtomicU64, Ordering}, - LazyLock, -}; +use std::sync::LazyLock; use std::time::{SystemTime, UNIX_EPOCH}; +static CLI_SESSION_TAG: LazyLock = LazyLock::new(|| { + let started_at = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(); + format!("{}-{started_at}", std::process::id()) +}); + fn workspace_root() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")) .ancestors() @@ -126,42 +132,43 @@ fn resolve_node_exe(nodejs_dir: Option<&Path>) -> Option { struct CliRootDir { path: PathBuf, + remove_on_drop: bool, } impl CliRootDir { fn path(&self) -> &Path { &self.path } -} -impl Drop for CliRootDir { - fn drop(&mut self) { - let _ = fs::remove_dir_all(&self.path); + fn remove(&self) -> Result<()> { + match fs::remove_dir_all(&self.path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error).with_context(|| format!("failed to remove CLI root {}", self.path.display())), + } } -} -fn isolated_cli_root() -> Result { - static COUNTER: AtomicU64 = AtomicU64::new(0); + fn claim_cleanup(&mut self) { + self.remove_on_drop = true; + } - for _ in 0..16 { - let nanos = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|duration| duration.as_nanos()) - .unwrap_or(0); - let id = COUNTER.fetch_add(1, Ordering::Relaxed); - let path = env::temp_dir().join(format!("stdb-llm-cli-{}-{nanos}-{id}", std::process::id())); - match fs::create_dir(&path) { - Ok(()) => return Ok(CliRootDir { path }), - Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue, - Err(error) => return Err(error.into()), - } + fn relinquish_cleanup(&mut self) { + self.remove_on_drop = false; } +} - bail!("failed to create isolated SpacetimeDB CLI root directory"); +impl Drop for CliRootDir { + fn drop(&mut self) { + if self.remove_on_drop + && let Err(error) = self.remove() + { + eprintln!("[cleanup] failed to remove CLI root {}: {error:#}", self.path.display()); + } + } } fn spacetime_cmd(cli_root: &CliRootDir) -> Command { - let mut cmd = Command::new("spacetime"); + let mut cmd = spacetime_command(); cmd.arg("--root-dir").arg(cli_root.path()); cmd } @@ -205,8 +212,86 @@ fn strip_ansi_codes(s: &str) -> Cow<'_, str> { /* Shared */ /* -------------------------------------------------------------------------- */ +pub struct PublishedDatabase { + host_url: String, + db: String, + // Keep the publishing CLI session alive so deletion uses the database + // owner's credentials without sharing mutable CLI state across routes. + cli_root: CliRootDir, + deleted: bool, +} + +impl PublishedDatabase { + fn new(host_url: &str, db: String, mut cli_root: CliRootDir) -> Self { + cli_root.claim_cleanup(); + Self { + host_url: host_url.to_owned(), + db, + cli_root, + deleted: false, + } + } + + fn delete_inner(&self) -> Result<()> { + let mut cmd = spacetime_cmd(&self.cli_root); + cmd.arg("delete") + .arg("-y") + .arg("--no-config") + .arg("--server") + .arg(&self.host_url) + .arg(&self.db); + run(&mut cmd, "spacetime delete").with_context(|| { + format!( + "failed to clean up benchmark database `{}` on {}", + self.db, self.host_url + ) + }) + } + + pub fn delete(mut self) -> Result<()> { + self.delete_inner()?; + self.deleted = true; + self.cli_root.remove()?; + self.cli_root.relinquish_cleanup(); + Ok(()) + } + + /// Relinquish cleanup ownership after another handle for the same database + /// has been created by a migration republish. + pub fn relinquish(mut self) { + self.deleted = true; + self.cli_root.relinquish_cleanup(); + } +} + +impl Drop for PublishedDatabase { + fn drop(&mut self) { + if !self.deleted + && let Err(error) = self.delete_inner() + { + eprintln!( + "[cleanup] failed to delete benchmark database `{}` during fallback cleanup: {error:#}", + self.db + ); + } + } +} + pub trait Publisher: Send + Sync { - fn publish(&self, host_url: &str, source: &Path, module_name: &str) -> Result<()>; + fn publish( + &self, + host_url: &str, + source: &Path, + module_name: &str, + clear_database: bool, + ) -> Result; +} + +fn database_cli_root(module_name: &str, remove_on_drop: bool) -> Result { + let db = sanitize_db_name(module_name); + let path = env::temp_dir().join(format!("stdb-llm-cli-{}-{db}", CLI_SESSION_TAG.as_str())); + fs::create_dir_all(&path)?; + Ok(CliRootDir { path, remove_on_drop }) } /// Check if the process was killed by a signal (e.g., SIGSEGV = 11) @@ -222,20 +307,248 @@ fn signal_killed_by(_status: &std::process::ExitStatus) -> Option { } /// Check if the failure is a transient error that should be retried. -/// These are resource contention issues in the dotnet WASI SDK. +/// These are resource contention issues and diagnostic-free child-process crashes. fn is_transient_build_error(stderr: &str, stdout: &str) -> bool { let combined = format!("{stderr}{stdout}"); - // "Pipe is broken" errors from WASI SDK parallel builds + let output = combined.to_ascii_lowercase(); + let has_source_diagnostic = output.lines().any(|line| { + let line = line.trim_start(); + line.starts_with("error[") + || line.contains(": error cs") + || line.contains(": error stdb") + || line.contains(": error il") + || line.starts_with("error ts") + || line.contains("rust-lld: error:") + || line.contains("lld-link: error:") + || line.strip_prefix("error:").is_some_and(|message| { + let message = message.trim_start(); + !message.starts_with("could not compile `") + && !message.starts_with("process didn't exit successfully") + && !message.starts_with("command [\"cargo\", \"build\"") + && !message.starts_with("command [\"dotnet\", \"publish\"") + && !message.starts_with("failed to run custom build command for `") + && !message.starts_with("linking with `rust-lld.exe` failed") + }) + }); + if has_source_diagnostic { + return false; + } + + let has_terminal_failure = output.lines().any(|line| { + let line = line.trim_start(); + line.starts_with("error: could not compile `") + || line.starts_with("failed:") + || line.starts_with("exception:") + || line.contains("build failed") + || line.contains("panicked at") + }); + let diagnostic_free_rust_exit = (output.contains("process didn't exit successfully") + && output.contains("--crate-name")) + || output.contains("linking with `rust-lld.exe` failed: exit code: 1"); + let wrapper_command_exit = output.contains("error: command [\"cargo\", \"build\"") + || output.contains("error: command [\"dotnet\", \"publish\"") + || (output.contains("error: failed to run custom build command for") + && output.contains("process didn't exit successfully")); + let progress_only_exit = output.trim().is_empty() + || output.lines().filter(|line| !line.trim().is_empty()).all(|line| { + let line = line.trim_start(); + [ + "building ", + "compiling ", + "downloaded ", + "downloading ", + "finished ", + "logged in with identity ", + "optimizing ", + "packages: ", + "progress: ", + "recreating ", + "restored ", + "restoring ", + "saving config ", + "warning: this login will not work for any other servers.", + "we have logged in directly to your target server.", + ] + .iter() + .any(|prefix| line.starts_with(prefix)) + || line.chars().all(|character| matches!(character, '+' | '-')) + }); + combined.contains("Pipe is broken") || combined.contains("EmitBundleObjectFiles") - // Other transient resource errors || combined.contains("Unable to read data from the transport connection") - // WASI SDK tar extraction race condition - multiple parallel builds - // trying to extract the same tarball simultaneously || (combined.contains("wasi-sdk") && combined.contains("tar")) || (combined.contains("MSB3073") && combined.contains("exited with code 2")) - // dotnet can crash below spacetime while spacetime exits 1. || combined.contains("code Result<()> { @@ -342,7 +655,13 @@ impl DotnetPublisher { } impl Publisher for DotnetPublisher { - fn publish(&self, host_url: &str, source: &Path, module_name: &str) -> Result<()> { + fn publish( + &self, + host_url: &str, + source: &Path, + module_name: &str, + clear_database: bool, + ) -> Result { if !source.exists() { bail!("no source: {}", source.display()); } @@ -354,12 +673,14 @@ impl Publisher for DotnetPublisher { let source = source .canonicalize() .with_context(|| format!("failed to resolve C# source path {}", source.display()))?; - let cli_root = isolated_cli_root()?; + let cli_root = database_cli_root(module_name, clear_database)?; let mut pubcmd = spacetime_cmd(&cli_root); + pubcmd.arg("publish"); + if clear_database { + pubcmd.arg("-c"); + } pubcmd - .arg("publish") - .arg("-c") .arg("-y") .arg("--server") .arg(host_url) @@ -370,7 +691,7 @@ impl Publisher for DotnetPublisher { Self::configure_dotnet_env(&mut pubcmd); run(&mut pubcmd, "spacetime publish (csharp)")?; - Ok(()) + Ok(PublishedDatabase::new(host_url, db, cli_root)) } } /* -------------------------------------------------------------------------- */ @@ -390,7 +711,13 @@ impl SpacetimeRustPublisher { } impl Publisher for SpacetimeRustPublisher { - fn publish(&self, host_url: &str, source: &Path, module_name: &str) -> Result<()> { + fn publish( + &self, + host_url: &str, + source: &Path, + module_name: &str, + clear_database: bool, + ) -> Result { if !source.exists() { bail!("no source: {}", source.display()); } @@ -401,22 +728,26 @@ impl Publisher for SpacetimeRustPublisher { // sanitize db + server let db = sanitize_db_name(module_name); - let cli_root = isolated_cli_root()?; + let cli_root = database_cli_root(module_name, clear_database)?; // 2) Publish - run( - spacetime_cmd(&cli_root) - .arg("publish") - .arg("-c") - .arg("-y") - .arg("--server") - .arg(host_url) - .arg(&db) - .current_dir(source), - "spacetime publish", - )?; + let mut pubcmd = spacetime_cmd(&cli_root); + pubcmd.arg("publish"); + if clear_database { + pubcmd.arg("-c"); + } + pubcmd + .arg("-y") + .arg("--server") + .arg(host_url) + .arg(&db) + .current_dir(source); + if let Some(target_dir) = env::var_os("LLM_BENCH_RUST_TARGET_DIR") { + pubcmd.env("CARGO_TARGET_DIR", target_dir); + } + run(&mut pubcmd, "spacetime publish")?; - Ok(()) + Ok(PublishedDatabase::new(host_url, db, cli_root)) } } @@ -437,7 +768,13 @@ impl TypeScriptPublisher { } impl Publisher for TypeScriptPublisher { - fn publish(&self, host_url: &str, source: &Path, module_name: &str) -> Result<()> { + fn publish( + &self, + host_url: &str, + source: &Path, + module_name: &str, + clear_database: bool, + ) -> Result { if !source.exists() { bail!("no source: {}", source.display()); } @@ -445,7 +782,7 @@ impl Publisher for TypeScriptPublisher { Self::ensure_package_json(source)?; let db = sanitize_db_name(module_name); - let cli_root = isolated_cli_root()?; + let cli_root = database_cli_root(module_name, clear_database)?; // Install dependencies (--ignore-workspace to avoid parent workspace interference). let nodejs_dir = configured_nodejs_dir(); @@ -520,9 +857,11 @@ impl Publisher for TypeScriptPublisher { // Publish (spacetime CLI handles TypeScript compilation internally) let mut publish_cmd = spacetime_cmd(&cli_root); + publish_cmd.arg("publish"); + if clear_database { + publish_cmd.arg("-c"); + } publish_cmd - .arg("publish") - .arg("-c") .arg("-y") .arg("--server") .arg(host_url) @@ -539,6 +878,6 @@ impl Publisher for TypeScriptPublisher { } run(&mut publish_cmd, "spacetime publish (typescript)")?; - Ok(()) + Ok(PublishedDatabase::new(host_url, db, cli_root)) } } diff --git a/tools/xtask-llm-benchmark/src/bench/runner.rs b/tools/xtask-llm-benchmark/src/bench/runner.rs index 2536b5e5fe1..fc6da3884af 100644 --- a/tools/xtask-llm-benchmark/src/bench/runner.rs +++ b/tools/xtask-llm-benchmark/src/bench/runner.rs @@ -1,24 +1,23 @@ use anyhow::{anyhow, bail, Context, Result}; use chrono::Utc; -use futures::{stream, StreamExt}; +use futures::{stream, StreamExt, TryStreamExt}; use serde_json::json; use spacetimedb_data_structures::map::{HashCollectionExt as _, HashMap, HashSet}; use std::fs; use std::path::{Path, PathBuf}; -use std::sync::OnceLock; -use std::time::Instant; -use tokio::sync::Mutex; +use std::time::{Duration, Instant}; use tokio::task; -use crate::bench::publishers::{DotnetPublisher, SpacetimeRustPublisher, TypeScriptPublisher}; +use crate::bench::publishers::{DotnetPublisher, PublishedDatabase, SpacetimeRustPublisher, TypeScriptPublisher}; use crate::bench::templates::materialize_project; use crate::bench::types::{BenchRunContext, PublishParams, RunContext, RunOneError}; pub(crate) use crate::bench::types::{RunOutcome, TaskPaths}; use crate::bench::utils::{ bench_concurrency, bench_csharp_concurrency, bench_rust_concurrency, category_slug, debug_llm, fmt_dur, - print_llm_output, sanitize_db_name, task_slug, work_server_dir_scoped, + golden_db_name, print_llm_output, run_scope_tag, sanitize_db_name, task_slug, work_server_dir_scoped, }; use crate::bench::Publisher; +use crate::eval::scorers::call_reducer_json_out; use crate::eval::{Lang, ScoreDetails}; use crate::generated::resolve_by_path; use crate::llm::model_routes::ModelRoute; @@ -30,56 +29,44 @@ pub struct TaskRunner { pub ts_publisher: TypeScriptPublisher, } -static BUILT_KEYS: OnceLock>> = OnceLock::new(); - struct PendingRetry { task: TaskPaths, error: anyhow::Error, } -fn build_key(lang: Lang, selectors: Option<&[String]>) -> String { - let v = match selectors { - Some(s) if !s.is_empty() => { - let mut t = s.to_vec(); - t.sort(); // stable key independent of order - t +fn task_result_or_infrastructure_error( + task: TaskPaths, + result: Result, + started: chrono::DateTime, +) -> Result<(TaskPaths, Result)> { + match result { + Err(RunOneError::Infrastructure { msg, llm_output }) => { + let output_note = llm_output + .as_deref() + .map(|output| format!("; generated output retained in work directory ({} bytes)", output.len())) + .unwrap_or_default(); + bail!("benchmark infrastructure failure: {msg}{output_note}"); } - _ => vec!["ALL".to_string()], - }; - let joined = v.join(","); - format!("{lang:?}:{joined}") + other => Ok(( + task, + other.map(|mut outcome| { + outcome.started_at.get_or_insert(started); + outcome + }), + )), + } } -/// Build goldens **once per (lang, selector-set)** in this process. -/// If selectors is None/empty, that means "ALL tasks". +/// Build the golden databases and return handles that keep their owning CLI +/// sessions alive until scoring has finished. pub async fn ensure_goldens_built_once( host: Option, bench_root: &Path, lang: Lang, selectors: Option<&[String]>, -) -> Result<()> { - let key = build_key(lang, selectors); - let set = BUILT_KEYS.get_or_init(|| Mutex::new(HashSet::new())); - { - let set = set.lock(); - if set.await.contains(&key) { - return Ok(()); - } - } - // single-flight for this key - let set_guard = set.lock().await; - if set_guard.contains(&key) { - return Ok(()); - } - - // IMPORTANT: pass selectors through so we only build needed goldens - build_goldens_only_for_lang(host, bench_root, lang, selectors).await?; - - // mark as built - drop(set_guard); - let mut set = BUILT_KEYS.get().unwrap().lock().await; - set.insert(key); - Ok(()) + golden_scope: &str, +) -> Result> { + build_goldens_only_for_lang(host, bench_root, lang, selectors, golden_scope).await } async fn publish_rust_async( @@ -87,16 +74,60 @@ async fn publish_rust_async( host_url: String, wdir: PathBuf, db: String, -) -> Result<()> { - task::spawn_blocking(move || publisher.publish(&host_url, &wdir, &db)).await??; + clear_database: bool, +) -> Result { + task::spawn_blocking(move || publisher.publish(&host_url, &wdir, &db, clear_database)).await? +} +async fn publish_cs_async( + publisher: DotnetPublisher, + host_url: String, + wdir: PathBuf, + db: String, + clear_database: bool, +) -> Result { + task::spawn_blocking(move || publisher.publish(&host_url, &wdir, &db, clear_database)).await? +} +async fn publish_ts_async( + publisher: TypeScriptPublisher, + host_url: String, + wdir: PathBuf, + db: String, + clear_database: bool, +) -> Result { + task::spawn_blocking(move || publisher.publish(&host_url, &wdir, &db, clear_database)).await? +} + +async fn delete_database_async(database: PublishedDatabase) -> Result<()> { + task::spawn_blocking(move || database.delete()).await??; Ok(()) } -async fn publish_cs_async(publisher: DotnetPublisher, host_url: String, wdir: PathBuf, db: String) -> Result<()> { - task::spawn_blocking(move || publisher.publish(&host_url, &wdir, &db)).await??; + +pub async fn delete_databases(databases: Vec) -> Result<()> { + stream::iter(databases.into_iter().map(delete_database_async)) + .buffer_unordered(8) + .try_collect::>() + .await?; Ok(()) } -async fn publish_ts_async(publisher: TypeScriptPublisher, host_url: String, wdir: PathBuf, db: String) -> Result<()> { - task::spawn_blocking(move || publisher.publish(&host_url, &wdir, &db)).await??; + +async fn ensure_server_healthy(host: Option<&str>) -> Result<()> { + let Some(host) = host else { + return Ok(()); + }; + let ping_url = format!("{}/v1/ping", host.trim_end_matches('/')); + let response = reqwest::Client::builder() + .timeout(Duration::from_secs(5)) + .build()? + .get(&ping_url) + .send() + .await + .with_context(|| format!("benchmark server at {host} is unavailable; refusing to upload results"))?; + if !response.status().is_success() { + bail!( + "benchmark server health check at {ping_url} returned {}; refusing to upload results", + response.status() + ); + } Ok(()) } @@ -115,35 +146,15 @@ impl TaskRunner { } } - pub async fn publish_golden_only( - &self, - lang: Lang, - category: &str, - task_id: &str, - golden_src_text: &str, - golden_db: String, - host: Option, - ) -> Result<()> { - self.publish( - PublishParams { - lang, - category, - task_id, - route_tag: "", - source_text: golden_src_text, - db_name: golden_db, - host, - }, - "golden", - ) - .await + pub async fn publish_golden_only(&self, params: PublishParams<'_>) -> Result { + self.publish(params, "golden").await } - async fn publish_llm(&self, params: PublishParams<'_>) -> Result<()> { + async fn publish_llm(&self, params: PublishParams<'_>) -> Result { self.publish(params, "llm").await } - async fn publish(&self, params: PublishParams<'_>, phase: &str) -> Result<()> { + async fn publish(&self, params: PublishParams<'_>, phase: &str) -> Result { let lang_name = match params.lang { Lang::Rust => "rust", Lang::CSharp => "csharp", @@ -164,13 +175,26 @@ impl TaskRunner { )?; let host_url = params.host.unwrap_or_else(|| "local".to_owned()); - match params.lang { - Lang::Rust => publish_rust_async(self.rust_publisher, host_url, wdir, params.db_name).await?, - Lang::CSharp => publish_cs_async(self.cs_publisher, host_url, wdir, params.db_name).await?, - Lang::TypeScript => publish_ts_async(self.ts_publisher, host_url, wdir, params.db_name).await?, - } + let database = match params.lang { + Lang::Rust => { + publish_rust_async( + self.rust_publisher, + host_url, + wdir, + params.db_name, + params.clear_database, + ) + .await? + } + Lang::CSharp => { + publish_cs_async(self.cs_publisher, host_url, wdir, params.db_name, params.clear_database).await? + } + Lang::TypeScript => { + publish_ts_async(self.ts_publisher, host_url, wdir, params.db_name, params.clear_database).await? + } + }; - Ok(()) + Ok(database) } pub async fn run_one(&self, task: &TaskPaths, cfg: &RunContext<'_>) -> Result { @@ -179,8 +203,8 @@ impl TaskRunner { let category = category_slug(&task.root); let task_id = task_slug(&task.root); - let route_tag = sanitize_db_name(&cfg.route.display_name); - let golden_db = sanitize_db_name(&format!("{}-{}-golden", category, task_id)); + let route_tag = run_scope_tag(cfg.mode, cfg.route.vendor.slug(), &cfg.route.api_model); + let golden_db = golden_db_name(&category, &task_id, &route_tag); let llm_db = sanitize_db_name(&format!("{}-{}-{}-llm", category, task_id, route_tag)); let ctor = resolve_by_path(&task.root)?; @@ -263,29 +287,81 @@ impl TaskRunner { let output_tokens = llm_result.output_tokens; let llm_output = llm_result.text; + let migration_setup = load_migration_setup_source(task, cfg.lang)?; + if debug_llm() { print_llm_output(&cfg.route.display_name, &task_id, &llm_output); } - let publish_error: Option = self - .publish_llm(PublishParams { - lang: cfg.lang, - category: &category, - task_id: &task_id, - route_tag: &route_tag, - source_text: &llm_output, - db_name: llm_db.clone(), - host: cfg.host.clone(), - }) - .await - .err() - .map(|e| { - eprintln!( - "⚠️ publish failed for {}/{}/{}: {e:#}", - category, task_id, cfg.route.display_name - ); - format!("{:#}", e) + let setup_database: Result> = if let Some(setup_source) = migration_setup.as_deref() { + let setup_result = self + .publish( + PublishParams { + lang: cfg.lang, + category: &category, + task_id: &task_id, + route_tag: &route_tag, + source_text: setup_source, + db_name: llm_db.clone(), + host: cfg.host.clone(), + clear_database: true, + }, + "llm-setup", + ) + .await; + match setup_result { + Ok(database) => { + call_reducer_json_out(&llm_db, "seed", &[], Some(cfg.host.as_deref().unwrap_or("local"))) + .map_err(anyhow::Error::msg)?; + Ok(Some(database)) + } + Err(error) => Err(error), + } + } else { + Ok(None) + }; + + let (published_database, publish_error) = match setup_database { + Err(error) => (None, Some(format!("migration setup failed: {error:#}"))), + Ok(mut setup_database) => { + let publish_result = self + .publish_llm(PublishParams { + lang: cfg.lang, + category: &category, + task_id: &task_id, + route_tag: &route_tag, + source_text: &llm_output, + db_name: llm_db.clone(), + host: cfg.host.clone(), + clear_database: migration_setup.is_none(), + }) + .await; + match publish_result { + Ok(database) => { + if let Some(setup_database) = setup_database.take() { + setup_database.relinquish(); + } + (Some(database), None) + } + Err(error) => { + eprintln!( + "⚠️ publish failed for {}/{}/{}: {error:#}", + category, task_id, cfg.route.display_name + ); + (None, Some(format!("{error:#}"))) + } + } + } + }; + + if publish_error.is_some() + && let Err(error) = ensure_server_healthy(cfg.host.as_deref()).await + { + return Err(RunOneError::Infrastructure { + msg: format!("publish failed and the benchmark server is unavailable: {error:#}"), + llm_output: Some(llm_output), }); + } let mut passed = 0usize; let mut partial_sum = 0f32; @@ -319,6 +395,15 @@ impl TaskRunner { ); } + if let Some(database) = published_database + && let Err(error) = delete_database_async(database).await + { + return Err(RunOneError::Infrastructure { + msg: format!("database cleanup failed for `{llm_db}`: {error:#}"), + llm_output: Some(llm_output), + }); + } + let score_pct = if total_tasks == 0 { 0.0 } else { @@ -354,7 +439,7 @@ impl TaskRunner { golden_db: Some(golden_db), llm_db: Some(llm_db), work_dir_golden: Some( - work_server_dir_scoped(&category, &task_id, cfg.lang_name, "golden", "") + work_server_dir_scoped(&category, &task_id, cfg.lang_name, "golden", &route_tag) .to_string_lossy() .into_owned(), ), @@ -381,7 +466,7 @@ fn partition_results( lang_name: &str, route: &ModelRoute, hash: &str, -) -> (Vec, Vec) { +) -> Result<(Vec, Vec)> { let mut good = Vec::new(); let mut retry = Vec::new(); @@ -400,6 +485,13 @@ fn partition_results( Some(llm_output), )); } + Err(RunOneError::Infrastructure { msg, llm_output }) => { + let output_note = llm_output + .as_deref() + .map(|output| format!("; generated output retained in work directory ({} bytes)", output.len())) + .unwrap_or_default(); + bail!("benchmark infrastructure failure: {msg}{output_note}"); + } Err(RunOneError::Other(e)) => { // No output at all — provider error, retry eprintln!("\u{26a0}\u{fe0f} provider error, will retry: {e:?}"); @@ -408,7 +500,7 @@ fn partition_results( } } - (good, retry) + Ok((good, retry)) } fn pending_retries_to_outcomes( @@ -532,21 +624,15 @@ pub async fn run_all_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> Resu host, }; - let res = runner.run_one(&task, &run_cfg).await; - ( - task, - res.map(|mut o| { - o.started_at.get_or_insert(started); - o - }), - ) + let result = runner.run_one(&task, &run_cfg).await; + task_result_or_infrastructure_error(task, result, started) } })) .buffer_unordered(buf) - .collect() - .await; + .try_collect() + .await?; - let (mut outcomes, mut retry_tasks) = partition_results(results, lang_name, cfg.route, cfg.hash); + let (mut outcomes, mut retry_tasks) = partition_results(results, lang_name, cfg.route, cfg.hash)?; // Retry provider-error tasks until all pass or none make progress const MAX_RETRY_ROUNDS: usize = 3; @@ -586,21 +672,15 @@ pub async fn run_all_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> Resu llm, host, }; - let res = runner.run_one(&task, &run_cfg).await; - ( - task, - res.map(|mut o| { - o.started_at.get_or_insert(started); - o - }), - ) + let result = runner.run_one(&task, &run_cfg).await; + task_result_or_infrastructure_error(task, result, started) } })) .buffer_unordered(buf) - .collect() - .await; + .try_collect() + .await?; - let (new_outcomes, still_failing) = partition_results(retry_results, lang_name, cfg.route, cfg.hash); + let (new_outcomes, still_failing) = partition_results(retry_results, lang_name, cfg.route, cfg.hash)?; if new_outcomes.is_empty() && !still_failing.is_empty() { // No progress — provider is likely down. Keep the failures and stop retrying. @@ -632,6 +712,10 @@ pub async fn run_all_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> Resu println!("[runner] completed batch: {} results", outcomes.len()); + // A dead local server turns later publish failures into misleading zeroes. + // Refuse to analyze or upload any batch that cannot pass a final health check. + ensure_server_healthy(cfg.host.as_deref()).await?; + if cfg.dry_run { let _ = match maybe_generate_analysis(cfg, &outcomes).await { Ok(text) => text, @@ -728,21 +812,15 @@ pub async fn run_selected_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> host: cfg.host.clone(), }; - let res = runner.run_one(&task, &run_cfg).await; - ( - task, - res.map(|mut o| { - o.started_at.get_or_insert(started); - o - }), - ) + let result = runner.run_one(&task, &run_cfg).await; + task_result_or_infrastructure_error(task, result, started) } })) .buffer_unordered(buf) - .collect() - .await; + .try_collect() + .await?; - let (mut outcomes, retry_tasks) = partition_results(results, lang_name, cfg.route, cfg.hash); + let (mut outcomes, retry_tasks) = partition_results(results, lang_name, cfg.route, cfg.hash)?; // Retry provider-error tasks until all pass or none make progress let dropped; @@ -785,21 +863,15 @@ pub async fn run_selected_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> llm, host, }; - let res = runner.run_one(&task, &run_cfg).await; - ( - task, - res.map(|mut o| { - o.started_at.get_or_insert(started); - o - }), - ) + let result = runner.run_one(&task, &run_cfg).await; + task_result_or_infrastructure_error(task, result, started) } })) .buffer_unordered(buf) - .collect() - .await; + .try_collect() + .await?; - let (new_outcomes, still_failing) = partition_results(retry_results, lang_name, cfg.route, cfg.hash); + let (new_outcomes, still_failing) = partition_results(retry_results, lang_name, cfg.route, cfg.hash)?; if new_outcomes.is_empty() && !still_failing.is_empty() { eprintln!( @@ -827,6 +899,8 @@ pub async fn run_selected_for_model_async_for_lang(cfg: &BenchRunContext<'_>) -> ); } + ensure_server_healthy(cfg.host.as_deref()).await?; + if cfg.dry_run { let _ = match maybe_generate_analysis(cfg, &outcomes).await { Ok(text) => text, @@ -886,7 +960,8 @@ pub async fn build_goldens_only_for_lang( bench_root: &Path, lang: Lang, selectors: Option<&[String]>, -) -> Result<()> { + golden_scope: &str, +) -> Result> { let tasks = if let Some(sels) = selectors { let wanted: HashSet = sels.iter().map(|s| normalize_task_selector(s)).collect::>()?; let all = discover_tasks(bench_root)?; @@ -922,18 +997,59 @@ pub async fn build_goldens_only_for_lang( _ => bench_concurrency(), }; - stream::iter(tasks.into_iter().map(|task| { + let databases = stream::iter(tasks.into_iter().map(|task| { let runner = &runner; let host_clone = host.clone(); async move { let category = category_slug(&task.root); let task_id = task_slug(&task.root); - let golden_db = sanitize_db_name(&format!("{}-{}-golden", category, task_id)); + let golden_db = golden_db_name(&category, &task_id, golden_scope); let golden_src_text = load_golden_source(&task, lang)?; + let migration_setup = load_migration_setup_source(&task, lang)?; println!("→ [{}] build golden {} {}", lang_name, category, task_id); - runner - .publish_golden_only(lang, &category, &task_id, &golden_src_text, golden_db, host_clone) - .await + if let Some(setup_source) = migration_setup { + let setup_database = runner + .publish_golden_only(PublishParams { + lang, + category: &category, + task_id: &task_id, + route_tag: golden_scope, + source_text: &setup_source, + db_name: golden_db.clone(), + host: host_clone.clone(), + clear_database: true, + }) + .await?; + call_reducer_json_out(&golden_db, "seed", &[], Some(host_clone.as_deref().unwrap_or("local"))) + .map_err(anyhow::Error::msg)?; + let database = runner + .publish_golden_only(PublishParams { + lang, + category: &category, + task_id: &task_id, + route_tag: golden_scope, + source_text: &golden_src_text, + db_name: golden_db, + host: host_clone, + clear_database: false, + }) + .await?; + setup_database.relinquish(); + Ok(database) + } else { + runner + .publish_golden_only(PublishParams { + lang, + category: &category, + task_id: &task_id, + route_tag: golden_scope, + source_text: &golden_src_text, + db_name: golden_db, + host: host_clone, + clear_database: true, + }) + .await + } } })) .buffer_unordered(buf) @@ -943,6 +1059,140 @@ pub async fn build_goldens_only_for_lang( .collect::>>()?; println!("✓ [{}] goldens build/publish: complete", lang_name); + Ok(databases) +} + +/// Build each selected golden, publish the same source as a stand-in candidate, +/// and run the task's complete scorer set against the pair. This proves more +/// than compilation: the golden must also satisfy every behavior and parity +/// assertion used to grade model output. +pub async fn validate_goldens_for_lang( + host: Option, + bench_root: &Path, + lang: Lang, + selectors: Option<&[String]>, +) -> Result<()> { + let route_tag = "golden-validation"; + let golden_databases = build_goldens_only_for_lang(host.clone(), bench_root, lang, selectors, route_tag).await?; + + let tasks = if let Some(sels) = selectors { + let wanted: HashSet = sels.iter().map(|s| normalize_task_selector(s)).collect::>()?; + let filtered: Vec = discover_tasks(bench_root)? + .into_iter() + .filter(|task| { + let name = task.root.file_name().and_then(|name| name.to_str()).unwrap_or(""); + wanted.iter().any(|wanted| name.starts_with(wanted)) + }) + .collect(); + if filtered.is_empty() { + bail!("no tasks matched {:?}", wanted); + } + filtered + } else { + discover_tasks(bench_root)? + }; + + let runner = TaskRunner::new( + PathBuf::from(bench_root), + SpacetimeRustPublisher, + DotnetPublisher, + TypeScriptPublisher, + ); + let lang_name = lang.as_str(); + let host_url = host.as_deref().unwrap_or("local"); + let mut failures = Vec::new(); + + // Scorers can mutate both databases, so validate one task at a time and + // preserve the scorer ordering declared by its spec. + for task in tasks { + let category = category_slug(&task.root); + let task_id = task_slug(&task.root); + let candidate_db = sanitize_db_name(&format!("{}-{}-{}-llm", category, task_id, route_tag)); + let golden_src_text = load_golden_source(&task, lang)?; + let migration_setup = load_migration_setup_source(&task, lang)?; + + println!("→ [{}] validate golden {} {}", lang_name, category, task_id); + let candidate_database = if let Some(setup_source) = migration_setup { + let setup_database = runner + .publish( + PublishParams { + lang, + category: &category, + task_id: &task_id, + route_tag, + source_text: &setup_source, + db_name: candidate_db.clone(), + host: host.clone(), + clear_database: true, + }, + "golden-validation-setup", + ) + .await?; + call_reducer_json_out(&candidate_db, "seed", &[], Some(host_url)).map_err(anyhow::Error::msg)?; + let database = runner + .publish( + PublishParams { + lang, + category: &category, + task_id: &task_id, + route_tag, + source_text: &golden_src_text, + db_name: candidate_db, + host: host.clone(), + clear_database: false, + }, + "golden-validation", + ) + .await?; + setup_database.relinquish(); + database + } else { + runner + .publish( + PublishParams { + lang, + category: &category, + task_id: &task_id, + route_tag, + source_text: &golden_src_text, + db_name: candidate_db, + host: host.clone(), + clear_database: true, + }, + "golden-validation", + ) + .await? + }; + + let ctor = resolve_by_path(&task.root)?; + let spec = ctor(); + for scorer in spec.scorers_for(lang, route_tag, host_url) { + let details = scorer.score(&golden_src_text); + if !details.pass { + failures.push(format!( + "{}/{} [{}]: {}", + category, + task_id, + scorer.id(), + serde_json::to_string(&details.notes).unwrap_or_else(|_| "".to_string()) + )); + } + } + delete_database_async(candidate_database).await?; + } + + delete_databases(golden_databases).await?; + + if !failures.is_empty() { + bail!( + "[{}] golden validation failed ({} scorer failures):\n{}", + lang_name, + failures.len(), + failures.join("\n") + ); + } + + println!("✓ [{}] goldens behavior/scorer validation: complete", lang_name); Ok(()) } @@ -1044,6 +1294,21 @@ fn load_golden_source(task: &TaskPaths, lang: Lang) -> Result { } } +fn load_migration_setup_source(task: &TaskPaths, lang: Lang) -> Result> { + let file = match lang { + Lang::Rust => "rust.rs", + Lang::CSharp => "csharp.cs", + Lang::TypeScript => "typescript.ts", + }; + let path = task.root.join("setup").join(file); + if !path.is_file() { + return Ok(None); + } + fs::read_to_string(&path) + .with_context(|| format!("read {}", path.display())) + .map(Some) +} + // "1" | "01" | "001" | "t_001" -> "t_001" // "t_000_empty_reducers" | "t_001_basic_tables" -> accepted as-is (full task dir name) fn normalize_task_selector(raw: &str) -> Result { diff --git a/tools/xtask-llm-benchmark/src/bench/templates.rs b/tools/xtask-llm-benchmark/src/bench/templates.rs index 35176de8200..bdb973b97e3 100644 --- a/tools/xtask-llm-benchmark/src/bench/templates.rs +++ b/tools/xtask-llm-benchmark/src/bench/templates.rs @@ -131,11 +131,11 @@ fn inject_rust(root: &Path, llm_code: &str) -> anyhow::Result<()> { if !sdk_path.is_dir() { bail!("local Rust SDK not found at {}", sdk_path.display()); } - let replacement = format!(r#"spacetimedb = {{ path = "{}" }}"#, relative); + let replacement = format!(r#"spacetimedb = {{ path = "{}", features = ["unstable"] }}"#, relative); let cargo_toml = root.join("Cargo.toml"); let mut toml = fs::read_to_string(&cargo_toml).with_context(|| format!("read {}", cargo_toml.display()))?; toml = toml.replace( - "spacetimedb = { path = \"../../../../../../sdks/rust/\" }", + "spacetimedb = { path = \"../../../../../../sdks/rust/\", features = [\"unstable\"] }", &replacement, ); fs::write(&cargo_toml, toml).with_context(|| format!("write {}", cargo_toml.display()))?; @@ -235,6 +235,7 @@ fn write_csharp_nuget_config(root: &Path) -> Result<()> { + @@ -244,6 +245,10 @@ fn write_csharp_nuget_config(root: &Path) -> Result<()> { + + + + diff --git a/tools/xtask-llm-benchmark/src/bench/types.rs b/tools/xtask-llm-benchmark/src/bench/types.rs index e54df0d4902..c99ca37ccd5 100644 --- a/tools/xtask-llm-benchmark/src/bench/types.rs +++ b/tools/xtask-llm-benchmark/src/bench/types.rs @@ -47,6 +47,7 @@ pub struct PublishParams<'a> { pub source_text: &'a str, pub db_name: String, pub host: Option, + pub clear_database: bool, } #[derive(Debug, Serialize, Deserialize, Clone)] @@ -134,6 +135,8 @@ pub struct RouteRun { pub enum RunOneError { #[error("{msg}")] WithOutput { msg: String, llm_output: String }, + #[error("{msg}")] + Infrastructure { msg: String, llm_output: Option }, #[error(transparent)] Other(#[from] anyhow::Error), } diff --git a/tools/xtask-llm-benchmark/src/bench/utils.rs b/tools/xtask-llm-benchmark/src/bench/utils.rs index 6e28315e4f6..4d4ab2f02c3 100644 --- a/tools/xtask-llm-benchmark/src/bench/utils.rs +++ b/tools/xtask-llm-benchmark/src/bench/utils.rs @@ -45,6 +45,14 @@ pub fn sanitize_db_name(raw: &str) -> String { out } +pub fn run_scope_tag(mode: &str, vendor: &str, api_model: &str) -> String { + sanitize_db_name(&format!("{mode}-{vendor}-{api_model}")) +} + +pub fn golden_db_name(category: &str, task: &str, scope: &str) -> String { + sanitize_db_name(&format!("{category}-{task}-{scope}-golden")) +} + pub fn work_server_dir_scoped(category: &str, task: &str, lang: &str, phase: &str, route_tag: &str) -> PathBuf { let target = env::var("CARGO_TARGET_DIR").unwrap_or_else(|_| "target".into()); Path::new(&target) @@ -137,3 +145,22 @@ pub fn fmt_dur(d: Duration) -> String { format!("{}m {:.1}s", m, s) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn run_scope_separates_modes_and_models() { + let guidelines = run_scope_tag("guidelines", "openai", "gpt-5.4-mini"); + let docs = run_scope_tag("docs", "openai", "gpt-5.4-mini"); + let other_model = run_scope_tag("guidelines", "openai", "gpt-5.5"); + + assert_ne!(guidelines, docs); + assert_ne!(guidelines, other_model); + assert_ne!( + golden_db_name("views", "t_067", &guidelines), + golden_db_name("views", "t_067", &docs) + ); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/answers/csharp.cs new file mode 100644 index 00000000000..c936d35d44d --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/answers/csharp.cs @@ -0,0 +1,26 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "SecretNote")] + public partial struct SecretNote + { + [PrimaryKey] public ulong Id; + [SpacetimeDB.Index.BTree] public Identity Owner; + public string Title; + public string SecretBody; + } + + [SpacetimeDB.Type] + public partial struct SafeNote { public ulong Id; public string Title; } + + [Reducer] + public static void SeedPrivateNote(ReducerContext ctx) => ctx.Db.SecretNote.Insert(new SecretNote + { + Id = 1, Owner = ctx.Sender, Title = "Visible title", SecretBody = "never expose this", + }); + + [SpacetimeDB.View(Accessor = "MySafeNote", Public = true)] + public static IEnumerable MySafeNote(ViewContext ctx) => + ctx.Db.SecretNote.Owner.Filter(ctx.Sender).Select(note => new SafeNote { Id = note.Id, Title = note.Title }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/answers/rust.rs new file mode 100644 index 00000000000..e1be4fa8325 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/answers/rust.rs @@ -0,0 +1,40 @@ +use spacetimedb::{reducer, table, view, Identity, ReducerContext, SpacetimeType, Table, ViewContext}; + +#[table(accessor = secret_note)] +pub struct SecretNote { + #[primary_key] + pub id: u64, + #[index(btree)] + pub owner: Identity, + pub title: String, + pub secret_body: String, +} + +#[derive(SpacetimeType)] +pub struct SafeNote { + pub id: u64, + pub title: String, +} + +#[reducer] +pub fn seed_private_note(ctx: &ReducerContext) { + ctx.db.secret_note().insert(SecretNote { + id: 1, + owner: ctx.sender(), + title: "Visible title".into(), + secret_body: "never expose this".into(), + }); +} + +#[view(accessor = my_safe_note, public)] +pub fn my_safe_note(ctx: &ViewContext) -> Vec { + ctx.db + .secret_note() + .owner() + .filter(ctx.sender()) + .map(|note| SafeNote { + id: note.id, + title: note.title, + }) + .collect() +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/answers/typescript.ts new file mode 100644 index 00000000000..89ba18d21a7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/answers/typescript.ts @@ -0,0 +1,20 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const secretNote = table({ name: 'secret_note' }, { + id: t.u64().primaryKey(), + owner: t.identity().index('btree'), + title: t.string(), + secretBody: t.string(), +}); +const SafeNote = t.row('SafeNoteRow', { id: t.u64(), title: t.string() }); +const spacetimedb = schema({ secretNote }); +export default spacetimedb; + +export const seed_private_note = spacetimedb.reducer(ctx => { + ctx.db.secretNote.insert({ id: 1n, owner: ctx.sender, title: 'Visible title', secretBody: 'never expose this' }); +}); + +export const my_safe_note = spacetimedb.view( + { name: 'my_safe_note', public: true }, t.array(SafeNote), + ctx => Array.from(ctx.db.secretNote.owner.filter(ctx.sender)).map(note => ({ id: note.id, title: note.title })) +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/spec.rs new file mode 100644 index 00000000000..0dc215476b1 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/spec.rs @@ -0,0 +1,24 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_data_parity_scorer}; +use crate::eval::{table_name, BenchmarkSpec, ReducerDataParityConfig}; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + let view = table_name("my_safe_note", lang); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "seed_private_note".into(), + args: vec![], + select_query: format!("SELECT * FROM {view}"), + id_str: "caller_safe_projection", + collapse_ws: true, + timeout: Duration::from_secs(10), + }, + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/csharp.txt new file mode 100644 index 00000000000..2282d2a24fc --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/csharp.txt @@ -0,0 +1,25 @@ +Write a SpacetimeDB backend module in C# that exposes a safe per-user projection of private rows. + +TYPES +- SafeNote + - Id: ulong + - Title: string + +TABLE +- SecretNote (private, accessor SecretNote) + - Fields: + - Id: ulong (primary key) + - Owner: Identity (btree index) + - Title: string + - SecretBody: string + +REDUCERS +- SeedPrivateNote() + - Insert Id 1 for the caller + - Set Title to "Visible title" + - Set SecretBody to "never expose this" + +VIEW +- MySafeNote (per-user, public) + - Return IEnumerable for only the caller + - Never expose Owner or SecretBody diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/rust.txt new file mode 100644 index 00000000000..34252c59cd8 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/rust.txt @@ -0,0 +1,25 @@ +Write a SpacetimeDB backend module in Rust that exposes a safe per-user projection of private rows. + +TYPES +- SafeNote + - id: u64 + - title: String + +TABLE +- secret_note (private), struct SecretNote + - Fields: + - id: u64 (primary key) + - owner: Identity (btree index) + - title: String + - secret_body: String + +REDUCERS +- seed_private_note() + - Insert id 1 for the caller + - Set title to "Visible title" + - Set secret_body to "never expose this" + +VIEW +- my_safe_note (per-user, public) + - Return Vec for only the caller + - Never expose owner or secret_body diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/typescript.txt new file mode 100644 index 00000000000..ca31bd4cc7d --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_068_secure_projection/tasks/typescript.txt @@ -0,0 +1,25 @@ +Write a SpacetimeDB backend module in TypeScript that exposes a safe per-user projection of private rows. + +TYPES +- SafeNote + - id: u64 + - title: string + +TABLE +- secret_note (private) + - Fields: + - id: u64 (primary key) + - owner: Identity (btree index) + - title: string + - secretBody: string + +REDUCERS +- seed_private_note() + - Insert id 1 for the caller + - Set title to "Visible title" + - Set secretBody to "never expose this" + +VIEW +- my_safe_note (per-user, public) + - Return an array of SafeNote for only the caller + - Never expose owner or secretBody diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/answers/csharp.cs new file mode 100644 index 00000000000..fc2b526804a --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/answers/csharp.cs @@ -0,0 +1,20 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "UserRecord", Public = true)] + public partial struct UserRecord + { + [PrimaryKey] public Identity Identity; + public string Name; + } + + [ClientVisibilityFilter] + public static readonly Filter UserRecordFilter = new Filter.Sql( + "SELECT * FROM UserRecord WHERE Identity = :sender" + ); + + [Reducer] + public static void RegisterSelf(ReducerContext ctx, string name) => + ctx.Db.UserRecord.Insert(new UserRecord { Identity = ctx.Sender, Name = name }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/answers/rust.rs new file mode 100644 index 00000000000..563b979cd17 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/answers/rust.rs @@ -0,0 +1,19 @@ +use spacetimedb::{Filter, Identity, ReducerContext, Table}; + +#[spacetimedb::table(accessor = user_record, public)] +pub struct UserRecord { + #[primary_key] + identity: Identity, + name: String, +} + +#[spacetimedb::client_visibility_filter] +const USER_RECORD_FILTER: Filter = Filter::Sql("SELECT * FROM user_record WHERE identity = :sender"); + +#[spacetimedb::reducer] +pub fn register_self(ctx: &ReducerContext, name: String) { + ctx.db.user_record().insert(UserRecord { + identity: ctx.sender(), + name, + }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/answers/typescript.ts new file mode 100644 index 00000000000..aae8e7058cd --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/answers/typescript.ts @@ -0,0 +1,17 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const userRecord = table( + { name: 'user_record', public: true }, + { identity: t.identity().primaryKey(), name: t.string() } +); +const spacetimedb = schema({ userRecord }); +export default spacetimedb; + +export const userRecordFilter = spacetimedb.clientVisibilityFilter.sql( + 'SELECT * FROM user_record WHERE identity = :sender' +); + +export const register_self = spacetimedb.reducer( + { name: t.string() }, + (ctx, { name }) => ctx.db.userRecord.insert({ identity: ctx.sender, name }) +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/spec.rs new file mode 100644 index 00000000000..ec545ee4033 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/spec.rs @@ -0,0 +1,27 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_data_parity_scorer}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec, ReducerDataParityConfig}; +use serde_json::json; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + let table = table_name("user_record", lang); + let identity = ident("identity", casing_for_lang(lang)); + let name = ident("name", casing_for_lang(lang)); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "register_self".into(), + args: vec![json!("caller")], + select_query: format!("SELECT {identity}, {name} FROM {table}"), + collapse_ws: true, + timeout: Duration::from_secs(10), + id_str: "caller_sees_own_row", + }, + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/csharp.txt new file mode 100644 index 00000000000..4a5868806d9 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/csharp.txt @@ -0,0 +1,15 @@ +Write a SpacetimeDB backend module in C# that applies row-level visibility to a public table. + +TABLE +- UserRecord (public, accessor UserRecord) + - Fields: + - Identity: Identity (primary key) + - Name: string + +CLIENT VISIBILITY FILTER +- Restrict UserRecord so each caller can see only rows whose Identity matches the caller + +REDUCERS +- RegisterSelf(name: string) + - Insert a UserRecord using ctx.Sender + - Do not accept an identity argument diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/rust.txt new file mode 100644 index 00000000000..a87cf289dda --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/rust.txt @@ -0,0 +1,15 @@ +Write a SpacetimeDB backend module in Rust that applies row-level visibility to a public table. + +TABLE +- user_record (public), struct UserRecord + - Fields: + - identity: Identity (primary key) + - name: String + +CLIENT VISIBILITY FILTER +- Restrict user_record so each caller can see only rows whose identity matches the caller + +REDUCERS +- register_self(name: String) + - Insert a UserRecord using ctx.sender() + - Do not accept an identity argument diff --git a/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/typescript.txt new file mode 100644 index 00000000000..5a6ea23495e --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/auth/t_083_row_level_security/tasks/typescript.txt @@ -0,0 +1,15 @@ +Write a SpacetimeDB backend module in TypeScript that applies row-level visibility to a public table. + +TABLE +- user_record (public) + - Fields: + - identity: Identity (primary key) + - name: string + +CLIENT VISIBILITY FILTER +- Restrict user_record so each caller can see only rows whose identity matches the caller + +REDUCERS +- register_self(name: string) + - Insert a user_record using ctx.sender + - Do not accept an identity argument diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_000_empty_reducers/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_000_empty_reducers/tasks/csharp.txt index 0821f3321a6..0f773c9268b 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_000_empty_reducers/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_000_empty_reducers/tasks/csharp.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in C# that defines a table and these five empty reducers. TABLES -- EmptyTable +- EmptyTable (private) - Struct: EmptyTable - Fields: - Id: int (primary key) @@ -11,4 +11,4 @@ REDUCERS - EmptyReducer_WithInt: (int count), returns void, empty body - EmptyReducer_WithString: (string name), returns void, empty body - EmptyReducer_WithTwoArgs: (int count, string name), returns void, empty body -- EmptyReducer_WithThreeArgs: (bool active, float ratio, string label), returns void, empty body \ No newline at end of file +- EmptyReducer_WithThreeArgs: (bool active, float ratio, string label), returns void, empty body diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_000_empty_reducers/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_000_empty_reducers/tasks/rust.txt index 7d42898b222..c512823bfbf 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_000_empty_reducers/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_000_empty_reducers/tasks/rust.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in Rust that defines a table and these five empty reducers. TABLES -- EmptyTable +- EmptyTable (private) - Struct: EmptyTable - Fields: - id: i32 (primary key) @@ -11,4 +11,4 @@ REDUCERS - empty_reducer_with_int: (count: i32), returns (), empty body - empty_reducer_with_string: (name: String), returns (), empty body - empty_reducer_with_two_args: (count: i32, name: String), returns (), empty body -- empty_reducer_with_three_args: (active: bool, ratio: f32, label: String), returns (), empty body \ No newline at end of file +- empty_reducer_with_three_args: (active: bool, ratio: f32, label: String), returns (), empty body diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_000_empty_reducers/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_000_empty_reducers/tasks/typescript.txt index 6580e17302f..5be728388b3 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_000_empty_reducers/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_000_empty_reducers/tasks/typescript.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in TypeScript that defines a table and these five empty reducers. TABLES -- empty_table +- empty_table (private) - Fields: - id: number (i32, primary key) @@ -10,4 +10,4 @@ REDUCERS - emptyReducerWithInt: count:number (i32) - emptyReducerWithString: name:string - emptyReducerWithTwoArgs: count:number (i32), name:string -- emptyReducerWithThreeArgs: active:boolean, ratio:number (f32), label:string \ No newline at end of file +- emptyReducerWithThreeArgs: active:boolean, ratio:number (f32), label:string diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_001_basic_tables/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_001_basic_tables/tasks/csharp.txt index 2dca848cb08..1409089ad36 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_001_basic_tables/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_001_basic_tables/tasks/csharp.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in C# that defines three tables with basic columns. TABLES -- User +- User (private) - Struct: User - Fields: - Id: int (primary key) @@ -9,7 +9,7 @@ TABLES - Age: int - Active: bool -- Product +- Product (private) - Struct: Product - Fields: - Id: int (primary key) @@ -17,7 +17,7 @@ TABLES - Price: float - InStock: bool -- Note +- Note (private) - Struct: Note - Fields: - Id: int (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_001_basic_tables/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_001_basic_tables/tasks/rust.txt index 02d385eb6c9..ab13a01581e 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_001_basic_tables/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_001_basic_tables/tasks/rust.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in Rust that defines three tables with basic columns. TABLES -- user +- user (private) - Struct: User - Fields: - id: i32 (primary key) @@ -9,7 +9,7 @@ TABLES - age: i32 - active: bool -- product +- product (private) - Struct: Product - Fields: - id: i32 (primary key) @@ -17,7 +17,7 @@ TABLES - price: f32 - in_stock: bool -- note +- note (private) - Struct: Note - Fields: - id: i32 (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_001_basic_tables/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_001_basic_tables/tasks/typescript.txt index f8f4666b502..af6e3e1128f 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_001_basic_tables/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_001_basic_tables/tasks/typescript.txt @@ -1,23 +1,23 @@ Write a SpacetimeDB backend module in TypeScript that defines three tables with basic columns. TABLES -- user +- user (private) - Fields: - id: number (i32, primary key) - name: string - age: number (i32) - active: boolean -- product +- product (private) - Fields: - id: number (i32, primary key) - title: string - price: number (f32) - inStock: boolean -- note +- note (private) - Fields: - id: number (i32, primary key) - body: string - rating: bigint (i64) - - pinned: boolean \ No newline at end of file + - pinned: boolean diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/csharp.txt index e00dca47a3c..add0113a306 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/csharp.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in C# that defines one table and two reducers for client lifecycle events. TABLE -- Event +- Event (private) - Struct: Event - Fields: - Id: int (primary key, auto-increment) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/rust.txt index a2fea3f3092..e4dcbf0da68 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/rust.txt @@ -1,10 +1,10 @@ Write a SpacetimeDB backend module in Rust that defines one table and two reducers for client lifecycle events. TABLE -- event +- event (private) - Struct: Event - Fields: - - id: i32 (primary key, auto-increment) + - id: u64 (primary key, auto-increment) - kind: String REDUCERS diff --git a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/typescript.txt index 21b4a04b2c4..a9a80b3415d 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/basics/t_010_connect/tasks/typescript.txt @@ -1,11 +1,11 @@ Write a SpacetimeDB backend module in TypeScript that defines one table and two reducers for client lifecycle events. TABLE -- event +- event (private) - Fields: - id: bigint (u64, primary key, auto-increment) - kind: string LIFECYCLE HOOKS (use spacetimedb.clientConnected / spacetimedb.clientDisconnected) - onConnect: insert event with kind="connected" -- onDisconnect: insert event with kind="disconnected" \ No newline at end of file +- onDisconnect: insert event with kind="disconnected" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/csharp.txt index 04c9a2a10b9..371545e5e4b 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/csharp.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in C# that defines a table with optional (nullable) fields and a reducer that inserts rows with and without optional values. TABLES -- Player +- Player (private) - Struct: Player - Fields: - Id: ulong (primary key, AutoInc) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/rust.txt index ac9b1d00a50..165c6d3426f 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/rust.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in Rust that defines a table with optional (nullable) fields and a reducer that inserts rows with and without optional values. TABLES -- player +- player (private) - Struct: Player - Fields: - id: u64 (primary key, auto_inc) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/typescript.txt index 48712f8abea..9960fb64f79 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_025_optional_fields/tasks/typescript.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in TypeScript that defines a table with optional (nullable) fields and a reducer that inserts rows with and without optional values. TABLES -- player +- player (private) - Fields: - id: number (u64, primary key, autoInc) - name: string diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/csharp.txt index c3d32fd1d12..9ed82d8e504 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/csharp.txt @@ -1,13 +1,13 @@ Write a SpacetimeDB backend module in C# that defines an Author table and a Post table with a foreign key, and a reducer that deletes an author along with all their posts. TABLES -- Author +- Author (private) - Struct: Author - Fields: - Id: ulong (primary key, AutoInc) - Name: string -- Post +- Post (private) - Struct: Post - Fields: - Id: ulong (primary key, AutoInc) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/rust.txt index 8a3c25ba08a..14da8c93bc5 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/rust.txt @@ -1,13 +1,13 @@ Write a SpacetimeDB backend module in Rust that defines an author table and a post table with a foreign key, and a reducer that deletes an author along with all their posts. TABLES -- author +- author (private) - Struct: Author - Fields: - id: u64 (primary key, auto_inc) - name: String -- post +- post (private) - Struct: Post - Fields: - id: u64 (primary key, auto_inc) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/typescript.txt index 5cb6d02e7b6..9436daa1913 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_028_cascade_delete/tasks/typescript.txt @@ -1,12 +1,12 @@ Write a SpacetimeDB backend module in TypeScript that defines an author table and a post table with a foreign key, and a reducer that deletes an author along with all their posts. TABLES -- author +- author (private) - Fields: - id: number (u64, primary key, autoInc) - name: string -- post +- post (private) - Fields: - id: number (u64, primary key, autoInc) - authorId: number (u64, index btree) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/csharp.txt index 2f761d52a22..8a84335cd07 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/csharp.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in C# that defines an Order table and a CategoryStats table, with a reducer that computes aggregate statistics from filtered orders. TABLES -- Order +- Order (private) - Struct: Order - Fields: - Id: ulong (primary key, AutoInc) @@ -9,7 +9,7 @@ TABLES - Amount: ulong - Fulfilled: bool -- CategoryStats +- CategoryStats (private) - Struct: CategoryStats - Fields: - Category: string (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/rust.txt index ebe29cf6a76..35cb98d38aa 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/rust.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in Rust that defines an order table and a stats table, with a reducer that computes aggregate statistics from filtered orders. TABLES -- order +- order (private) - Struct: Order - Fields: - id: u64 (primary key, auto_inc) @@ -9,7 +9,7 @@ TABLES - amount: u64 - fulfilled: bool -- category_stats +- category_stats (private) - Struct: CategoryStats - Fields: - category: String (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/typescript.txt index aad4fbc62d5..c3b1d7cb066 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_029_filter_and_aggregate/tasks/typescript.txt @@ -1,14 +1,14 @@ Write a SpacetimeDB backend module in TypeScript that defines an order table and a stats table, with a reducer that computes aggregate statistics from filtered orders. TABLES -- order +- order (private) - Fields: - id: number (u64, primary key, autoInc) - category: string (index btree) - amount: number (u64) - fulfilled: boolean -- category_stats +- category_stats (private) - Fields: - category: string (primary key) - totalAmount: number (u64) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/csharp.txt index 6dfd47c6580..4551a920adf 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/csharp.txt @@ -1,13 +1,13 @@ Write a SpacetimeDB backend module in C# that defines a Customer table, an Order table with a foreign key to Customer, and a denormalized result table. Include a reducer that joins the two tables and writes results. TABLES -- Customer +- Customer (private) - Struct: Customer - Fields: - Id: ulong (primary key, AutoInc) - Name: string -- Order +- Order (private) - Struct: Order - Fields: - Id: ulong (primary key, AutoInc) @@ -15,7 +15,7 @@ TABLES - Product: string - Amount: uint -- OrderDetail +- OrderDetail (private) - Struct: OrderDetail - Fields: - OrderId: ulong (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/rust.txt index 21170c6e3c1..dd5a5c10564 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/rust.txt @@ -1,13 +1,13 @@ Write a SpacetimeDB backend module in Rust that defines a customer table, an order table with a foreign key to customer, and a denormalized result table. Include a reducer that joins the two tables and writes results. TABLES -- customer +- customer (private) - Struct: Customer - Fields: - id: u64 (primary key, auto_inc) - name: String -- order +- order (private) - Struct: Order - Fields: - id: u64 (primary key, auto_inc) @@ -15,7 +15,7 @@ TABLES - product: String - amount: u32 -- order_detail +- order_detail (private) - Struct: OrderDetail - Fields: - order_id: u64 (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/typescript.txt index 589938a7566..3cf12d4233d 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_030_two_table_join/tasks/typescript.txt @@ -1,19 +1,19 @@ Write a SpacetimeDB backend module in TypeScript that defines a customer table, an order table with a foreign key to customer, and a denormalized result table. Include a reducer that joins the two tables and writes results. TABLES -- customer +- customer (private) - Fields: - id: number (u64, primary key, autoInc) - name: string -- order +- order (private) - Fields: - id: number (u64, primary key, autoInc) - customerId: number (u64, index btree) - product: string - amount: number (u32) -- order_detail +- order_detail (private) - Fields: - orderId: number (u64, primary key) - customerName: string diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/csharp.txt index 433a4df53aa..f71c512fbe3 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/csharp.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in C# that defines a table with a unique constraint on a non-primary-key column and a reducer that inserts rows. TABLES -- Account +- Account (private) - Struct: Account - Fields: - Id: ulong (primary key, AutoInc) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/rust.txt index 2482f76d578..ab63be2c3a1 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/rust.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in Rust that defines a table with a unique constraint on a non-primary-key column and a reducer that inserts rows. TABLES -- account +- account (private) - Struct: Account - Fields: - id: u64 (primary key, auto_inc) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/typescript.txt index b1e90089c63..9f0d668afde 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/data_modeling/t_031_unique_constraint/tasks/typescript.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in TypeScript that defines a table with a unique constraint on a non-primary-key column and a reducer that inserts rows. TABLES -- account +- account (private) - Fields: - id: number (u64, primary key, autoInc) - email: string (unique) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/csharp.cs new file mode 100644 index 00000000000..ef512f2ec75 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/csharp.cs @@ -0,0 +1,42 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "MaterializedState", Public = true)] + public partial struct MaterializedState + { + [PrimaryKey] public ulong Id; + public string Status; + public ulong Version; + public Timestamp RefreshedAt; + } + + [Table(Accessor = "RefreshJob", Scheduled = nameof(RefreshMaterialized), ScheduledAt = nameof(RefreshJob.ScheduledAt))] + public partial struct RefreshJob + { + [PrimaryKey, AutoInc] public ulong ScheduledId; + public ScheduleAt ScheduledAt; + public ulong StateId; + } + + [Reducer] + public static void StartRefresh(ReducerContext ctx) + { + var pending = new MaterializedState { Id = 1, Status = "pending", Version = 0, RefreshedAt = new Timestamp(0) }; + if (ctx.Db.MaterializedState.Id.Find(1) is null) ctx.Db.MaterializedState.Insert(pending); + else ctx.Db.MaterializedState.Id.Update(pending); + ctx.Db.RefreshJob.Insert(new RefreshJob { + ScheduledAt = new ScheduleAt.Time(ctx.Timestamp + new TimeDuration { Microseconds = 1_000 }), StateId = 1, + }); + } + + [Reducer] + public static void RefreshMaterialized(ReducerContext ctx, RefreshJob job) + { + var state = ctx.Db.MaterializedState.Id.Find(job.StateId) ?? throw new InvalidOperationException("materialized state missing"); + state.Status = "ready"; + state.Version = 1; + state.RefreshedAt = ctx.Timestamp; + ctx.Db.MaterializedState.Id.Update(state); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/rust.rs new file mode 100644 index 00000000000..78d44301106 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/rust.rs @@ -0,0 +1,54 @@ +use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table, Timestamp}; +use std::time::Duration; + +#[table(accessor = materialized_state, public)] +pub struct MaterializedState { + #[primary_key] + pub id: u64, + pub status: String, + pub version: u64, + pub refreshed_at: Timestamp, +} + +#[table(accessor = refresh_job, scheduled(refresh_materialized))] +pub struct RefreshJob { + #[primary_key] + #[auto_inc] + pub scheduled_id: u64, + pub scheduled_at: ScheduleAt, + pub state_id: u64, +} + +#[reducer] +pub fn start_refresh(ctx: &ReducerContext) { + let pending = MaterializedState { + id: 1, + status: "pending".into(), + version: 0, + refreshed_at: Timestamp::UNIX_EPOCH, + }; + if ctx.db.materialized_state().id().find(1).is_some() { + ctx.db.materialized_state().id().update(pending); + } else { + ctx.db.materialized_state().insert(pending); + } + ctx.db.refresh_job().insert(RefreshJob { + scheduled_id: 0, + scheduled_at: ScheduleAt::Time(ctx.timestamp + Duration::from_millis(1)), + state_id: 1, + }); +} + +#[reducer] +pub fn refresh_materialized(ctx: &ReducerContext, job: RefreshJob) { + let mut state = ctx + .db + .materialized_state() + .id() + .find(job.state_id) + .expect("materialized state missing"); + state.status = "ready".into(); + state.version = 1; + state.refreshed_at = ctx.timestamp; + ctx.db.materialized_state().id().update(state); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/typescript.ts new file mode 100644 index 00000000000..678785f8e83 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/answers/typescript.ts @@ -0,0 +1,26 @@ +import { ScheduleAt, Timestamp } from 'spacetimedb'; +import { schema, table, t } from 'spacetimedb/server'; + +const materializedState = table({ name: 'materialized_state', public: true }, { + id: t.u64().primaryKey(), status: t.string(), version: t.u64(), refreshedAt: t.timestamp(), +}); +const refreshJob = table({ name: 'refresh_job', scheduled: (): any => refreshMaterialized }, { + scheduledId: t.u64().primaryKey().autoInc(), scheduledAt: t.scheduleAt(), stateId: t.u64(), +}); +const spacetimedb = schema({ materializedState, refreshJob }); +export default spacetimedb; + +export const start_refresh = spacetimedb.reducer(ctx => { + const pending = { id: 1n, status: 'pending', version: 0n, refreshedAt: new Timestamp(0n) }; + if (ctx.db.materializedState.id.find(1n)) ctx.db.materializedState.id.update(pending); + else ctx.db.materializedState.insert(pending); + ctx.db.refreshJob.insert({ + scheduledId: 0n, scheduledAt: ScheduleAt.time(ctx.timestamp.microsSinceUnixEpoch + 1_000n), stateId: 1n, + }); +}); + +export const refreshMaterialized = spacetimedb.reducer({ job: refreshJob.rowType }, (ctx, { job }) => { + const state = ctx.db.materializedState.id.find(job.stateId); + if (!state) throw new Error('materialized state missing'); + ctx.db.materializedState.id.update({ ...state, status: 'ready', version: 1n, refreshedAt: ctx.timestamp }); +}); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/spec.rs new file mode 100644 index 00000000000..e579c919f00 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/spec.rs @@ -0,0 +1,42 @@ +use crate::eval::defaults::{ + default_schema_parity_scorers, make_eventually_sql_count_scorer, make_reducer_call_both_scorer, + make_sql_output_excludes_scorer, +}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec}; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_reducer_call_both_scorer( + host_url, + file!(), + route_tag, + "start_refresh", + vec![], + "start_refresh", + )); + let table = table_name("materialized_state", lang); + let status = ident("status", casing_for_lang(lang)); + let version = ident("version", casing_for_lang(lang)); + scorers.push(make_eventually_sql_count_scorer( + host_url, + file!(), + route_tag, + format!("SELECT COUNT(*) AS n FROM {table} WHERE {status}='ready' AND {version}=1"), + 1, + "scheduled_refresh_completes", + Duration::from_secs(10), + )); + let refreshed_at = ident("refreshed_at", casing_for_lang(lang)); + scorers.push(make_sql_output_excludes_scorer( + host_url, + file!(), + route_tag, + format!("SELECT {refreshed_at} FROM {table}"), + vec!["1970-01-01T00:00:00+00:00".into()], + "scheduled_refresh_timestamped", + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/csharp.txt new file mode 100644 index 00000000000..1c479fdef11 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/csharp.txt @@ -0,0 +1,23 @@ +Write a SpacetimeDB backend module in C# that refreshes materialized state through a scheduled reducer. + +TABLES +- MaterializedState (public, accessor MaterializedState) + - Fields: + - Id: ulong (primary key) + - Status: string + - Version: ulong + - RefreshedAt: Timestamp +- RefreshJob (private, scheduled for RefreshMaterialized, accessor RefreshJob) + - Fields, in this order: + - ScheduledId: ulong (primary key, auto-increment) + - ScheduledAt: ScheduleAt + - StateId: ulong + +REDUCERS +- StartRefresh() + - Write Id 1 with Status "pending", Version 0, and RefreshedAt at the Unix epoch + - Schedule state 1 for one millisecond later +- RefreshMaterialized(job: RefreshJob) (scheduled reducer) + - Update state 1 to Status "ready" and Version 1 + - Replace RefreshedAt with the scheduled reducer timestamp + - Complete without any further client call diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/rust.txt new file mode 100644 index 00000000000..fc567eb570a --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/rust.txt @@ -0,0 +1,23 @@ +Write a SpacetimeDB backend module in Rust that refreshes materialized state through a scheduled reducer. + +TABLES +- materialized_state (public), struct MaterializedState + - Fields: + - id: u64 (primary key) + - status: String + - version: u64 + - refreshed_at: Timestamp +- refresh_job (private, scheduled for refresh_materialized), struct RefreshJob + - Fields, in this order: + - scheduled_id: u64 (primary key, auto-increment) + - scheduled_at: ScheduleAt + - state_id: u64 + +REDUCERS +- start_refresh() + - Write id 1 with status "pending", version 0, and refreshed_at at the Unix epoch + - Schedule state 1 for one millisecond later +- refresh_materialized(job: RefreshJob) (scheduled reducer) + - Update state 1 to status "ready" and version 1 + - Replace refreshed_at with the scheduled reducer timestamp + - Complete without any further client call diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/typescript.txt new file mode 100644 index 00000000000..91ee348827d --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_069_scheduled_materialization/tasks/typescript.txt @@ -0,0 +1,23 @@ +Write a SpacetimeDB backend module in TypeScript that refreshes materialized state through a scheduled reducer. + +TABLES +- materialized_state (public) + - Fields: + - id: u64 (primary key) + - status: string + - version: u64 + - refreshedAt: Timestamp +- refresh_job (private, scheduled for refreshMaterialized) + - Fields, in this order: + - scheduledId: u64 (primary key, auto-increment) + - scheduledAt: ScheduleAt + - stateId: u64 + +REDUCERS +- start_refresh() + - Write id 1 with status "pending", version 0, and refreshedAt at the Unix epoch + - Schedule state 1 for one millisecond later +- refreshMaterialized(job: refresh_job row) (scheduled reducer) + - Update state 1 to status "ready" and version 1 + - Replace refreshedAt with the scheduled reducer timestamp + - Complete without any further client call diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/answers/csharp.cs new file mode 100644 index 00000000000..fa43d7ed84f --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/answers/csharp.cs @@ -0,0 +1,36 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "PresenceSession", Public = true)] + public partial struct PresenceSession + { + [PrimaryKey] public ConnectionId ConnectionId; + [SpacetimeDB.Index.BTree] public Identity Identity; + public Timestamp ConnectedAt; + } + + private static void AddSession(ReducerContext ctx, ConnectionId connectionId) => + ctx.Db.PresenceSession.Insert(new PresenceSession { ConnectionId = connectionId, Identity = ctx.Sender, ConnectedAt = ctx.Timestamp }); + + private static void RemoveSession(ReducerContext ctx, ConnectionId connectionId) => + ctx.Db.PresenceSession.ConnectionId.Delete(connectionId); + + [Reducer(ReducerKind.ClientConnected)] + public static void ClientConnected(ReducerContext ctx) => AddSession(ctx, ctx.ConnectionId ?? throw new InvalidOperationException("connection id missing")); + + [Reducer(ReducerKind.ClientDisconnected)] + public static void ClientDisconnected(ReducerContext ctx) => RemoveSession(ctx, ctx.ConnectionId ?? throw new InvalidOperationException("connection id missing")); + + [Reducer] + public static void ExercisePresence(ReducerContext ctx) + { + var firstBytes = new byte[16]; firstBytes[0] = 1; + var secondBytes = new byte[16]; secondBytes[0] = 2; + var first = ConnectionId.From(firstBytes) ?? throw new InvalidOperationException(); + var second = ConnectionId.From(secondBytes) ?? throw new InvalidOperationException(); + AddSession(ctx, first); + AddSession(ctx, second); + RemoveSession(ctx, first); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/answers/rust.rs new file mode 100644 index 00000000000..5121a6286d5 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/answers/rust.rs @@ -0,0 +1,41 @@ +use spacetimedb::{reducer, table, ConnectionId, Identity, ReducerContext, Table, Timestamp}; + +#[table(accessor = presence_session, public)] +pub struct PresenceSession { + #[primary_key] + pub connection_id: ConnectionId, + #[index(btree)] + pub identity: Identity, + pub connected_at: Timestamp, +} + +fn add_session(ctx: &ReducerContext, connection_id: ConnectionId) { + ctx.db.presence_session().insert(PresenceSession { + connection_id, + identity: ctx.sender(), + connected_at: ctx.timestamp, + }); +} + +fn remove_session(ctx: &ReducerContext, connection_id: ConnectionId) { + ctx.db.presence_session().connection_id().delete(connection_id); +} + +#[reducer(client_connected)] +pub fn client_connected(ctx: &ReducerContext) { + add_session(ctx, ctx.connection_id().expect("connection id missing")); +} + +#[reducer(client_disconnected)] +pub fn client_disconnected(ctx: &ReducerContext) { + remove_session(ctx, ctx.connection_id().expect("connection id missing")); +} + +#[reducer] +pub fn exercise_presence(ctx: &ReducerContext) { + let first = ConnectionId::from_u128(1); + let second = ConnectionId::from_u128(2); + add_session(ctx, first); + add_session(ctx, second); + remove_session(ctx, first); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/answers/typescript.ts new file mode 100644 index 00000000000..c833e4d48f6 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/answers/typescript.ts @@ -0,0 +1,31 @@ +import { ConnectionId } from 'spacetimedb'; +import { schema, table, t } from 'spacetimedb/server'; +import type { InferSchema, ReducerCtx } from 'spacetimedb/server'; + +const presenceSession = table({ name: 'presence_session', public: true }, { + connectionId: t.connectionId().primaryKey(), identity: t.identity().index('btree'), connectedAt: t.timestamp(), +}); +const spacetimedb = schema({ presenceSession }); +export default spacetimedb; +type Ctx = ReducerCtx>; + +function addSession(ctx: Ctx, connectionId: ConnectionId) { + ctx.db.presenceSession.insert({ connectionId, identity: ctx.sender, connectedAt: ctx.timestamp }); +} +function removeSession(ctx: Ctx, connectionId: ConnectionId) { ctx.db.presenceSession.connectionId.delete(connectionId); } + +export const clientConnected = spacetimedb.clientConnected(ctx => { + if (!ctx.connectionId) throw new Error('connection id missing'); + addSession(ctx, ctx.connectionId); +}); +export const clientDisconnected = spacetimedb.clientDisconnected(ctx => { + if (!ctx.connectionId) throw new Error('connection id missing'); + removeSession(ctx, ctx.connectionId); +}); +export const exercise_presence = spacetimedb.reducer(ctx => { + const first = new ConnectionId(1n); + const second = new ConnectionId(2n); + addSession(ctx, first); + addSession(ctx, second); + removeSession(ctx, first); +}); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/spec.rs new file mode 100644 index 00000000000..686cd2c6c00 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/spec.rs @@ -0,0 +1,31 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_call_both_scorer, make_sql_count_only_scorer}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec}; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_reducer_call_both_scorer( + host_url, + file!(), + route_tag, + "exercise_presence", + vec![], + "exercise_presence", + )); + let table = table_name("presence_session", lang); + let connection_id = ident("connection_id", casing_for_lang(lang)); + for (value, expected, id) in [(1, 0, "first_connection_removed"), (2, 1, "second_connection_retained")] { + scorers.push(make_sql_count_only_scorer( + host_url, + file!(), + route_tag, + format!("SELECT COUNT(*) AS n FROM {table} WHERE {connection_id}=0x{value:032x}"), + expected, + id, + Duration::from_secs(10), + )); + } + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt new file mode 100644 index 00000000000..c75fba666f6 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/csharp.txt @@ -0,0 +1,23 @@ +Write a SpacetimeDB backend module in C# that tracks presence separately for each connection. + +TABLE +- PresenceSession (public, accessor PresenceSession) + - Fields: + - ConnectionId: ConnectionId (primary key) + - Identity: Identity (btree index) + - ConnectedAt: Timestamp + +LIFECYCLE HOOKS +- ClientConnected + - Export with this exact method name + - Insert one row for the connected connection +- ClientDisconnected + - Export with this exact method name + - Delete only the disconnected connection's row + +REDUCERS +- ExercisePresence() + - Insert synthetic connection IDs 1 and 2 for the caller + - Use the 128-bit numeric values 1 and 2 + - Remove connection ID 1 + - Leave connection ID 2 intact diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/rust.txt new file mode 100644 index 00000000000..19f4e53dc7b --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/rust.txt @@ -0,0 +1,23 @@ +Write a SpacetimeDB backend module in Rust that tracks presence separately for each connection. + +TABLE +- presence_session (public), struct PresenceSession + - Fields: + - connection_id: ConnectionId (primary key) + - identity: Identity (btree index) + - connected_at: Timestamp + +LIFECYCLE HOOKS +- client_connected + - Export with this exact identifier + - Insert one row for the connected connection +- client_disconnected + - Export with this exact identifier + - Delete only the disconnected connection's row + +REDUCERS +- exercise_presence() + - Insert synthetic connection IDs 1 and 2 for the caller + - Use the 128-bit numeric values 1 and 2 + - Remove connection ID 1 + - Leave connection ID 2 intact diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/typescript.txt new file mode 100644 index 00000000000..53332175766 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_070_connection_scoped_presence/tasks/typescript.txt @@ -0,0 +1,23 @@ +Write a SpacetimeDB backend module in TypeScript that tracks presence separately for each connection. + +TABLE +- presence_session (public) + - Fields: + - connectionId: ConnectionId (primary key) + - identity: Identity (btree index) + - connectedAt: Timestamp + +LIFECYCLE HOOKS +- clientConnected + - Export with this exact identifier + - Insert one row for the connected connection +- clientDisconnected + - Export with this exact identifier + - Delete only the disconnected connection's row + +REDUCERS +- exercise_presence() + - Insert synthetic connection IDs 1 and 2 for the caller + - Use the 128-bit numeric values 1 and 2 + - Remove connection ID 1 + - Leave connection ID 2 intact diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/answers/csharp.cs new file mode 100644 index 00000000000..b786613fae7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/answers/csharp.cs @@ -0,0 +1,32 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "JobResult", Public = true)] + public partial struct JobResult { [PrimaryKey] public ulong Id; public string Status; } + + [Table(Accessor = "PrivateJob", Scheduled = nameof(RunPrivateJob), ScheduledAt = nameof(PrivateJob.ScheduledAt))] + public partial struct PrivateJob + { + [PrimaryKey, AutoInc] public ulong ScheduledId; + public ScheduleAt ScheduledAt; + public ulong ResultId; + } + + [Reducer] + public static void EnqueuePrivate(ReducerContext ctx, ulong id) + { + ctx.Db.JobResult.Insert(new JobResult { Id = id, Status = "queued" }); + ctx.Db.PrivateJob.Insert(new PrivateJob { + ScheduledAt = new ScheduleAt.Time(ctx.Timestamp + new TimeDuration { Microseconds = 1_000 }), ResultId = id, + }); + } + + [Reducer] + public static void RunPrivateJob(ReducerContext ctx, PrivateJob job) + { + var result = ctx.Db.JobResult.Id.Find(job.ResultId) ?? throw new InvalidOperationException("job result missing"); + result.Status = "complete"; + ctx.Db.JobResult.Id.Update(result); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/answers/rust.rs new file mode 100644 index 00000000000..85fc0f9bcab --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/answers/rust.rs @@ -0,0 +1,43 @@ +use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; +use std::time::Duration; + +#[table(accessor = job_result, public)] +pub struct JobResult { + #[primary_key] + pub id: u64, + pub status: String, +} + +#[table(accessor = private_job, scheduled(run_private_job))] +pub struct PrivateJob { + #[primary_key] + #[auto_inc] + pub scheduled_id: u64, + pub scheduled_at: ScheduleAt, + pub result_id: u64, +} + +#[reducer] +pub fn enqueue_private(ctx: &ReducerContext, id: u64) { + ctx.db.job_result().insert(JobResult { + id, + status: "queued".into(), + }); + ctx.db.private_job().insert(PrivateJob { + scheduled_id: 0, + scheduled_at: ScheduleAt::Time(ctx.timestamp + Duration::from_millis(1)), + result_id: id, + }); +} + +#[reducer] +pub fn run_private_job(ctx: &ReducerContext, job: PrivateJob) { + let mut result = ctx + .db + .job_result() + .id() + .find(job.result_id) + .expect("job result missing"); + result.status = "complete".into(); + ctx.db.job_result().id().update(result); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/answers/typescript.ts new file mode 100644 index 00000000000..29535b8a82a --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/answers/typescript.ts @@ -0,0 +1,24 @@ +import { ScheduleAt } from 'spacetimedb'; +import { schema, table, t } from 'spacetimedb/server'; + +const jobResult = table({ name: 'job_result', public: true }, { + id: t.u64().primaryKey(), status: t.string(), +}); +const privateJob = table({ name: 'private_job', scheduled: (): any => runPrivateJob }, { + scheduledId: t.u64().primaryKey().autoInc(), scheduledAt: t.scheduleAt(), resultId: t.u64(), +}); +const spacetimedb = schema({ jobResult, privateJob }); +export default spacetimedb; + +export const enqueue_private = spacetimedb.reducer({ id: t.u64() }, (ctx, { id }) => { + ctx.db.jobResult.insert({ id, status: 'queued' }); + ctx.db.privateJob.insert({ + scheduledId: 0n, scheduledAt: ScheduleAt.time(ctx.timestamp.microsSinceUnixEpoch + 1_000n), resultId: id, + }); +}); + +export const runPrivateJob = spacetimedb.reducer({ job: privateJob.rowType }, (ctx, { job }) => { + const result = ctx.db.jobResult.id.find(job.resultId); + if (!result) throw new Error('job result missing'); + ctx.db.jobResult.id.update({ ...result, status: 'complete' }); +}); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/spec.rs new file mode 100644 index 00000000000..06e6d8f0b2d --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/spec.rs @@ -0,0 +1,33 @@ +use crate::eval::defaults::{ + default_schema_parity_scorers, make_eventually_sql_count_scorer, make_reducer_call_both_scorer, +}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec}; +use serde_json::json; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_reducer_call_both_scorer( + host_url, + file!(), + route_tag, + "enqueue_private", + vec![json!(7)], + "enqueue_private", + )); + let table = table_name("job_result", lang); + let id = ident("id", casing_for_lang(lang)); + let status = ident("status", casing_for_lang(lang)); + scorers.push(make_eventually_sql_count_scorer( + host_url, + file!(), + route_tag, + format!("SELECT COUNT(*) AS n FROM {table} WHERE {id}=7 AND {status}='complete'"), + 1, + "private_scheduled_job_completes", + Duration::from_secs(10), + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/csharp.txt new file mode 100644 index 00000000000..6f9254ae94e --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/csharp.txt @@ -0,0 +1,20 @@ +Write a SpacetimeDB backend module in C# with a private scheduled table that is not client-callable. + +TABLES +- JobResult (public, accessor JobResult) + - Fields: + - Id: ulong (primary key) + - Status: string +- PrivateJob (private, scheduled for RunPrivateJob, accessor PrivateJob) + - Fields, in this order: + - ScheduledId: ulong (primary key, auto-increment) + - ScheduledAt: ScheduleAt + - ResultId: ulong + +REDUCERS +- EnqueuePrivate(id: ulong) + - Write JobResult Status "queued" + - Schedule that Id for one millisecond later +- RunPrivateJob(job: PrivateJob) (scheduled reducer) + - Must be scheduler-only and not client-callable + - Update the matching JobResult Status to "complete" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/rust.txt new file mode 100644 index 00000000000..0e1f732ecfc --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/rust.txt @@ -0,0 +1,20 @@ +Write a SpacetimeDB backend module in Rust with a private scheduled table that is not client-callable. + +TABLES +- job_result (public), struct JobResult + - Fields: + - id: u64 (primary key) + - status: String +- private_job (private, scheduled for run_private_job), struct PrivateJob + - Fields, in this order: + - scheduled_id: u64 (primary key, auto-increment) + - scheduled_at: ScheduleAt + - result_id: u64 + +REDUCERS +- enqueue_private(id: u64) + - Write job_result status "queued" + - Schedule that id for one millisecond later +- run_private_job(job: PrivateJob) (scheduled reducer) + - Must be scheduler-only and not client-callable + - Update the matching job_result status to "complete" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/typescript.txt new file mode 100644 index 00000000000..0dc1884a710 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/lifecycle/t_071_scheduled_private/tasks/typescript.txt @@ -0,0 +1,20 @@ +Write a SpacetimeDB backend module in TypeScript with a private scheduled table that is not client-callable. + +TABLES +- job_result (public) + - Fields: + - id: u64 (primary key) + - status: string +- private_job (private, scheduled for runPrivateJob) + - Fields, in this order: + - scheduledId: u64 (primary key, auto-increment) + - scheduledAt: ScheduleAt + - resultId: u64 + +REDUCERS +- enqueue_private(id: u64) + - Write job_result status "queued" + - Schedule that id for one millisecond later +- runPrivateJob(job: private_job row) (scheduled reducer) + - Must be scheduler-only and not client-callable + - Update the matching job_result status to "complete" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/answers/csharp.cs new file mode 100644 index 00000000000..1471824e930 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/answers/csharp.cs @@ -0,0 +1,29 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Product", Public = true)] + public partial struct Product + { + [PrimaryKey] public ulong Id; + public string Name; + } + + [Table(Accessor = "Category", Public = true)] + public partial struct Category + { + [PrimaryKey] public ulong Id; + public string Label; + } + + [Reducer] + public static void Seed(ReducerContext ctx) => + ctx.Db.Product.Insert(new Product { Id = 1, Name = "legacy" }); + + [Reducer] + public static void Touch(ReducerContext ctx) { } + + [Reducer] + public static void CreateCategory(ReducerContext ctx, ulong id, string label) => + ctx.Db.Category.Insert(new Category { Id = id, Label = label }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/answers/rust.rs new file mode 100644 index 00000000000..4609b6f9bc9 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/answers/rust.rs @@ -0,0 +1,31 @@ +use spacetimedb::{ReducerContext, Table}; + +#[spacetimedb::table(accessor = product, public)] +pub struct Product { + #[primary_key] + id: u64, + name: String, +} + +#[spacetimedb::table(accessor = category, public)] +pub struct Category { + #[primary_key] + id: u64, + label: String, +} + +#[spacetimedb::reducer] +pub fn seed(ctx: &ReducerContext) { + ctx.db.product().insert(Product { + id: 1, + name: "legacy".into(), + }); +} + +#[spacetimedb::reducer] +pub fn touch(_ctx: &ReducerContext) {} + +#[spacetimedb::reducer] +pub fn create_category(ctx: &ReducerContext, id: u64, label: String) { + ctx.db.category().insert(Category { id, label }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/answers/typescript.ts new file mode 100644 index 00000000000..8f7098d07bb --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/answers/typescript.ts @@ -0,0 +1,21 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const product = table( + { name: 'product', public: true }, + { id: t.u64().primaryKey(), name: t.string() } +); +const category = table( + { name: 'category', public: true }, + { id: t.u64().primaryKey(), label: t.string() } +); +const spacetimedb = schema({ product, category }); +export default spacetimedb; + +export const seed = spacetimedb.reducer(ctx => { + ctx.db.product.insert({ id: 1n, name: 'legacy' }); +}); +export const touch = spacetimedb.reducer(_ctx => {}); +export const create_category = spacetimedb.reducer( + { id: t.u64(), label: t.string() }, + (ctx, { id, label }) => ctx.db.category.insert({ id, label }) +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/setup/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/setup/csharp.cs new file mode 100644 index 00000000000..3e82962a547 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/setup/csharp.cs @@ -0,0 +1,18 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Product", Public = true)] + public partial struct Product + { + [PrimaryKey] public ulong Id; + public string Name; + } + + [Reducer] + public static void Seed(ReducerContext ctx) => + ctx.Db.Product.Insert(new Product { Id = 1, Name = "legacy" }); + + [Reducer] + public static void Touch(ReducerContext ctx) { } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/setup/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/setup/rust.rs new file mode 100644 index 00000000000..7eb1cc95792 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/setup/rust.rs @@ -0,0 +1,19 @@ +use spacetimedb::{ReducerContext, Table}; + +#[spacetimedb::table(accessor = product, public)] +pub struct Product { + #[primary_key] + id: u64, + name: String, +} + +#[spacetimedb::reducer] +pub fn seed(ctx: &ReducerContext) { + ctx.db.product().insert(Product { + id: 1, + name: "legacy".into(), + }); +} + +#[spacetimedb::reducer] +pub fn touch(_ctx: &ReducerContext) {} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/setup/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/setup/typescript.ts new file mode 100644 index 00000000000..e0ed489261c --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/setup/typescript.ts @@ -0,0 +1,13 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const product = table( + { name: 'product', public: true }, + { id: t.u64().primaryKey(), name: t.string() } +); +const spacetimedb = schema({ product }); +export default spacetimedb; + +export const seed = spacetimedb.reducer(ctx => { + ctx.db.product.insert({ id: 1n, name: 'legacy' }); +}); +export const touch = spacetimedb.reducer(_ctx => {}); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/spec.rs new file mode 100644 index 00000000000..bb3ee0fb7f4 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/spec.rs @@ -0,0 +1,41 @@ +use crate::eval::defaults::{ + default_schema_parity_scorers, make_reducer_data_parity_scorer, make_sql_count_only_scorer, +}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec, ReducerDataParityConfig}; +use serde_json::json; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + let casing = casing_for_lang(lang); + let product = table_name("product", lang); + let product_name = ident("name", casing); + scorers.push(make_sql_count_only_scorer( + host_url, + file!(), + route_tag, + format!("SELECT COUNT(*) AS n FROM {product} WHERE {product_name}='legacy'"), + 1, + "existing_data_preserved", + Duration::from_secs(10), + )); + let category = table_name("category", lang); + let id = ident("id", casing); + let label = ident("label", casing); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "create_category".into(), + args: vec![json!(7), json!("general")], + select_query: format!("SELECT {id}, {label} FROM {category}"), + collapse_ws: true, + timeout: Duration::from_secs(10), + id_str: "new_schema_usable", + }, + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/tasks/csharp.txt new file mode 100644 index 00000000000..f2f2d6fe3c7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/tasks/csharp.txt @@ -0,0 +1,17 @@ +Update the existing SpacetimeDB backend module in C# with a compatible schema change. + +TABLES +- Product (public, accessor Product) + - Preserve the existing table and its fields +- Category (public, accessor Category) + - Fields: + - Id: ulong (primary key) + - Label: string + +REDUCERS +- Preserve Seed and Touch +- CreateCategory(id: ulong, label: string) + - Insert a Category with the supplied values + +MIGRATION +- The populated Product table must survive a normal republish without deleting data diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/tasks/rust.txt new file mode 100644 index 00000000000..92134a461db --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/tasks/rust.txt @@ -0,0 +1,17 @@ +Update the existing SpacetimeDB backend module in Rust with a compatible schema change. + +TABLES +- product (public), struct Product + - Preserve the existing table and its fields +- category (public), struct Category + - Fields: + - id: u64 (primary key) + - label: String + +REDUCERS +- Preserve seed and touch +- create_category(id: u64, label: String) + - Insert a Category with the supplied values + +MIGRATION +- The populated product table must survive a normal republish without deleting data diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/tasks/typescript.txt new file mode 100644 index 00000000000..b9e084866f4 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_080_automatic_migration/tasks/typescript.txt @@ -0,0 +1,17 @@ +Update the existing SpacetimeDB backend module in TypeScript with a compatible schema change. + +TABLES +- product (public) + - Preserve the existing table and its fields +- category (public) + - Fields: + - id: u64 (primary key) + - label: string + +REDUCERS +- Preserve seed and touch +- create_category(id: u64, label: string) + - Insert a category with the supplied values + +MIGRATION +- The populated product table must survive a normal republish without deleting data diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/answers/csharp.cs new file mode 100644 index 00000000000..9601ba34cb6 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/answers/csharp.cs @@ -0,0 +1,40 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "LegacyItem", Public = true)] + public partial struct LegacyItem + { + [PrimaryKey] public ulong Id; + public string Value; + } + + [Table(Accessor = "ItemV2", Public = true)] + public partial struct ItemV2 + { + [PrimaryKey] public ulong Id; + public string Value; + public uint Version; + } + + [Reducer] + public static void Seed(ReducerContext ctx) => + ctx.Db.LegacyItem.Insert(new LegacyItem { Id = 1, Value = "old" }); + + [Reducer] + public static void Migrate(ReducerContext ctx) + { + foreach (var row in ctx.Db.LegacyItem.Iter()) + { + if (ctx.Db.ItemV2.Id.Find(row.Id) is null) + ctx.Db.ItemV2.Insert(new ItemV2 { Id = row.Id, Value = row.Value, Version = 2 }); + } + } + + [Reducer] + public static void DualWrite(ReducerContext ctx, ulong id, string value) + { + ctx.Db.LegacyItem.Insert(new LegacyItem { Id = id, Value = value }); + ctx.Db.ItemV2.Insert(new ItemV2 { Id = id, Value = value, Version = 2 }); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/answers/rust.rs new file mode 100644 index 00000000000..0ac6567d852 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/answers/rust.rs @@ -0,0 +1,46 @@ +use spacetimedb::{ReducerContext, Table}; + +#[spacetimedb::table(accessor = legacy_item, public)] +pub struct LegacyItem { + #[primary_key] + id: u64, + value: String, +} + +#[spacetimedb::table(accessor = item_v2, public)] +pub struct ItemV2 { + #[primary_key] + id: u64, + value: String, + version: u32, +} + +#[spacetimedb::reducer] +pub fn seed(ctx: &ReducerContext) { + ctx.db.legacy_item().insert(LegacyItem { + id: 1, + value: "old".into(), + }); +} + +#[spacetimedb::reducer] +pub fn migrate(ctx: &ReducerContext) { + for row in ctx.db.legacy_item().iter() { + if ctx.db.item_v2().id().find(row.id).is_none() { + ctx.db.item_v2().insert(ItemV2 { + id: row.id, + value: row.value, + version: 2, + }); + } + } +} + +#[spacetimedb::reducer] +pub fn dual_write(ctx: &ReducerContext, id: u64, value: String) { + ctx.db.legacy_item().insert(LegacyItem { + id, + value: value.clone(), + }); + ctx.db.item_v2().insert(ItemV2 { id, value, version: 2 }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/answers/typescript.ts new file mode 100644 index 00000000000..758e4d5be79 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/answers/typescript.ts @@ -0,0 +1,32 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const legacyItem = table( + { name: 'legacy_item', public: true }, + { id: t.u64().primaryKey(), value: t.string() } +); +const itemV2 = table( + { name: 'item_v2', public: true }, + { id: t.u64().primaryKey(), value: t.string(), version: t.u32() } +); +const spacetimedb = schema({ legacyItem, itemV2 }); +export default spacetimedb; + +export const seed = spacetimedb.reducer(ctx => { + ctx.db.legacyItem.insert({ id: 1n, value: 'old' }); +}); + +export const migrate = spacetimedb.reducer(ctx => { + for (const row of ctx.db.legacyItem.iter()) { + if (!ctx.db.itemV2.id.find(row.id)) { + ctx.db.itemV2.insert({ id: row.id, value: row.value, version: 2 }); + } + } +}); + +export const dual_write = spacetimedb.reducer( + { id: t.u64(), value: t.string() }, + (ctx, { id, value }) => { + ctx.db.legacyItem.insert({ id, value }); + ctx.db.itemV2.insert({ id, value, version: 2 }); + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/setup/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/setup/csharp.cs new file mode 100644 index 00000000000..2dbe7c6f2c9 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/setup/csharp.cs @@ -0,0 +1,15 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "LegacyItem", Public = true)] + public partial struct LegacyItem + { + [PrimaryKey] public ulong Id; + public string Value; + } + + [Reducer] + public static void Seed(ReducerContext ctx) => + ctx.Db.LegacyItem.Insert(new LegacyItem { Id = 1, Value = "old" }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/setup/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/setup/rust.rs new file mode 100644 index 00000000000..68e9f6f1db4 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/setup/rust.rs @@ -0,0 +1,16 @@ +use spacetimedb::{ReducerContext, Table}; + +#[spacetimedb::table(accessor = legacy_item, public)] +pub struct LegacyItem { + #[primary_key] + id: u64, + value: String, +} + +#[spacetimedb::reducer] +pub fn seed(ctx: &ReducerContext) { + ctx.db.legacy_item().insert(LegacyItem { + id: 1, + value: "old".into(), + }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/setup/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/setup/typescript.ts new file mode 100644 index 00000000000..246be8c6bec --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/setup/typescript.ts @@ -0,0 +1,12 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const legacyItem = table( + { name: 'legacy_item', public: true }, + { id: t.u64().primaryKey(), value: t.string() } +); +const spacetimedb = schema({ legacyItem }); +export default spacetimedb; + +export const seed = spacetimedb.reducer(ctx => { + ctx.db.legacyItem.insert({ id: 1n, value: 'old' }); +}); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/spec.rs new file mode 100644 index 00000000000..d51b23f243a --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/spec.rs @@ -0,0 +1,57 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_data_parity_scorer}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec, ReducerDataParityConfig}; +use serde_json::json; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + let casing = casing_for_lang(lang); + let item_v2 = table_name("item_v2", lang); + let id = ident("id", casing); + let value = ident("value", casing); + let version = ident("version", casing); + let v2_query = format!("SELECT {id}, {value}, {version} FROM {item_v2}"); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "migrate".into(), + args: vec![], + select_query: v2_query.clone(), + collapse_ws: true, + timeout: Duration::from_secs(10), + id_str: "legacy_rows_migrated", + }, + )); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "dual_write".into(), + args: vec![json!(2), json!("new")], + select_query: v2_query, + collapse_ws: true, + timeout: Duration::from_secs(10), + id_str: "dual_write_keeps_v2_current", + }, + )); + let legacy = table_name("legacy_item", lang); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "migrate".into(), + args: vec![], + select_query: format!("SELECT {id}, {value} FROM {legacy}"), + collapse_ws: true, + timeout: Duration::from_secs(10), + id_str: "legacy_table_remains_current", + }, + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/tasks/csharp.txt new file mode 100644 index 00000000000..78d33872c6d --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/tasks/csharp.txt @@ -0,0 +1,19 @@ +Update the existing SpacetimeDB backend module in C# with an incremental migration. + +TABLES +- LegacyItem (public, accessor LegacyItem) + - Preserve the existing table and fields +- ItemV2 (public, accessor ItemV2) + - Fields: + - Id: ulong (primary key) + - Value: string + - Version: uint + +REDUCERS +- Preserve Seed +- Migrate() + - Be idempotent + - Copy missing LegacyItem rows into ItemV2 with Version 2 +- DualWrite(id: ulong, value: string) + - Write the same new item to both tables + - Set Version 2 in ItemV2 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/tasks/rust.txt new file mode 100644 index 00000000000..11bfd78b357 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/tasks/rust.txt @@ -0,0 +1,19 @@ +Update the existing SpacetimeDB backend module in Rust with an incremental migration. + +TABLES +- legacy_item (public), struct LegacyItem + - Preserve the existing table and fields +- item_v2 (public), struct ItemV2 + - Fields: + - id: u64 (primary key) + - value: String + - version: u32 + +REDUCERS +- Preserve seed +- migrate() + - Be idempotent + - Copy missing legacy_item rows into item_v2 with version 2 +- dual_write(id: u64, value: String) + - Write the same new item to both tables + - Set version 2 in item_v2 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/tasks/typescript.txt new file mode 100644 index 00000000000..e13eacad628 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_081_incremental_migration/tasks/typescript.txt @@ -0,0 +1,19 @@ +Update the existing SpacetimeDB backend module in TypeScript with an incremental migration. + +TABLES +- legacy_item (public) + - Preserve the existing table and fields +- item_v2 (public) + - Fields: + - id: u64 (primary key) + - value: string + - version: u32 + +REDUCERS +- Preserve seed +- migrate() + - Be idempotent + - Copy missing legacy_item rows into item_v2 with version 2 +- dual_write(id: u64, value: string) + - Write the same new item to both tables + - Set version 2 in item_v2 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/answers/csharp.cs new file mode 100644 index 00000000000..0f358013e5b --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/answers/csharp.cs @@ -0,0 +1,33 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Counter", Public = true)] + public partial struct Counter + { + [PrimaryKey] public ulong Id; + public long Value; + } + + [Table(Accessor = "Release")] + public partial struct Release + { + [PrimaryKey] public uint Version; + } + + [Reducer] + public static void Seed(ReducerContext ctx) => + ctx.Db.Counter.Insert(new Counter { Id = 1, Value = 1 }); + + [Reducer] + public static void Increment(ReducerContext ctx, ulong id, long amount) + { + var row = ctx.Db.Counter.Id.Find(id) ?? throw new Exception("counter"); + row.Value += amount; + ctx.Db.Counter.Id.Update(row); + } + + [Reducer] + public static void RecordRelease(ReducerContext ctx, uint version) => + ctx.Db.Release.Insert(new Release { Version = version }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/answers/rust.rs new file mode 100644 index 00000000000..a0f1a3498d3 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/answers/rust.rs @@ -0,0 +1,31 @@ +use spacetimedb::{ReducerContext, Table}; + +#[spacetimedb::table(accessor = counter, public)] +pub struct Counter { + #[primary_key] + id: u64, + value: i64, +} + +#[spacetimedb::table(accessor = release)] +pub struct Release { + #[primary_key] + version: u32, +} + +#[spacetimedb::reducer] +pub fn seed(ctx: &ReducerContext) { + ctx.db.counter().insert(Counter { id: 1, value: 1 }); +} + +#[spacetimedb::reducer] +pub fn increment(ctx: &ReducerContext, id: u64, amount: i64) { + let mut row = ctx.db.counter().id().find(id).expect("counter"); + row.value += amount; + ctx.db.counter().id().update(row); +} + +#[spacetimedb::reducer] +pub fn record_release(ctx: &ReducerContext, version: u32) { + ctx.db.release().insert(Release { version }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/answers/typescript.ts new file mode 100644 index 00000000000..0e90ee4a465 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/answers/typescript.ts @@ -0,0 +1,28 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const counter = table( + { name: 'counter', public: true }, + { id: t.u64().primaryKey(), value: t.i64() } +); +const release = table( + { name: 'release' }, + { version: t.u32().primaryKey() } +); +const spacetimedb = schema({ counter, release }); +export default spacetimedb; + +export const seed = spacetimedb.reducer(ctx => { + ctx.db.counter.insert({ id: 1n, value: 1n }); +}); +export const increment = spacetimedb.reducer( + { id: t.u64(), amount: t.i64() }, + (ctx, { id, amount }) => { + const row = ctx.db.counter.id.find(id); + if (!row) throw new Error('counter'); + ctx.db.counter.id.update({ ...row, value: row.value + amount }); + } +); +export const record_release = spacetimedb.reducer( + { version: t.u32() }, + (ctx, { version }) => ctx.db.release.insert({ version }) +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/setup/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/setup/csharp.cs new file mode 100644 index 00000000000..315677e3901 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/setup/csharp.cs @@ -0,0 +1,23 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Counter", Public = true)] + public partial struct Counter + { + [PrimaryKey] public ulong Id; + public long Value; + } + + [Reducer] + public static void Seed(ReducerContext ctx) => + ctx.Db.Counter.Insert(new Counter { Id = 1, Value = 1 }); + + [Reducer] + public static void Increment(ReducerContext ctx, ulong id, long amount) + { + var row = ctx.Db.Counter.Id.Find(id) ?? throw new Exception("counter"); + row.Value += amount; + ctx.Db.Counter.Id.Update(row); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/setup/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/setup/rust.rs new file mode 100644 index 00000000000..341fa49e039 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/setup/rust.rs @@ -0,0 +1,20 @@ +use spacetimedb::{ReducerContext, Table}; + +#[spacetimedb::table(accessor = counter, public)] +pub struct Counter { + #[primary_key] + id: u64, + value: i64, +} + +#[spacetimedb::reducer] +pub fn seed(ctx: &ReducerContext) { + ctx.db.counter().insert(Counter { id: 1, value: 1 }); +} + +#[spacetimedb::reducer] +pub fn increment(ctx: &ReducerContext, id: u64, amount: i64) { + let mut row = ctx.db.counter().id().find(id).expect("counter"); + row.value += amount; + ctx.db.counter().id().update(row); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/setup/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/setup/typescript.ts new file mode 100644 index 00000000000..e2aca9da9fd --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/setup/typescript.ts @@ -0,0 +1,20 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const counter = table( + { name: 'counter', public: true }, + { id: t.u64().primaryKey(), value: t.i64() } +); +const spacetimedb = schema({ counter }); +export default spacetimedb; + +export const seed = spacetimedb.reducer(ctx => { + ctx.db.counter.insert({ id: 1n, value: 1n }); +}); +export const increment = spacetimedb.reducer( + { id: t.u64(), amount: t.i64() }, + (ctx, { id, amount }) => { + const row = ctx.db.counter.id.find(id); + if (!row) throw new Error('counter'); + ctx.db.counter.id.update({ ...row, value: row.value + amount }); + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/spec.rs new file mode 100644 index 00000000000..7223b5d9571 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/spec.rs @@ -0,0 +1,28 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_data_parity_scorer}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec, ReducerDataParityConfig}; +use serde_json::json; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + let counter = table_name("counter", lang); + let casing = casing_for_lang(lang); + let id = ident("id", casing); + let value = ident("value", casing); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "increment".into(), + args: vec![json!(1), json!(2)], + select_query: format!("SELECT {id}, {value} FROM {counter}"), + collapse_ws: true, + timeout: Duration::from_secs(10), + id_str: "existing_api_survives_republish", + }, + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/tasks/csharp.txt new file mode 100644 index 00000000000..b77b572b46f --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/tasks/csharp.txt @@ -0,0 +1,17 @@ +Update the existing SpacetimeDB backend module in C# through a compatible republish. + +TABLES +- Counter (public, accessor Counter) + - Preserve the existing table and its exact fields +- Release (private, accessor Release) + - Fields: + - Version: uint (primary key) + +REDUCERS +- Preserve Seed +- Preserve Increment without renaming it or changing its signature +- RecordRelease(version: uint) + +MIGRATION +- Existing Counter data must remain usable after republish +- The existing server API must remain usable after republish diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/tasks/rust.txt new file mode 100644 index 00000000000..e4c7cf786ac --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/tasks/rust.txt @@ -0,0 +1,17 @@ +Update the existing SpacetimeDB backend module in Rust through a compatible republish. + +TABLES +- counter (public), struct Counter + - Preserve the existing table and its exact fields +- release (private), struct Release + - Fields: + - version: u32 (primary key) + +REDUCERS +- Preserve seed +- Preserve increment without renaming it or changing its signature +- record_release(version: u32) + +MIGRATION +- Existing counter data must remain usable after republish +- The existing server API must remain usable after republish diff --git a/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/tasks/typescript.txt new file mode 100644 index 00000000000..2a1c3acebd9 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/migrations/t_082_hot_swap_compatibility/tasks/typescript.txt @@ -0,0 +1,17 @@ +Update the existing SpacetimeDB backend module in TypeScript through a compatible republish. + +TABLES +- counter (public) + - Preserve the existing table and its exact fields +- release (private) + - Fields: + - version: u32 (primary key) + +REDUCERS +- Preserve seed +- Preserve increment without renaming it or changing its signature +- record_release(version: u32) + +MIGRATION +- Existing counter data must remain usable after republish +- The existing server API must remain usable after republish diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/answers/csharp.cs new file mode 100644 index 00000000000..d8ab28724ea --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/answers/csharp.cs @@ -0,0 +1,12 @@ +using SpacetimeDB; +#pragma warning disable STDB_UNSTABLE + +[SpacetimeDB.Type] +public partial struct Summary { public uint Total; public string Label; } + +public static partial class Module +{ + [SpacetimeDB.Procedure] + public static Summary CalculateSummary(ProcedureContext ctx, uint lhs, uint rhs) => + new() { Total = lhs + rhs, Label = "calculated" }; +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/answers/rust.rs new file mode 100644 index 00000000000..298a0ef5802 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/answers/rust.rs @@ -0,0 +1,15 @@ +use spacetimedb::{procedure, ProcedureContext, SpacetimeType}; + +#[derive(SpacetimeType)] +pub struct Summary { + pub total: u32, + pub label: String, +} + +#[procedure] +pub fn calculate_summary(_ctx: &mut ProcedureContext, lhs: u32, rhs: u32) -> Summary { + Summary { + total: lhs + rhs, + label: "calculated".into(), + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/answers/typescript.ts new file mode 100644 index 00000000000..1a66b711d85 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/answers/typescript.ts @@ -0,0 +1,10 @@ +import { schema, t } from 'spacetimedb/server'; + +const Summary = t.object('Summary', { total: t.u32(), label: t.string() }); +const spacetimedb = schema({}); +export default spacetimedb; + +export const calculate_summary = spacetimedb.procedure( + { lhs: t.u32(), rhs: t.u32() }, Summary, + (_ctx, { lhs, rhs }) => ({ total: lhs + rhs, label: 'calculated' }) +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/spec.rs new file mode 100644 index 00000000000..b0fa5340997 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/spec.rs @@ -0,0 +1,18 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_call_output_parity_scorer}; +use crate::eval::BenchmarkSpec; +use serde_json::json; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_call_output_parity_scorer( + host_url, + file!(), + route_tag, + "calculate_summary", + vec![json!(7), json!(5)], + "typed_procedure_return", + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/csharp.txt new file mode 100644 index 00000000000..cdbb236e4e9 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/csharp.txt @@ -0,0 +1,11 @@ +Write a SpacetimeDB backend module in C# that defines a procedure with a typed return value. + +TYPES +- Summary + - Total: uint + - Label: string + +PROCEDURE +- CalculateSummary(lhs: uint, rhs: uint) -> Summary + - Return Total as lhs + rhs + - Return Label as "calculated" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/rust.txt new file mode 100644 index 00000000000..f2df9647087 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/rust.txt @@ -0,0 +1,11 @@ +Write a SpacetimeDB backend module in Rust that defines a procedure with a typed return value. + +TYPES +- Summary + - total: u32 + - label: String + +PROCEDURE +- calculate_summary(lhs: u32, rhs: u32) -> Summary + - Return total as lhs + rhs + - Return label as "calculated" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/typescript.txt new file mode 100644 index 00000000000..48ee3ce58f6 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_072_procedure_return/tasks/typescript.txt @@ -0,0 +1,11 @@ +Write a SpacetimeDB backend module in TypeScript that defines a procedure with a typed return value. + +TYPES +- Summary + - total: u32 + - label: string + +PROCEDURE +- calculate_summary(lhs: u32, rhs: u32) -> Summary + - Return total as lhs + rhs + - Return label as "calculated" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/csharp.cs new file mode 100644 index 00000000000..a3295bb0f93 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/csharp.cs @@ -0,0 +1,25 @@ +using SpacetimeDB; +using System.Text; +#pragma warning disable STDB_UNSTABLE + +[SpacetimeDB.Type] +public partial struct FetchSummary { public ushort Status; public bool HtmlContentType; public bool HasExampleDomain; } + +public static partial class Module +{ + [SpacetimeDB.Procedure] + public static FetchSummary FetchPageSummary(ProcedureContext ctx, string url) + { + var result = ctx.Http.Get(url); + return result.Match(response => { + var contentType = response.Headers.FirstOrDefault(h => h.Name.Equals("content-type", StringComparison.OrdinalIgnoreCase)); + var contentTypeValue = contentType.Value is null ? "" : Encoding.UTF8.GetString(contentType.Value); + var body = response.Body.ToStringUtf8Lossy(); + return new FetchSummary { + Status = response.StatusCode, + HtmlContentType = contentTypeValue.Contains("text/html"), + HasExampleDomain = body.Contains("Example Domain"), + }; + }, error => throw new Exception(error.Message)); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/rust.rs new file mode 100644 index 00000000000..eeef78345b1 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/rust.rs @@ -0,0 +1,25 @@ +use spacetimedb::{procedure, ProcedureContext, SpacetimeType}; + +#[derive(SpacetimeType)] +pub struct FetchSummary { + pub status: u16, + pub html_content_type: bool, + pub has_example_domain: bool, +} + +#[procedure] +pub fn fetch_page_summary(ctx: &mut ProcedureContext, url: String) -> FetchSummary { + let response = ctx.http.get(url).expect("page request failed"); + let status = response.status().as_u16(); + let html_content_type = response + .headers() + .get("content-type") + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.contains("text/html")); + let body = response.into_body().into_string_lossy(); + FetchSummary { + status, + html_content_type, + has_example_domain: body.contains("Example Domain"), + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/typescript.ts new file mode 100644 index 00000000000..c027095ce68 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/answers/typescript.ts @@ -0,0 +1,19 @@ +import { schema, t } from 'spacetimedb/server'; + +const FetchSummary = t.object('FetchSummary', { + status: t.u16(), htmlContentType: t.bool(), hasExampleDomain: t.bool(), +}); +const spacetimedb = schema({}); +export default spacetimedb; + +export const fetch_page_summary = spacetimedb.procedure( + { url: t.string() }, FetchSummary, + (ctx, { url }) => { + const response = ctx.http.fetch(url); + return { + status: response.status, + htmlContentType: (response.headers.get('content-type') ?? '').includes('text/html'), + hasExampleDomain: response.text().includes('Example Domain'), + }; + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/spec.rs new file mode 100644 index 00000000000..18001ff17ac --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/spec.rs @@ -0,0 +1,19 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_call_output_parity_scorer_with_attempts}; +use crate::eval::BenchmarkSpec; +use serde_json::json; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_call_output_parity_scorer_with_attempts( + host_url, + file!(), + route_tag, + "fetch_page_summary", + vec![json!("https://example.com")], + "http_response_summary", + 3, + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/csharp.txt new file mode 100644 index 00000000000..128c436ca71 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/csharp.txt @@ -0,0 +1,14 @@ +Write a SpacetimeDB backend module in C# that performs outbound HTTP in a procedure. + +TYPES +- FetchSummary + - Status: ushort + - HtmlContentType: bool + - HasExampleDomain: bool + +PROCEDURE +- FetchPageSummary(url: string) -> FetchSummary + - Perform an outbound GET to the supplied public URL + - Set Status from the response status + - Set HtmlContentType based on whether Content-Type contains "text/html" + - Set HasExampleDomain based on whether the body contains "Example Domain" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/rust.txt new file mode 100644 index 00000000000..059a78203ab --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/rust.txt @@ -0,0 +1,14 @@ +Write a SpacetimeDB backend module in Rust that performs outbound HTTP in a procedure. + +TYPES +- FetchSummary + - status: u16 + - html_content_type: bool + - has_example_domain: bool + +PROCEDURE +- fetch_page_summary(url: String) -> FetchSummary + - Perform an outbound GET to the supplied public URL + - Set status from the response status + - Set html_content_type based on whether Content-Type contains "text/html" + - Set has_example_domain based on whether the body contains "Example Domain" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/typescript.txt new file mode 100644 index 00000000000..7d0e9808a5f --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_073_http_fetch/tasks/typescript.txt @@ -0,0 +1,14 @@ +Write a SpacetimeDB backend module in TypeScript that performs outbound HTTP in a procedure. + +TYPES +- FetchSummary + - status: u16 + - htmlContentType: bool + - hasExampleDomain: bool + +PROCEDURE +- fetch_page_summary(url: string) -> FetchSummary + - Perform an outbound GET to the supplied public URL + - Set status from the response status + - Set htmlContentType based on whether Content-Type contains "text/html" + - Set hasExampleDomain based on whether the body contains "Example Domain" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/csharp.cs new file mode 100644 index 00000000000..9b118401472 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/csharp.cs @@ -0,0 +1,19 @@ +using SpacetimeDB; +#pragma warning disable STDB_UNSTABLE + +public static partial class Module +{ + [Table(Accessor = "FetchedRecord", Public = true)] + public partial struct FetchedRecord { [PrimaryKey] public ulong Id; public ushort Status; public bool ValidBody; } + + [SpacetimeDB.Procedure] + public static void FetchAndStore(ProcedureContext ctx, string url) + { + var result = ctx.Http.Get(url); + result.Match(response => { + var validBody = response.Body.ToStringUtf8Lossy().Contains("Example Domain"); + ctx.WithTx(tx => { tx.Db.FetchedRecord.Insert(new FetchedRecord { Id = 1, Status = response.StatusCode, ValidBody = validBody }); return 0; }); + return 0; + }, error => throw new Exception(error.Message)); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/rust.rs new file mode 100644 index 00000000000..57aa7883f7b --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/rust.rs @@ -0,0 +1,23 @@ +use spacetimedb::{procedure, table, ProcedureContext, Table}; + +#[table(accessor = fetched_record, public)] +pub struct FetchedRecord { + #[primary_key] + pub id: u64, + pub status: u16, + pub valid_body: bool, +} + +#[procedure] +pub fn fetch_and_store(ctx: &mut ProcedureContext, url: String) { + let response = ctx.http.get(url).expect("page request failed"); + let status = response.status().as_u16(); + let valid_body = response.into_body().into_string_lossy().contains("Example Domain"); + ctx.with_tx(|tx| { + tx.db.fetched_record().insert(FetchedRecord { + id: 1, + status, + valid_body, + }); + }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/typescript.ts new file mode 100644 index 00000000000..a4e23be8b00 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/answers/typescript.ts @@ -0,0 +1,18 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const fetchedRecord = table({ name: 'fetched_record', public: true }, { + id: t.u64().primaryKey(), status: t.u16(), validBody: t.bool(), +}); +const spacetimedb = schema({ fetchedRecord }); +export default spacetimedb; + +export const fetch_and_store = spacetimedb.procedure( + { url: t.string() }, t.unit(), + (ctx, { url }) => { + const response = ctx.http.fetch(url); + const status = response.status; + const validBody = response.text().includes('Example Domain'); + ctx.withTx(tx => tx.db.fetchedRecord.insert({ id: 1n, status, validBody })); + return {}; + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/spec.rs new file mode 100644 index 00000000000..148548fa342 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/spec.rs @@ -0,0 +1,34 @@ +use crate::eval::defaults::{ + default_schema_parity_scorers, make_reducer_call_both_scorer_with_attempts, make_sql_count_only_scorer, +}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec}; +use serde_json::json; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_reducer_call_both_scorer_with_attempts( + host_url, + file!(), + route_tag, + "fetch_and_store", + vec![json!("https://example.com")], + "fetch_and_store", + 3, + )); + let table = table_name("fetched_record", lang); + let status = ident("status", casing_for_lang(lang)); + let valid = ident("valid_body", casing_for_lang(lang)); + scorers.push(make_sql_count_only_scorer( + host_url, + file!(), + route_tag, + format!("SELECT COUNT(*) AS n FROM {table} WHERE {status}=200 AND {valid}=true"), + 1, + "fetched_row_stored", + Duration::from_secs(10), + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/csharp.txt new file mode 100644 index 00000000000..2e042f4e476 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/csharp.txt @@ -0,0 +1,14 @@ +Write a SpacetimeDB backend module in C# that fetches external data before storing a result in a transaction. + +TABLE +- FetchedRecord (public, accessor FetchedRecord) + - Fields: + - Id: ulong (primary key) + - Status: ushort + - ValidBody: bool + +PROCEDURE +- FetchAndStore(url: string) + - Perform an outbound GET to the supplied public URL before opening a transaction + - Inside a short transaction, store Id 1 and the response status + - Set ValidBody based on whether the response body contains "Example Domain" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/rust.txt new file mode 100644 index 00000000000..5b42d9d9811 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/rust.txt @@ -0,0 +1,14 @@ +Write a SpacetimeDB backend module in Rust that fetches external data before storing a result in a transaction. + +TABLE +- fetched_record (public), struct FetchedRecord + - Fields: + - id: u64 (primary key) + - status: u16 + - valid_body: bool + +PROCEDURE +- fetch_and_store(url: String) + - Perform an outbound GET to the supplied public URL before opening a transaction + - Inside a short transaction, store id 1 and the response status + - Set valid_body based on whether the response body contains "Example Domain" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/typescript.txt new file mode 100644 index 00000000000..6ae5f60bfc8 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_074_fetch_and_store/tasks/typescript.txt @@ -0,0 +1,14 @@ +Write a SpacetimeDB backend module in TypeScript that fetches external data before storing a result in a transaction. + +TABLE +- fetched_record (public) + - Fields: + - id: u64 (primary key) + - status: u16 + - validBody: bool + +PROCEDURE +- fetch_and_store(url: string) + - Perform an outbound GET to the supplied public URL before opening a transaction + - Inside a short transaction, store id 1 and the response status + - Set validBody based on whether the response body contains "Example Domain" diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/answers/csharp.cs new file mode 100644 index 00000000000..a54ba551864 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/answers/csharp.cs @@ -0,0 +1,28 @@ +using SpacetimeDB; +#pragma warning disable STDB_UNSTABLE + +public static partial class Module +{ + [Table(Accessor = "ProcedureResult", Public = true)] + public partial struct ProcedureResult { [PrimaryKey] public ulong Id; public uint Value; } + + [Table(Accessor = "ProcedureJob", Scheduled = nameof(RunScheduledProcedure), ScheduledAt = nameof(ProcedureJob.ScheduledAt))] + public partial struct ProcedureJob + { + [PrimaryKey, AutoInc] public ulong ScheduledId; + public ScheduleAt ScheduledAt; + public ulong Id; + public uint Lhs; + public uint Rhs; + } + + [Reducer] + public static void ScheduleProcedure(ReducerContext ctx, ulong id, uint lhs, uint rhs) => + ctx.Db.ProcedureJob.Insert(new ProcedureJob { + ScheduledAt = new ScheduleAt.Time(ctx.Timestamp + new TimeDuration { Microseconds = 1_000 }), Id = id, Lhs = lhs, Rhs = rhs, + }); + + [SpacetimeDB.Procedure] + public static void RunScheduledProcedure(ProcedureContext ctx, ProcedureJob job) => + ctx.WithTx(tx => { tx.Db.ProcedureResult.Insert(new ProcedureResult { Id = job.Id, Value = job.Lhs + job.Rhs }); return 0; }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/answers/rust.rs new file mode 100644 index 00000000000..fb3904b20eb --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/answers/rust.rs @@ -0,0 +1,41 @@ +use spacetimedb::{procedure, reducer, table, ProcedureContext, ReducerContext, ScheduleAt, Table}; +use std::time::Duration; + +#[table(accessor = procedure_result, public)] +pub struct ProcedureResult { + #[primary_key] + pub id: u64, + pub value: u32, +} + +#[table(accessor = procedure_job, scheduled(run_scheduled_procedure))] +pub struct ProcedureJob { + #[primary_key] + #[auto_inc] + pub scheduled_id: u64, + pub scheduled_at: ScheduleAt, + pub id: u64, + pub lhs: u32, + pub rhs: u32, +} + +#[reducer] +pub fn schedule_procedure(ctx: &ReducerContext, id: u64, lhs: u32, rhs: u32) { + ctx.db.procedure_job().insert(ProcedureJob { + scheduled_id: 0, + scheduled_at: ScheduleAt::Time(ctx.timestamp + Duration::from_millis(1)), + id, + lhs, + rhs, + }); +} + +#[procedure] +pub fn run_scheduled_procedure(ctx: &mut ProcedureContext, job: ProcedureJob) { + ctx.with_tx(|tx| { + tx.db.procedure_result().insert(ProcedureResult { + id: job.id, + value: job.lhs + job.rhs, + }); + }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/answers/typescript.ts new file mode 100644 index 00000000000..28d1108af48 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/answers/typescript.ts @@ -0,0 +1,27 @@ +import { ScheduleAt } from 'spacetimedb'; +import { schema, table, t } from 'spacetimedb/server'; + +const procedureResult = table({ name: 'procedure_result', public: true }, { + id: t.u64().primaryKey(), value: t.u32(), +}); +const procedureJob = table({ name: 'procedure_job', scheduled: (): any => runScheduledProcedure }, { + scheduledId: t.u64().primaryKey().autoInc(), scheduledAt: t.scheduleAt(), + id: t.u64(), lhs: t.u32(), rhs: t.u32(), +}); +const spacetimedb = schema({ procedureResult, procedureJob }); +export default spacetimedb; + +export const schedule_procedure = spacetimedb.reducer( + { id: t.u64(), lhs: t.u32(), rhs: t.u32() }, + (ctx, { id, lhs, rhs }) => ctx.db.procedureJob.insert({ + scheduledId: 0n, scheduledAt: ScheduleAt.time(ctx.timestamp.microsSinceUnixEpoch + 1_000n), id, lhs, rhs, + }) +); + +export const runScheduledProcedure = spacetimedb.procedure( + { job: procedureJob.rowType }, t.unit(), + (ctx, { job }) => { + ctx.withTx(tx => tx.db.procedureResult.insert({ id: job.id, value: job.lhs + job.rhs })); + return {}; + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/spec.rs new file mode 100644 index 00000000000..378c61676fd --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/spec.rs @@ -0,0 +1,33 @@ +use crate::eval::defaults::{ + default_schema_parity_scorers, make_eventually_sql_count_scorer, make_reducer_call_both_scorer, +}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec}; +use serde_json::json; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_reducer_call_both_scorer( + host_url, + file!(), + route_tag, + "schedule_procedure", + vec![json!(9), json!(7), json!(5)], + "schedule_procedure", + )); + let table = table_name("procedure_result", lang); + let id = ident("id", casing_for_lang(lang)); + let value = ident("value", casing_for_lang(lang)); + scorers.push(make_eventually_sql_count_scorer( + host_url, + file!(), + route_tag, + format!("SELECT COUNT(*) AS n FROM {table} WHERE {id}=9 AND {value}=12"), + 1, + "scheduled_procedure_completes", + Duration::from_secs(10), + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/csharp.txt new file mode 100644 index 00000000000..31d717818ff --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/csharp.txt @@ -0,0 +1,23 @@ +Write a SpacetimeDB backend module in C# that schedules a procedure which writes inside a transaction. + +TABLES +- ProcedureResult (public, accessor ProcedureResult) + - Fields: + - Id: ulong (primary key) + - Value: uint +- ProcedureJob (private, scheduled for RunScheduledProcedure, accessor ProcedureJob) + - Fields, in this order: + - ScheduledId: ulong (primary key, auto-increment) + - ScheduledAt: ScheduleAt + - Id: ulong + - Lhs: uint + - Rhs: uint + +REDUCERS +- ScheduleProcedure(id: ulong, lhs: uint, rhs: uint) + - Enqueue RunScheduledProcedure for one millisecond later + +PROCEDURE +- RunScheduledProcedure(job: ProcedureJob) -> void (scheduled procedure) + - Open a transaction + - Insert Id and Value lhs + rhs into ProcedureResult diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/rust.txt new file mode 100644 index 00000000000..f6d9ad78d8a --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/rust.txt @@ -0,0 +1,23 @@ +Write a SpacetimeDB backend module in Rust that schedules a procedure which writes inside a transaction. + +TABLES +- procedure_result (public), struct ProcedureResult + - Fields: + - id: u64 (primary key) + - value: u32 +- procedure_job (private, scheduled for run_scheduled_procedure), struct ProcedureJob + - Fields, in this order: + - scheduled_id: u64 (primary key, auto-increment) + - scheduled_at: ScheduleAt + - id: u64 + - lhs: u32 + - rhs: u32 + +REDUCERS +- schedule_procedure(id: u64, lhs: u32, rhs: u32) + - Enqueue run_scheduled_procedure for one millisecond later + +PROCEDURE +- run_scheduled_procedure(job: ProcedureJob) -> () (scheduled procedure) + - Open a transaction + - Insert id and value lhs + rhs into procedure_result diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/typescript.txt new file mode 100644 index 00000000000..97cf9738c3d --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_075_scheduled_procedure/tasks/typescript.txt @@ -0,0 +1,23 @@ +Write a SpacetimeDB backend module in TypeScript that schedules a procedure which writes inside a transaction. + +TABLES +- procedure_result (public) + - Fields: + - id: u64 (primary key) + - value: u32 +- procedure_job (private, scheduled for runScheduledProcedure) + - Fields, in this order: + - scheduledId: u64 (primary key, auto-increment) + - scheduledAt: ScheduleAt + - id: u64 + - lhs: u32 + - rhs: u32 + +REDUCERS +- schedule_procedure(id: u64, lhs: u32, rhs: u32) + - Enqueue runScheduledProcedure for one millisecond later + +PROCEDURE +- runScheduledProcedure(job: procedure_job row) -> unit (scheduled procedure) + - Open a transaction + - Insert id and value lhs + rhs into procedure_result diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/answers/csharp.cs new file mode 100644 index 00000000000..d3ce2fc9239 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/answers/csharp.cs @@ -0,0 +1,15 @@ +using SpacetimeDB; +#pragma warning disable STDB_UNSTABLE + +public static partial class Module +{ + [SpacetimeDB.HttpHandler] + public static HttpResponse Echo(HandlerContext ctx, HttpRequest request) => new( + 201, HttpVersion.Http11, + new List { new("content-type", System.Text.Encoding.UTF8.GetBytes("text/plain")) }, + HttpBody.FromString($"echo:{request.Body.ToStringUtf8Lossy()}") + ); + + [SpacetimeDB.HttpRouter] + public static Router Routes() => SpacetimeDB.Router.New().Post("/echo", Handlers.Echo); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/answers/rust.rs new file mode 100644 index 00000000000..5114d5fd4b4 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/answers/rust.rs @@ -0,0 +1,16 @@ +use spacetimedb::http::{handler, router, Body, HandlerContext, Request, Response, Router}; + +#[handler] +fn echo(_ctx: &mut HandlerContext, request: Request) -> Response { + let body = request.into_body().into_string_lossy(); + Response::builder() + .status(201) + .header("content-type", "text/plain") + .body(Body::from_bytes(format!("echo:{body}"))) + .unwrap() +} + +#[router] +fn routes() -> Router { + Router::new().post("/echo", echo) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/answers/typescript.ts new file mode 100644 index 00000000000..4cd17257190 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/answers/typescript.ts @@ -0,0 +1,9 @@ +import { Router, SyncResponse, schema } from 'spacetimedb/server'; + +const spacetimedb = schema({}); +export default spacetimedb; + +export const echo = spacetimedb.httpHandler((_ctx, request) => + new SyncResponse(`echo:${request.text()}`, { status: 201, headers: { 'content-type': 'text/plain' } }) +); +export const routes = spacetimedb.httpRouter(new Router().post('/echo', echo)); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/spec.rs new file mode 100644 index 00000000000..5d3c96379dc --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/spec.rs @@ -0,0 +1,17 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_http_route_parity_scorer}; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_http_route_parity_scorer( + host_url, + file!(), + route_tag, + vec![("POST", "/echo", Some("hello"))], + true, + "http_handler_response", + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/csharp.txt new file mode 100644 index 00000000000..519ad043afc --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/csharp.txt @@ -0,0 +1,11 @@ +Write a SpacetimeDB backend module in C# that defines and routes an inbound HTTP handler. + +HANDLER +- POST /echo + - Read the request body + - Return status 201 + - Return Content-Type text/plain + - Return body "echo:" followed by the input body + +ROUTER +- Register the POST /echo handler through an HTTP router diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/rust.txt new file mode 100644 index 00000000000..628b58a290b --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/rust.txt @@ -0,0 +1,11 @@ +Write a SpacetimeDB backend module in Rust that defines and routes an inbound HTTP handler. + +HANDLER +- POST /echo + - Read the request body + - Return status 201 + - Return Content-Type text/plain + - Return body "echo:" followed by the input body + +ROUTER +- Register the POST /echo handler through an HTTP router diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/typescript.txt new file mode 100644 index 00000000000..9595ae843f7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_076_http_handler/tasks/typescript.txt @@ -0,0 +1,11 @@ +Write a SpacetimeDB backend module in TypeScript that defines and routes an inbound HTTP handler. + +HANDLER +- POST /echo + - Read the request body + - Return status 201 + - Return Content-Type text/plain + - Return body "echo:" followed by the input body + +ROUTER +- Register the POST /echo handler through an HTTP router diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/answers/csharp.cs new file mode 100644 index 00000000000..b6c460e20a4 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/answers/csharp.cs @@ -0,0 +1,20 @@ +using SpacetimeDB; +#pragma warning disable STDB_UNSTABLE + +public static partial class Module +{ + [SpacetimeDB.HttpHandler] + public static HttpResponse ListItems(HandlerContext ctx, HttpRequest request) => new( + 200, HttpVersion.Http11, new List(), HttpBody.FromString("list") + ); + + [SpacetimeDB.HttpHandler] + public static HttpResponse CreateItem(HandlerContext ctx, HttpRequest request) => new( + 201, HttpVersion.Http11, new List(), HttpBody.FromString($"created:{request.Body.ToStringUtf8Lossy()}") + ); + + [SpacetimeDB.HttpRouter] + public static Router Routes() => SpacetimeDB.Router.New() + .Get("/items", Handlers.ListItems) + .Post("/items", Handlers.CreateItem); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/answers/rust.rs new file mode 100644 index 00000000000..83cabcf594a --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/answers/rust.rs @@ -0,0 +1,20 @@ +use spacetimedb::http::{handler, router, Body, HandlerContext, Request, Response, Router}; + +#[handler] +fn list_items(_ctx: &mut HandlerContext, _request: Request) -> Response { + Response::new(Body::from_bytes("list")) +} + +#[handler] +fn create_item(_ctx: &mut HandlerContext, request: Request) -> Response { + let body = request.into_body().into_string_lossy(); + Response::builder() + .status(201) + .body(Body::from_bytes(format!("created:{body}"))) + .unwrap() +} + +#[router] +fn routes() -> Router { + Router::new().get("/items", list_items).post("/items", create_item) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/answers/typescript.ts new file mode 100644 index 00000000000..1ab18ab0276 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/answers/typescript.ts @@ -0,0 +1,12 @@ +import { Router, SyncResponse, schema } from 'spacetimedb/server'; + +const spacetimedb = schema({}); +export default spacetimedb; + +export const listItems = spacetimedb.httpHandler((_ctx, _request) => new SyncResponse('list')); +export const createItem = spacetimedb.httpHandler((_ctx, request) => + new SyncResponse(`created:${request.text()}`, { status: 201 }) +); +export const routes = spacetimedb.httpRouter( + new Router().get('/items', listItems).post('/items', createItem) +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/spec.rs new file mode 100644 index 00000000000..740c6cc7419 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/spec.rs @@ -0,0 +1,22 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_http_route_parity_scorer}; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_http_route_parity_scorer( + host_url, + file!(), + route_tag, + vec![ + ("GET", "/items", None), + ("POST", "/items", Some("book")), + ("PUT", "/items", Some("no")), + ("GET", "/missing", None), + ], + false, + "router_method_matrix", + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/csharp.txt new file mode 100644 index 00000000000..e1dcc494794 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/csharp.txt @@ -0,0 +1,15 @@ +Write a SpacetimeDB backend module in C# that routes two HTTP methods on the same path. + +HANDLERS +- GET /items + - Return status 200 + - Return body "list" +- POST /items + - Read the request body + - Return status 201 + - Return body "created:" followed by the request body + +ROUTER +- Register both handlers for /items +- Reject unsupported methods on /items +- Reject unknown paths diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/rust.txt new file mode 100644 index 00000000000..ed040b322f9 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/rust.txt @@ -0,0 +1,15 @@ +Write a SpacetimeDB backend module in Rust that routes two HTTP methods on the same path. + +HANDLERS +- GET /items + - Return status 200 + - Return body "list" +- POST /items + - Read the request body + - Return status 201 + - Return body "created:" followed by the request body + +ROUTER +- Register both handlers for /items +- Reject unsupported methods on /items +- Reject unknown paths diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/typescript.txt new file mode 100644 index 00000000000..cfa7421bee6 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_077_http_router/tasks/typescript.txt @@ -0,0 +1,15 @@ +Write a SpacetimeDB backend module in TypeScript that routes two HTTP methods on the same path. + +HANDLERS +- GET /items + - Return status 200 + - Return body "list" +- POST /items + - Read the request body + - Return status 201 + - Return body "created:" followed by the request body + +ROUTER +- Register both handlers for /items +- Reject unsupported methods on /items +- Reject unknown paths diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/answers/csharp.cs new file mode 100644 index 00000000000..3e84f4089e0 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/answers/csharp.cs @@ -0,0 +1,33 @@ +using SpacetimeDB; +#pragma warning disable STDB_UNSTABLE + +public static partial class Module +{ + [Table(Accessor = "ProcessedEvent")] + public partial struct ProcessedEvent { [PrimaryKey] public string EventId; } + + [Table(Accessor = "WebhookState", Public = true)] + public partial struct WebhookState { [PrimaryKey] public string Key; public ulong LastSequence; public string Value; } + + [SpacetimeDB.HttpHandler] + public static HttpResponse Webhook(HandlerContext ctx, HttpRequest request) + { + var parts = request.Body.ToStringUtf8Lossy().Split('|', 3); + if (parts.Length != 3) return new(400, HttpVersion.Http11, new(), HttpBody.FromString("invalid")); + var eventId = parts[0]; var sequence = ulong.Parse(parts[1]); var value = parts[2]; + var outcome = ctx.WithTx(tx => { + if (tx.Db.ProcessedEvent.EventId.Find(eventId) is not null) return "duplicate"; + tx.Db.ProcessedEvent.Insert(new ProcessedEvent { EventId = eventId }); + var state = tx.Db.WebhookState.Key.Find("account"); + if (state is not null) { + if (sequence <= state.Value.LastSequence) return "stale"; + var row = state.Value; row.LastSequence = sequence; row.Value = value; tx.Db.WebhookState.Key.Update(row); + } else tx.Db.WebhookState.Insert(new WebhookState { Key = "account", LastSequence = sequence, Value = value }); + return "applied"; + }); + return new(200, HttpVersion.Http11, new(), HttpBody.FromString(outcome)); + } + + [SpacetimeDB.HttpRouter] + public static Router Routes() => SpacetimeDB.Router.New().Post("/webhook", Handlers.Webhook); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/answers/rust.rs new file mode 100644 index 00000000000..8be8e888c2b --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/answers/rust.rs @@ -0,0 +1,61 @@ +use spacetimedb::http::{handler, router, Body, HandlerContext, Request, Response, Router}; +use spacetimedb::{table, Table}; + +#[table(accessor = processed_event)] +pub struct ProcessedEvent { + #[primary_key] + pub event_id: String, +} + +#[table(accessor = webhook_state, public)] +pub struct WebhookState { + #[primary_key] + pub key: String, + pub last_sequence: u64, + pub value: String, +} + +#[handler] +fn webhook(ctx: &mut HandlerContext, request: Request) -> Response { + let body = request.into_body().into_string_lossy(); + let parts: Vec<_> = body.splitn(3, '|').collect(); + if parts.len() != 3 { + return Response::builder() + .status(400) + .body(Body::from_bytes("invalid")) + .unwrap(); + } + let event_id = parts[0].to_string(); + let sequence: u64 = parts[1].parse().expect("invalid sequence"); + let value = parts[2].to_string(); + let outcome = ctx.with_tx(|tx| { + if tx.db.processed_event().event_id().find(&event_id).is_some() { + return "duplicate"; + } + tx.db.processed_event().insert(ProcessedEvent { + event_id: event_id.clone(), + }); + let key = "account".to_string(); + if let Some(mut state) = tx.db.webhook_state().key().find(&key) { + if sequence <= state.last_sequence { + return "stale"; + } + state.last_sequence = sequence; + state.value = value.clone(); + tx.db.webhook_state().key().update(state); + } else { + tx.db.webhook_state().insert(WebhookState { + key, + last_sequence: sequence, + value: value.clone(), + }); + } + "applied" + }); + Response::new(Body::from_bytes(outcome)) +} + +#[router] +fn routes() -> Router { + Router::new().post("/webhook", webhook) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/answers/typescript.ts new file mode 100644 index 00000000000..5c748904ba5 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/answers/typescript.ts @@ -0,0 +1,27 @@ +import { Router, SyncResponse, schema, table, t } from 'spacetimedb/server'; + +const processedEvent = table({ name: 'processed_event' }, { eventId: t.string().primaryKey() }); +const webhookState = table({ name: 'webhook_state', public: true }, { + key: t.string().primaryKey(), lastSequence: t.u64(), value: t.string(), +}); +const spacetimedb = schema({ processedEvent, webhookState }); +export default spacetimedb; + +export const webhook = spacetimedb.httpHandler((ctx, request) => { + const parts = request.text().split('|', 3); + if (parts.length !== 3) return new SyncResponse('invalid', { status: 400 }); + const [eventId, sequenceText, value] = parts; + const sequence = BigInt(sequenceText); + const outcome = ctx.withTx(tx => { + if (tx.db.processedEvent.eventId.find(eventId)) return 'duplicate'; + tx.db.processedEvent.insert({ eventId }); + const state = tx.db.webhookState.key.find('account'); + if (state) { + if (sequence <= state.lastSequence) return 'stale'; + tx.db.webhookState.key.update({ ...state, lastSequence: sequence, value }); + } else tx.db.webhookState.insert({ key: 'account', lastSequence: sequence, value }); + return 'applied'; + }); + return new SyncResponse(outcome); +}); +export const routes = spacetimedb.httpRouter(new Router().post('/webhook', webhook)); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/spec.rs new file mode 100644 index 00000000000..178befb0192 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/spec.rs @@ -0,0 +1,35 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_http_route_parity_scorer, make_sql_count_only_scorer}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec}; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_http_route_parity_scorer( + host_url, + file!(), + route_tag, + vec![ + ("POST", "/webhook", Some("evt-1|2|new")), + ("POST", "/webhook", Some("evt-1|2|new")), + ("POST", "/webhook", Some("evt-2|1|old")), + ], + false, + "webhook_idempotency", + )); + let table = table_name("webhook_state", lang); + let key = ident("key", casing_for_lang(lang)); + let sequence = ident("last_sequence", casing_for_lang(lang)); + let value = ident("value", casing_for_lang(lang)); + scorers.push(make_sql_count_only_scorer( + host_url, + file!(), + route_tag, + format!("SELECT COUNT(*) AS n FROM {table} WHERE {key}='account' AND {sequence}=2 AND {value}='new'"), + 1, + "webhook_state_is_current", + Duration::from_secs(10), + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/csharp.txt new file mode 100644 index 00000000000..604a8d3eb13 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/csharp.txt @@ -0,0 +1,22 @@ +Write a SpacetimeDB backend module in C# that handles idempotent, ordered webhook events. + +TABLES +- ProcessedEvent (private, accessor ProcessedEvent) + - Fields: + - EventId: string (primary key) +- WebhookState (public, accessor WebhookState) + - Fields: + - Key: string (primary key) + - LastSequence: ulong + - Value: string + +HANDLER +- POST /webhook + - Accept a pipe-delimited body: event_id|sequence|value + - Store state under Key "account" + - For a newly applied event, return status 200 and body "applied" + - For a duplicate EventId, return status 200 and body "duplicate" without applying twice + - When sequence is not newer, return status 200 and body "stale" without replacing state + +ROUTER +- Register POST /webhook through an HTTP router diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/rust.txt new file mode 100644 index 00000000000..ece1fe91867 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/rust.txt @@ -0,0 +1,22 @@ +Write a SpacetimeDB backend module in Rust that handles idempotent, ordered webhook events. + +TABLES +- processed_event (private), struct ProcessedEvent + - Fields: + - event_id: String (primary key) +- webhook_state (public), struct WebhookState + - Fields: + - key: String (primary key) + - last_sequence: u64 + - value: String + +HANDLER +- POST /webhook + - Accept a pipe-delimited body: event_id|sequence|value + - Store state under key "account" + - For a newly applied event, return status 200 and body "applied" + - For a duplicate event_id, return status 200 and body "duplicate" without applying twice + - When sequence is not newer, return status 200 and body "stale" without replacing state + +ROUTER +- Register POST /webhook through an HTTP router diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/typescript.txt new file mode 100644 index 00000000000..dc053af7d9d --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_078_idempotent_webhook/tasks/typescript.txt @@ -0,0 +1,22 @@ +Write a SpacetimeDB backend module in TypeScript that handles idempotent, ordered webhook events. + +TABLES +- processed_event (private) + - Fields: + - eventId: string (primary key) +- webhook_state (public) + - Fields: + - key: string (primary key) + - lastSequence: u64 + - value: string + +HANDLER +- POST /webhook + - Accept a pipe-delimited body: event_id|sequence|value + - Store state under key "account" + - For a newly applied event, return status 200 and body "applied" + - For a duplicate eventId, return status 200 and body "duplicate" without applying twice + - When sequence is not newer, return status 200 and body "stale" without replacing state + +ROUTER +- Register POST /webhook through an HTTP router diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/csharp.cs new file mode 100644 index 00000000000..1970290d374 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/csharp.cs @@ -0,0 +1,49 @@ +using SpacetimeDB; +#pragma warning disable STDB_UNSTABLE + +public static partial class Module +{ + [Table(Accessor = "UploadedAsset", Public = true)] + public partial struct UploadedAsset + { + [PrimaryKey] public ulong Id; + public string Url; + public ulong Size; + public ushort Status; + public bool ResponseBodyPresent; + } + + [SpacetimeDB.HttpHandler] + public static HttpResponse Upload(HandlerContext ctx, HttpRequest request) => new( + 201, HttpVersion.Http11, new(), HttpBody.FromString("https://files.local/object-1") + ); + + [SpacetimeDB.HttpRouter] + public static Router Routes() => SpacetimeDB.Router.New().Post("/upload", Handlers.Upload); + + [SpacetimeDB.Procedure] + public static string UploadAndRegister(ProcedureContext ctx, string uploadUrl, byte[] data) + { + var request = new HttpRequest { + Uri = uploadUrl, + Method = SpacetimeDB.HttpMethod.Post, + Headers = new() { new HttpHeader("content-type", "application/octet-stream") }, + Body = new HttpBody(data), + }; + return ctx.Http.Send(request).Match(response => { + if (response.StatusCode < 200 || response.StatusCode >= 300) throw new Exception($"upload failed: {response.StatusCode}"); + var responseBodyPresent = response.Body.ToBytes().Length > 0; + ctx.WithTx(tx => { + tx.Db.UploadedAsset.Insert(new UploadedAsset { + Id = 1, + Url = uploadUrl, + Size = (ulong)data.Length, + Status = response.StatusCode, + ResponseBodyPresent = responseBodyPresent, + }); + return 0; + }); + return uploadUrl; + }, error => throw new Exception(error.Message)); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/rust.rs new file mode 100644 index 00000000000..6e389a9df60 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/rust.rs @@ -0,0 +1,50 @@ +use spacetimedb::http::{handler, router, Body, HandlerContext, Request, Response, Router}; +use spacetimedb::{procedure, table, ProcedureContext, Table}; + +#[table(accessor = uploaded_asset, public)] +pub struct UploadedAsset { + #[primary_key] + pub id: u64, + pub url: String, + pub size: u64, + pub status: u16, + pub response_body_present: bool, +} + +#[handler] +fn upload(_ctx: &mut HandlerContext, _request: Request) -> Response { + Response::builder() + .status(201) + .body(Body::from_bytes("https://files.local/object-1")) + .unwrap() +} + +#[router] +fn routes() -> Router { + Router::new().post("/upload", upload) +} + +#[procedure] +pub fn upload_and_register(ctx: &mut ProcedureContext, upload_url: String, data: Vec) -> String { + let request = Request::builder() + .method("POST") + .uri(upload_url.clone()) + .header("content-type", "application/octet-stream") + .body(Body::from_bytes(data.clone())) + .unwrap(); + let response = ctx.http.send(request).expect("upload failed"); + assert!(response.status().is_success(), "upload failed: {}", response.status()); + let status = response.status().as_u16(); + let response_body_present = !response.into_body().into_bytes().is_empty(); + let row_url = upload_url.clone(); + ctx.with_tx(|tx| { + tx.db.uploaded_asset().insert(UploadedAsset { + id: 1, + url: row_url.clone(), + size: data.len() as u64, + status, + response_body_present, + }); + }); + upload_url +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/typescript.ts new file mode 100644 index 00000000000..56e6e096c62 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/answers/typescript.ts @@ -0,0 +1,28 @@ +import { Router, SyncResponse, schema, table, t } from 'spacetimedb/server'; + +const uploadedAsset = table({ name: 'uploaded_asset', public: true }, { + id: t.u64().primaryKey(), url: t.string(), size: t.u64(), status: t.u16(), responseBodyPresent: t.bool(), +}); +const spacetimedb = schema({ uploadedAsset }); +export default spacetimedb; + +export const upload = spacetimedb.httpHandler((_ctx, _request) => + new SyncResponse('https://files.local/object-1', { status: 201 }) +); +export const routes = spacetimedb.httpRouter(new Router().post('/upload', upload)); + +export const upload_and_register = spacetimedb.procedure( + { uploadUrl: t.string(), data: t.array(t.u8()) }, t.string(), + (ctx, { uploadUrl, data }) => { + const response = ctx.http.fetch( + uploadUrl, + { method: 'POST', headers: { 'content-type': 'application/octet-stream' }, body: new Uint8Array(data) } + ); + if (response.status < 200 || response.status >= 300) throw new Error(`upload failed: ${response.status}`); + const responseBodyPresent = response.bytes().length > 0; + ctx.withTx(tx => tx.db.uploadedAsset.insert({ + id: 1n, url: uploadUrl, size: BigInt(data.length), status: response.status, responseBodyPresent, + })); + return uploadUrl; + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/spec.rs new file mode 100644 index 00000000000..efa06980138 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/spec.rs @@ -0,0 +1,42 @@ +use crate::eval::defaults::{ + default_schema_parity_scorers, make_call_output_parity_scorer, make_http_route_parity_scorer, + make_sql_count_only_scorer, +}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec}; +use serde_json::json; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_call_output_parity_scorer( + host_url, + file!(), + route_tag, + "upload_and_register", + vec![json!("https://postman-echo.com/post"), json!([1, 2, 3, 4])], + "upload_return_url", + )); + scorers.push(make_http_route_parity_scorer( + host_url, + file!(), + route_tag, + vec![("POST", "/upload", Some("payload"))], + false, + "upload_route", + )); + let table = table_name("uploaded_asset", lang); + let url = ident("url", casing_for_lang(lang)); + let size = ident("size", casing_for_lang(lang)); + let status = ident("status", casing_for_lang(lang)); + let response_body_present = ident("response_body_present", casing_for_lang(lang)); + scorers.push(make_sql_count_only_scorer( + host_url, file!(), route_tag, + format!( + "SELECT COUNT(*) AS n FROM {table} WHERE {url}='https://postman-echo.com/post' AND {size}=4 AND {status}=200 AND {response_body_present}=true" + ), + 1, "upload_metadata_stored", Duration::from_secs(10), + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/csharp.txt new file mode 100644 index 00000000000..5817b13ce75 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/csharp.txt @@ -0,0 +1,25 @@ +Write a SpacetimeDB backend module in C# that exposes an upload route and an external upload procedure. + +TABLE +- UploadedAsset (public, accessor UploadedAsset) + - Fields: + - Id: ulong (primary key) + - Url: string + - Size: ulong + - Status: ushort + - ResponseBodyPresent: bool + +HANDLER +- POST /upload + - Return status 201 + - Return body "https://files.local/object-1" + +PROCEDURE +- UploadAndRegister(uploadUrl: string, data: byte[]) -> string + - POST the binary data to the supplied public URL + - Require a successful response + - Open a transaction and store Id 1, the supplied URL, the byte length, the response status, and whether the response body is non-empty + - Return the supplied URL + +ROUTER +- Register POST /upload through an HTTP router diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/rust.txt new file mode 100644 index 00000000000..f528ede25ac --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/rust.txt @@ -0,0 +1,25 @@ +Write a SpacetimeDB backend module in Rust that exposes an upload route and an external upload procedure. + +TABLE +- uploaded_asset (public), struct UploadedAsset + - Fields: + - id: u64 (primary key) + - url: String + - size: u64 + - status: u16 + - response_body_present: bool + +HANDLER +- POST /upload + - Return status 201 + - Return body "https://files.local/object-1" + +PROCEDURE +- upload_and_register(upload_url: String, data: Vec) -> String + - POST the binary data to the supplied public URL + - Require a successful response + - Open a transaction and store id 1, the supplied URL, the byte length, the response status, and whether the response body is non-empty + - Return the supplied URL + +ROUTER +- Register POST /upload through an HTTP router diff --git a/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/typescript.txt new file mode 100644 index 00000000000..3f06b4e2dcb --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/procedures/t_079_external_upload_flow/tasks/typescript.txt @@ -0,0 +1,25 @@ +Write a SpacetimeDB backend module in TypeScript that exposes an upload route and an external upload procedure. + +TABLE +- uploaded_asset (public) + - Fields: + - id: u64 (primary key) + - url: string + - size: u64 + - status: u16 + - responseBodyPresent: bool + +HANDLER +- POST /upload + - Return status 201 + - Return body "https://files.local/object-1" + +PROCEDURE +- upload_and_register(uploadUrl: string, data: u8[]) -> string + - POST the binary data to the supplied public URL + - Require a successful response + - Open a transaction and store id 1, the supplied URL, the byte length, the response status, and whether the response body is non-empty + - Return the supplied URL + +ROUTER +- Register POST /upload through an HTTP router diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/csharp.txt index 03b091f6fe2..3d88dc5c931 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/csharp.txt @@ -1,14 +1,14 @@ Write a SpacetimeDB backend module in C# that defines a Product table with a BTree-indexed Price column, a result table, and a reducer that filters products within a price range and writes matches to the result table. TABLES -- Product +- Product (private) - Struct: Product - Fields: - Id: ulong (primary key, AutoInc) - Name: string - Price: uint (index BTree) -- PriceRangeResult +- PriceRangeResult (private) - Struct: PriceRangeResult - Fields: - ProductId: ulong (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/rust.txt index 7396da179b4..48d8f2f6f3f 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/rust.txt @@ -1,14 +1,14 @@ Write a SpacetimeDB backend module in Rust that defines a product table with a BTree-indexed price column, a result table, and a reducer that filters products within a price range and writes matches to the result table. TABLES -- product +- product (private) - Struct: Product - Fields: - id: u64 (primary key, auto_inc) - name: String - price: u32 (index btree) -- price_range_result +- price_range_result (private) - Struct: PriceRangeResult - Fields: - product_id: u64 (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/typescript.txt index fdc20994867..0faeeb9f4d3 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_032_range_query/tasks/typescript.txt @@ -1,13 +1,13 @@ Write a SpacetimeDB backend module in TypeScript that defines a product table with a BTree-indexed price column, a result table, and a reducer that filters products within a price range and writes matches to the result table. TABLES -- product +- product (private) - Fields: - id: number (u64, primary key, autoInc) - name: string - price: number (u32, index btree) -- price_range_result +- price_range_result (private) - Fields: - productId: number (u64, primary key) - name: string diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/csharp.txt index f59eca1f12c..d8b797ed2af 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/csharp.txt @@ -1,14 +1,14 @@ Write a SpacetimeDB backend module in C# that defines a Player table with scores, a Leaderboard result table, and a reducer that finds the top N players by score (descending) and writes them to the result table. TABLES -- Player +- Player (private) - Struct: Player - Fields: - Id: ulong (primary key, AutoInc) - Name: string - Score: ulong -- Leaderboard +- Leaderboard (private) - Struct: LeaderboardEntry - Fields: - Rank: uint (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/rust.txt index cfc38860762..3a6ea9986f8 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/rust.txt @@ -1,14 +1,14 @@ Write a SpacetimeDB backend module in Rust that defines a player table with scores, a leaderboard result table, and a reducer that finds the top N players by score (descending) and writes them to the result table. TABLES -- player +- player (private) - Struct: Player - Fields: - id: u64 (primary key, auto_inc) - name: String - score: u64 -- leaderboard +- leaderboard (private) - Struct: LeaderboardEntry - Fields: - rank: u32 (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/typescript.txt index d888f39528e..83e8368d6af 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_033_sort_and_limit/tasks/typescript.txt @@ -1,13 +1,13 @@ Write a SpacetimeDB backend module in TypeScript that defines a player table with scores, a leaderboard result table, and a reducer that finds the top N players by score (descending) and writes them to the result table. TABLES -- player +- player (private) - Fields: - id: number (u64, primary key, autoInc) - name: string - score: number (u64) -- leaderboard +- leaderboard (private) - Fields: - rank: number (u32, primary key) - playerName: string diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/csharp.txt index c89d54ed9a6..59fe2f4a20e 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/csharp.txt @@ -1,14 +1,14 @@ Write a SpacetimeDB backend module in C# that defines a Task table and a result table, with a reducer that finds the first incomplete task and writes it to the result table. TABLES -- Task +- Task (private) - Struct: Task - Fields: - Id: ulong (primary key, AutoInc) - Title: string - Completed: bool -- FirstIncomplete +- FirstIncomplete (private) - Struct: FirstIncomplete - Fields: - TaskId: ulong (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/rust.txt index dec8df0b43e..3fe727c5b2b 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/rust.txt @@ -1,14 +1,14 @@ Write a SpacetimeDB backend module in Rust that defines a task table and a result table, with a reducer that finds the first incomplete task and writes it to the result table. TABLES -- task +- task (private) - Struct: Task - Fields: - id: u64 (primary key, auto_inc) - title: String - completed: bool -- first_incomplete +- first_incomplete (private) - Struct: FirstIncomplete - Fields: - task_id: u64 (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/typescript.txt index 940e400a22e..59f899d0829 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_034_find_first/tasks/typescript.txt @@ -1,13 +1,13 @@ Write a SpacetimeDB backend module in TypeScript that defines a task table and a result table, with a reducer that finds the first incomplete task and writes it to the result table. TABLES -- task +- task (private) - Fields: - id: number (u64, primary key, autoInc) - title: string - completed: boolean -- first_incomplete +- first_incomplete (private) - Fields: - taskId: number (u64, primary key) - title: string diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/csharp.txt index 5ac96c35a88..0fb60362dc5 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/csharp.txt @@ -1,14 +1,14 @@ Write a SpacetimeDB backend module in C# that defines an Order table and a result table, with a reducer that collects all distinct Category values from orders into the result table. TABLES -- Order +- Order (private) - Struct: Order - Fields: - Id: ulong (primary key, AutoInc) - Category: string - Amount: uint -- DistinctCategory +- DistinctCategory (private) - Struct: DistinctCategory - Fields: - Category: string (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/rust.txt index bc7d9e915cd..44eee546d60 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/rust.txt @@ -1,14 +1,14 @@ Write a SpacetimeDB backend module in Rust that defines an order table and a result table, with a reducer that collects all distinct category values from orders into the result table. TABLES -- order +- order (private) - Struct: Order - Fields: - id: u64 (primary key, auto_inc) - category: String - amount: u32 -- distinct_category +- distinct_category (private) - Struct: DistinctCategory - Fields: - category: String (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/typescript.txt index fe4cd415ed6..809d808c5d5 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_035_select_distinct/tasks/typescript.txt @@ -1,13 +1,13 @@ Write a SpacetimeDB backend module in TypeScript that defines an order table and a result table, with a reducer that collects all distinct category values from orders into the result table. TABLES -- order +- order (private) - Fields: - id: number (u64, primary key, autoInc) - category: string - amount: number (u32) -- distinct_category +- distinct_category (private) - Fields: - category: string (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/csharp.txt index 1508c532fc8..693d4d3faee 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/csharp.txt @@ -1,14 +1,14 @@ Write a SpacetimeDB backend module in C# that defines a User table and a stats table, with a reducer that counts total users and active users without collecting into a list. TABLES -- User +- User (private) - Struct: User - Fields: - Id: ulong (primary key, AutoInc) - Name: string - Active: bool -- UserStats +- UserStats (private) - Struct: UserStats - Fields: - Key: string (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/rust.txt index 24c987810d7..f6a3b3dfbfb 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/rust.txt @@ -1,14 +1,14 @@ Write a SpacetimeDB backend module in Rust that defines a user table and a stats table, with a reducer that counts total users and active users without collecting into a Vec. TABLES -- user +- user (private) - Struct: User - Fields: - id: u64 (primary key, auto_inc) - name: String - active: bool -- user_stats +- user_stats (private) - Struct: UserStats - Fields: - key: String (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/typescript.txt index 83d9957a584..6018961e127 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_036_count_without_collect/tasks/typescript.txt @@ -1,13 +1,13 @@ Write a SpacetimeDB backend module in TypeScript that defines a user table and a stats table, with a reducer that counts total users and active users without collecting into an array. TABLES -- user +- user (private) - Fields: - id: number (u64, primary key, autoInc) - name: string - active: boolean -- user_stats +- user_stats (private) - Fields: - key: string (primary key) - count: number (u64) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/csharp.txt index c1f49d42fac..7aefcaaea5b 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/csharp.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/csharp.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in C# that defines an EventLog table with a multi-column BTree index and a result table, with a reducer that filters events by both category and severity level. TABLES -- EventLog +- EventLog (private) - Struct: EventLog - Fields: - Id: ulong (primary key, AutoInc) @@ -10,7 +10,7 @@ TABLES - Message: string - Index: multi-column btree index on (Category, Severity) -- FilteredEvent +- FilteredEvent (private) - Struct: FilteredEvent - Fields: - EventId: ulong (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/rust.txt index 88516a7ab47..2e7ea47806f 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/rust.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/rust.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in Rust that defines an event log table with a multi-column BTree index and a result table, with a reducer that filters events by both category and severity level. TABLES -- event_log +- event_log (private) - Struct: EventLog - Fields: - id: u64 (primary key, auto_inc) @@ -10,7 +10,7 @@ TABLES - message: String - Index: multi-column btree index on (category, severity) -- filtered_event +- filtered_event (private) - Struct: FilteredEvent - Fields: - event_id: u64 (primary key) diff --git a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/typescript.txt index bf9a3845053..70ddcea8163 100644 --- a/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/typescript.txt +++ b/tools/xtask-llm-benchmark/src/benchmarks/queries/t_037_multi_column_filter/tasks/typescript.txt @@ -1,7 +1,7 @@ Write a SpacetimeDB backend module in TypeScript that defines an event log table with a multi-column BTree index and a result table, with a reducer that filters events by both category and severity level. TABLES -- event_log +- event_log (private) - Fields: - id: number (u64, primary key, autoInc) - category: string @@ -9,7 +9,7 @@ TABLES - message: string - Index: multi-column btree index on (category, severity) -- filtered_event +- filtered_event (private) - Fields: - eventId: number (u64, primary key) - message: string diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/answers/csharp.cs new file mode 100644 index 00000000000..a873dcc0bdc --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/answers/csharp.cs @@ -0,0 +1,33 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Account", Public = true)] + public partial struct Account { [PrimaryKey] public ulong Id; public long Balance; } + + [Table(Accessor = "TransferRequest", Public = true)] + public partial struct TransferRequest + { + [PrimaryKey] public string RequestId; + public ulong FromId; + public ulong ToId; + public long Amount; + } + + [Reducer] + public static void CreateAccount(ReducerContext ctx, ulong id, long balance) => + ctx.Db.Account.Insert(new Account { Id = id, Balance = balance }); + + [Reducer] + public static void Transfer(ReducerContext ctx, string requestId, ulong fromId, ulong toId, long amount) + { + if (ctx.Db.TransferRequest.RequestId.Find(requestId) is not null) return; + if (amount <= 0 || fromId == toId) throw new Exception("invalid transfer"); + var source = ctx.Db.Account.Id.Find(fromId) ?? throw new Exception("source account not found"); + var to = ctx.Db.Account.Id.Find(toId) ?? throw new Exception("destination account not found"); + if (source.Balance < amount) throw new Exception("insufficient balance"); + ctx.Db.Account.Id.Update(source with { Balance = source.Balance - amount }); + ctx.Db.Account.Id.Update(to with { Balance = to.Balance + amount }); + ctx.Db.TransferRequest.Insert(new TransferRequest { RequestId = requestId, FromId = fromId, ToId = toId, Amount = amount }); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/answers/rust.rs new file mode 100644 index 00000000000..71d0741fe2d --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/answers/rust.rs @@ -0,0 +1,53 @@ +use spacetimedb::{reducer, table, ReducerContext, Table}; + +#[table(accessor = account, public)] +pub struct Account { + #[primary_key] + pub id: u64, + pub balance: i64, +} + +#[table(accessor = transfer_request, public)] +pub struct TransferRequest { + #[primary_key] + pub request_id: String, + pub from_id: u64, + pub to_id: u64, + pub amount: i64, +} + +#[reducer] +pub fn create_account(ctx: &ReducerContext, id: u64, balance: i64) { + ctx.db.account().insert(Account { id, balance }); +} + +#[reducer] +pub fn transfer(ctx: &ReducerContext, request_id: String, from_id: u64, to_id: u64, amount: i64) -> Result<(), String> { + if ctx.db.transfer_request().request_id().find(&request_id).is_some() { + return Ok(()); + } + if amount <= 0 || from_id == to_id { + return Err("invalid transfer".into()); + } + let mut from = ctx.db.account().id().find(from_id).ok_or("source account not found")?; + let mut to = ctx + .db + .account() + .id() + .find(to_id) + .ok_or("destination account not found")?; + if from.balance < amount { + return Err("insufficient balance".into()); + } + from.balance -= amount; + to.balance += amount; + ctx.db.account().id().update(from); + ctx.db.account().id().update(to); + ctx.db.transfer_request().insert(TransferRequest { + request_id, + from_id, + to_id, + amount, + }); + Ok(()) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/answers/typescript.ts new file mode 100644 index 00000000000..092c4fc6c3f --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/answers/typescript.ts @@ -0,0 +1,29 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const account = table({ name: 'account', public: true }, { id: t.u64().primaryKey(), balance: t.i64() }); +const transferRequest = table( + { name: 'transfer_request', public: true }, + { requestId: t.string().primaryKey(), fromId: t.u64(), toId: t.u64(), amount: t.i64() } +); +const spacetimedb = schema({ account, transferRequest }); +export default spacetimedb; + +export const create_account = spacetimedb.reducer( + { id: t.u64(), balance: t.i64() }, + (ctx, { id, balance }) => ctx.db.account.insert({ id, balance }) +); + +export const transfer = spacetimedb.reducer( + { requestId: t.string(), fromId: t.u64(), toId: t.u64(), amount: t.i64() }, + (ctx, { requestId, fromId, toId, amount }) => { + if (ctx.db.transferRequest.requestId.find(requestId)) return; + if (amount <= 0n || fromId === toId) throw new Error('invalid transfer'); + const from = ctx.db.account.id.find(fromId); + const to = ctx.db.account.id.find(toId); + if (!from || !to) throw new Error('account not found'); + if (from.balance < amount) throw new Error('insufficient balance'); + ctx.db.account.id.update({ ...from, balance: from.balance - amount }); + ctx.db.account.id.update({ ...to, balance: to.balance + amount }); + ctx.db.transferRequest.insert({ requestId, fromId, toId, amount }); + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/spec.rs new file mode 100644 index 00000000000..409784e7db5 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/spec.rs @@ -0,0 +1,44 @@ +use crate::eval::defaults::{ + default_schema_parity_scorers, make_reducer_call_both_scorer, make_reducer_data_parity_scorer, +}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec, ReducerDataParityConfig, SqlBuilder}; +use serde_json::json; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + for (id, balance) in [(1, 100), (2, 25)] { + scorers.push(make_reducer_call_both_scorer( + host_url, + file!(), + route_tag, + "create_account", + vec![json!(id), json!(balance)], + "create_account", + )); + } + let sql = SqlBuilder::new(casing_for_lang(lang)); + let account = table_name("account", lang); + let columns = sql.cols(&["id", "balance"]).join(", "); + let id = ident("id", casing_for_lang(lang)); + let query = format!("SELECT {columns} FROM {account} WHERE {id}=1 OR {id}=2"); + + for scorer_id in ["transfer_once", "duplicate_transfer_is_noop"] { + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "transfer".into(), + args: vec![json!("request-1"), json!(1), json!(2), json!(40)], + select_query: query.clone(), + id_str: scorer_id, + collapse_ws: true, + timeout: Duration::from_secs(10), + }, + )); + } + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/csharp.txt new file mode 100644 index 00000000000..2791f08bec4 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/csharp.txt @@ -0,0 +1,22 @@ +Write a SpacetimeDB backend module in C# that performs atomic, idempotent transfers between accounts. + +TABLES +- Account (public, accessor Account) + - Fields: + - Id: ulong (primary key) + - Balance: long +- TransferRequest (public, accessor TransferRequest) + - Fields: + - RequestId: string (primary key) + - FromId: ulong + - ToId: ulong + - Amount: long + +REDUCERS +- CreateAccount(id: ulong, balance: long) + - Insert an Account with the supplied values +- Transfer(requestId: string, fromId: ulong, toId: ulong, amount: long) + - Reject invalid or insufficient transfers + - Update both Account balances atomically + - Record RequestId in TransferRequest + - If RequestId is already recorded, make no changes diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/rust.txt new file mode 100644 index 00000000000..5dc491ee6be --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/rust.txt @@ -0,0 +1,22 @@ +Write a SpacetimeDB backend module in Rust that performs atomic, idempotent transfers between accounts. + +TABLES +- account (public), struct Account + - Fields: + - id: u64 (primary key) + - balance: i64 +- transfer_request (public), struct TransferRequest + - Fields: + - request_id: String (primary key) + - from_id: u64 + - to_id: u64 + - amount: i64 + +REDUCERS +- create_account(id: u64, balance: i64) + - Insert an account with the supplied values +- transfer(request_id: String, from_id: u64, to_id: u64, amount: i64) + - Reject invalid or insufficient transfers + - Update both account balances atomically + - Record request_id in transfer_request + - If request_id is already recorded, make no changes diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/typescript.txt new file mode 100644 index 00000000000..fc87b8ae076 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_055_atomic_idempotent_transfer/tasks/typescript.txt @@ -0,0 +1,22 @@ +Write a SpacetimeDB backend module in TypeScript that performs atomic, idempotent transfers between accounts. + +TABLES +- account (public) + - Fields: + - id: u64 (primary key) + - balance: i64 +- transfer_request (public) + - Fields: + - requestId: string (primary key) + - fromId: u64 + - toId: u64 + - amount: i64 + +REDUCERS +- create_account(id: u64, balance: i64) + - Insert an account with the supplied values +- transfer(requestId: string, fromId: u64, toId: u64, amount: i64) + - Reject invalid or insufficient transfers + - Update both account balances atomically + - Record requestId in transfer_request + - If requestId is already recorded, make no changes diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/answers/csharp.cs new file mode 100644 index 00000000000..154f8749e4c --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/answers/csharp.cs @@ -0,0 +1,32 @@ +using SpacetimeDB; + +public static partial class Module +{ + [SpacetimeDB.Type] + public partial struct Preferences + { + public string Theme; + public bool EmailNotifications; + public string Timezone; + } + + [Table(Accessor = "Profile", Public = true)] + public partial struct Profile { [PrimaryKey] public ulong Id; public Preferences Preferences; } + + [Reducer] + public static void CreateProfile(ReducerContext ctx, ulong id, string theme, bool emailNotifications, string timezone) => + ctx.Db.Profile.Insert(new Profile + { + Id = id, + Preferences = new Preferences { Theme = theme, EmailNotifications = emailNotifications, Timezone = timezone } + }); + + [Reducer] + public static void UpdateTheme(ReducerContext ctx, ulong id, string theme) + { + var profile = ctx.Db.Profile.Id.Find(id) ?? throw new Exception("profile not found"); + var preferences = profile.Preferences; + preferences.Theme = theme; + ctx.Db.Profile.Id.Update(profile with { Preferences = preferences }); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/answers/rust.rs new file mode 100644 index 00000000000..0a59cae7261 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/answers/rust.rs @@ -0,0 +1,35 @@ +use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table}; + +#[derive(SpacetimeType)] +pub struct Preferences { + pub theme: String, + pub email_notifications: bool, + pub timezone: String, +} + +#[table(accessor = profile, public)] +pub struct Profile { + #[primary_key] + pub id: u64, + pub preferences: Preferences, +} + +#[reducer] +pub fn create_profile(ctx: &ReducerContext, id: u64, theme: String, email_notifications: bool, timezone: String) { + ctx.db.profile().insert(Profile { + id, + preferences: Preferences { + theme, + email_notifications, + timezone, + }, + }); +} + +#[reducer] +pub fn update_theme(ctx: &ReducerContext, id: u64, theme: String) -> Result<(), String> { + let mut profile = ctx.db.profile().id().find(id).ok_or("profile not found")?; + profile.preferences.theme = theme; + ctx.db.profile().id().update(profile); + Ok(()) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/answers/typescript.ts new file mode 100644 index 00000000000..545e3b89e3b --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/answers/typescript.ts @@ -0,0 +1,32 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const Preferences = t.object('Preferences', { + theme: t.string(), + emailNotifications: t.bool(), + timezone: t.string(), +}); +const profile = table( + { name: 'profile', public: true }, + { id: t.u64().primaryKey(), preferences: Preferences } +); +const spacetimedb = schema({ profile }); +export default spacetimedb; + +export const create_profile = spacetimedb.reducer( + { id: t.u64(), theme: t.string(), emailNotifications: t.bool(), timezone: t.string() }, + (ctx, args) => ctx.db.profile.insert({ id: args.id, preferences: { + theme: args.theme, emailNotifications: args.emailNotifications, timezone: args.timezone, + } }) +); + +export const update_theme = spacetimedb.reducer( + { id: t.u64(), theme: t.string() }, + (ctx, { id, theme }) => { + const found = ctx.db.profile.id.find(id); + if (!found) throw new Error('profile not found'); + ctx.db.profile.id.update({ + ...found, + preferences: { ...found.preferences, theme }, + }); + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/spec.rs new file mode 100644 index 00000000000..eb1023b0c7a --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/spec.rs @@ -0,0 +1,38 @@ +use crate::eval::defaults::{ + default_schema_parity_scorers, make_reducer_call_both_scorer, make_reducer_data_parity_scorer, +}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec, ReducerDataParityConfig, SqlBuilder}; +use serde_json::json; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_reducer_call_both_scorer( + host_url, + file!(), + route_tag, + "create_profile", + vec![json!(1), json!("light"), json!(true), json!("UTC")], + "create_profile", + )); + let sql = SqlBuilder::new(casing_for_lang(lang)); + let table = table_name("profile", lang); + let columns = sql.cols(&["id", "preferences"]).join(", "); + let id = ident("id", casing_for_lang(lang)); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "update_theme".into(), + args: vec![json!(1), json!("dark")], + select_query: format!("SELECT {columns} FROM {table} WHERE {id}=1"), + id_str: "nested_siblings_preserved", + collapse_ws: true, + timeout: Duration::from_secs(10), + }, + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/tasks/csharp.txt new file mode 100644 index 00000000000..508762521fe --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/tasks/csharp.txt @@ -0,0 +1,20 @@ +Write a SpacetimeDB backend module in C# that stores and updates nested profile preferences. + +TYPES +- Preferences + - Theme: string + - EmailNotifications: bool + - Timezone: string + +TABLE +- Profile (public, accessor Profile) + - Fields: + - Id: ulong (primary key) + - Preferences: Preferences + +REDUCERS +- CreateProfile(id: ulong, theme: string, emailNotifications: bool, timezone: string) + - Insert a Profile with all three preference fields +- UpdateTheme(id: ulong, theme: string) + - Change only Preferences.Theme + - Preserve Preferences.EmailNotifications and Preferences.Timezone diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/tasks/rust.txt new file mode 100644 index 00000000000..8c0f974aba3 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/tasks/rust.txt @@ -0,0 +1,20 @@ +Write a SpacetimeDB backend module in Rust that stores and updates nested profile preferences. + +TYPES +- Preferences + - theme: String + - email_notifications: bool + - timezone: String + +TABLE +- profile (public), struct Profile + - Fields: + - id: u64 (primary key) + - preferences: Preferences + +REDUCERS +- create_profile(id: u64, theme: String, email_notifications: bool, timezone: String) + - Insert a profile with all three preference fields +- update_theme(id: u64, theme: String) + - Change only preferences.theme + - Preserve preferences.email_notifications and preferences.timezone diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/tasks/typescript.txt new file mode 100644 index 00000000000..cec8627b326 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_056_nested_update/tasks/typescript.txt @@ -0,0 +1,20 @@ +Write a SpacetimeDB backend module in TypeScript that stores and updates nested profile preferences. + +TYPES +- Preferences + - theme: string + - emailNotifications: boolean + - timezone: string + +TABLE +- profile (public) + - Fields: + - id: u64 (primary key) + - preferences: Preferences + +REDUCERS +- create_profile(id: u64, theme: string, emailNotifications: boolean, timezone: string) + - Insert a profile with all three preference fields +- update_theme(id: u64, theme: string) + - Change only preferences.theme + - Preserve preferences.emailNotifications and preferences.timezone diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/answers/csharp.cs new file mode 100644 index 00000000000..48de17c6c05 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/answers/csharp.cs @@ -0,0 +1,36 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Workspace", Public = true)] public partial struct Workspace { [PrimaryKey] public ulong Id; } + [Table(Accessor = "Project", Public = true)] public partial struct Project { [PrimaryKey] public ulong Id; [SpacetimeDB.Index.BTree] public ulong WorkspaceId; } + [Table(Accessor = "TaskItem", Public = true)] public partial struct TaskItem { [PrimaryKey] public ulong Id; [SpacetimeDB.Index.BTree] public ulong ProjectId; } + [Table(Accessor = "TaskNote", Public = true)] public partial struct TaskNote { [PrimaryKey] public ulong Id; [SpacetimeDB.Index.BTree] public ulong TaskId; } + + [Reducer] + public static void Seed(ReducerContext ctx) + { + foreach (ulong id in new ulong[] { 1, 2 }) + { + ctx.Db.Workspace.Insert(new Workspace { Id = id }); + ctx.Db.Project.Insert(new Project { Id = id, WorkspaceId = id }); + ctx.Db.TaskItem.Insert(new TaskItem { Id = id, ProjectId = id }); + ctx.Db.TaskNote.Insert(new TaskNote { Id = id, TaskId = id }); + } + } + + [Reducer] + public static void DeleteWorkspace(ReducerContext ctx, ulong id) + { + foreach (var project in ctx.Db.Project.WorkspaceId.Filter(id).ToList()) + { + foreach (var task in ctx.Db.TaskItem.ProjectId.Filter(project.Id).ToList()) + { + foreach (var note in ctx.Db.TaskNote.TaskId.Filter(task.Id).ToList()) ctx.Db.TaskNote.Id.Delete(note.Id); + ctx.Db.TaskItem.Id.Delete(task.Id); + } + ctx.Db.Project.Id.Delete(project.Id); + } + ctx.Db.Workspace.Id.Delete(id); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/answers/rust.rs new file mode 100644 index 00000000000..910f00814d6 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/answers/rust.rs @@ -0,0 +1,55 @@ +use spacetimedb::{reducer, table, ReducerContext, Table}; + +#[table(accessor = workspace, public)] +pub struct Workspace { + #[primary_key] + pub id: u64, +} +#[table(accessor = project, public)] +pub struct Project { + #[primary_key] + pub id: u64, + #[index(btree)] + pub workspace_id: u64, +} +#[table(accessor = task_item, public)] +pub struct TaskItem { + #[primary_key] + pub id: u64, + #[index(btree)] + pub project_id: u64, +} +#[table(accessor = task_note, public)] +pub struct TaskNote { + #[primary_key] + pub id: u64, + #[index(btree)] + pub task_id: u64, +} + +#[reducer] +pub fn seed(ctx: &ReducerContext) { + for id in [1, 2] { + ctx.db.workspace().insert(Workspace { id }); + ctx.db.project().insert(Project { id, workspace_id: id }); + ctx.db.task_item().insert(TaskItem { id, project_id: id }); + ctx.db.task_note().insert(TaskNote { id, task_id: id }); + } +} + +#[reducer] +pub fn delete_workspace(ctx: &ReducerContext, id: u64) { + let projects: Vec<_> = ctx.db.project().workspace_id().filter(id).collect(); + for project in projects { + let tasks: Vec<_> = ctx.db.task_item().project_id().filter(project.id).collect(); + for task in tasks { + let notes: Vec<_> = ctx.db.task_note().task_id().filter(task.id).collect(); + for note in notes { + ctx.db.task_note().id().delete(note.id); + } + ctx.db.task_item().id().delete(task.id); + } + ctx.db.project().id().delete(project.id); + } + ctx.db.workspace().id().delete(id); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/answers/typescript.ts new file mode 100644 index 00000000000..db060c4a5e9 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/answers/typescript.ts @@ -0,0 +1,28 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const workspace = table({ name: 'workspace', public: true }, { id: t.u64().primaryKey() }); +const project = table({ name: 'project', public: true }, { id: t.u64().primaryKey(), workspaceId: t.u64().index('btree') }); +const taskItem = table({ name: 'task_item', public: true }, { id: t.u64().primaryKey(), projectId: t.u64().index('btree') }); +const taskNote = table({ name: 'task_note', public: true }, { id: t.u64().primaryKey(), taskId: t.u64().index('btree') }); +const spacetimedb = schema({ workspace, project, taskItem, taskNote }); +export default spacetimedb; + +export const seed = spacetimedb.reducer(ctx => { + for (const id of [1n, 2n]) { + ctx.db.workspace.insert({ id }); + ctx.db.project.insert({ id, workspaceId: id }); + ctx.db.taskItem.insert({ id, projectId: id }); + ctx.db.taskNote.insert({ id, taskId: id }); + } +}); + +export const delete_workspace = spacetimedb.reducer({ id: t.u64() }, (ctx, { id }) => { + for (const project of [...ctx.db.project.workspaceId.filter(id)]) { + for (const task of [...ctx.db.taskItem.projectId.filter(project.id)]) { + for (const note of [...ctx.db.taskNote.taskId.filter(task.id)]) ctx.db.taskNote.id.delete(note.id); + ctx.db.taskItem.id.delete(task.id); + } + ctx.db.project.id.delete(project.id); + } + ctx.db.workspace.id.delete(id); +}); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/spec.rs new file mode 100644 index 00000000000..03ab60fe4ad --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/spec.rs @@ -0,0 +1,54 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_call_both_scorer, make_sql_count_only_scorer}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec}; +use serde_json::json; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_reducer_call_both_scorer( + host_url, + file!(), + route_tag, + "seed", + vec![], + "seed_nested_rows", + )); + scorers.push(make_reducer_call_both_scorer( + host_url, + file!(), + route_tag, + "delete_workspace", + vec![json!(1)], + "delete_workspace", + )); + let id = ident("id", casing_for_lang(lang)); + for (table, count_id, preserved_id) in [ + ("workspace", "workspace_count", "workspace_two_preserved"), + ("project", "project_count", "project_two_preserved"), + ("task_item", "task_count", "task_two_preserved"), + ("task_note", "note_count", "note_two_preserved"), + ] { + let table = table_name(table, lang); + scorers.push(make_sql_count_only_scorer( + host_url, + file!(), + route_tag, + format!("SELECT COUNT(*) AS n FROM {table}"), + 1, + count_id, + Duration::from_secs(10), + )); + scorers.push(make_sql_count_only_scorer( + host_url, + file!(), + route_tag, + format!("SELECT COUNT(*) AS n FROM {table} WHERE {id}=2"), + 1, + preserved_id, + Duration::from_secs(10), + )); + } + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/tasks/csharp.txt new file mode 100644 index 00000000000..49a158aaae6 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/tasks/csharp.txt @@ -0,0 +1,25 @@ +Write a SpacetimeDB backend module in C# that performs a nested cascade delete across normalized tables. + +TABLES +- Workspace (public, accessor Workspace) + - Fields: + - Id: ulong (primary key) +- Project (public, accessor Project) + - Fields: + - Id: ulong (primary key) + - WorkspaceId: ulong (btree index) +- TaskItem (public, accessor TaskItem) + - Fields: + - Id: ulong (primary key) + - ProjectId: ulong (btree index) +- TaskNote (public, accessor TaskNote) + - Fields: + - Id: ulong (primary key) + - TaskId: ulong (btree index) + +REDUCERS +- Seed() + - Create two independent Workspace -> Project -> TaskItem -> TaskNote chains using IDs 1 and 2 +- DeleteWorkspace(id: ulong) + - Delete the selected Workspace and every level of its descendants + - Preserve the unrelated chain diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/tasks/rust.txt new file mode 100644 index 00000000000..98897315991 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/tasks/rust.txt @@ -0,0 +1,25 @@ +Write a SpacetimeDB backend module in Rust that performs a nested cascade delete across normalized tables. + +TABLES +- workspace (public), struct Workspace + - Fields: + - id: u64 (primary key) +- project (public), struct Project + - Fields: + - id: u64 (primary key) + - workspace_id: u64 (btree index) +- task_item (public), struct TaskItem + - Fields: + - id: u64 (primary key) + - project_id: u64 (btree index) +- task_note (public), struct TaskNote + - Fields: + - id: u64 (primary key) + - task_id: u64 (btree index) + +REDUCERS +- seed() + - Create two independent workspace -> project -> task_item -> task_note chains using IDs 1 and 2 +- delete_workspace(id: u64) + - Delete the selected workspace and every level of its descendants + - Preserve the unrelated chain diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/tasks/typescript.txt new file mode 100644 index 00000000000..c33a2395d4b --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_057_nested_cascade_delete/tasks/typescript.txt @@ -0,0 +1,25 @@ +Write a SpacetimeDB backend module in TypeScript that performs a nested cascade delete across normalized tables. + +TABLES +- workspace (public) + - Fields: + - id: u64 (primary key) +- project (public) + - Fields: + - id: u64 (primary key) + - workspaceId: u64 (btree index) +- task_item (public) + - Fields: + - id: u64 (primary key) + - projectId: u64 (btree index) +- task_note (public) + - Fields: + - id: u64 (primary key) + - taskId: u64 (btree index) + +REDUCERS +- seed() + - Create two independent workspace -> project -> task_item -> task_note chains using IDs 1 and 2 +- delete_workspace(id: u64) + - Delete the selected workspace and every level of its descendants + - Preserve the unrelated chain diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/answers/csharp.cs new file mode 100644 index 00000000000..042dbd0ca33 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/answers/csharp.cs @@ -0,0 +1,37 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "WorkItem", Public = true)] + public partial struct WorkItem { [PrimaryKey] public ulong Id; [SpacetimeDB.Index.BTree] public ulong GroupId; } + + [Table(Accessor = "DeleteJob", Scheduled = nameof(RunDeleteBatch), ScheduledAt = nameof(DeleteJob.ScheduledAt))] + public partial struct DeleteJob + { + [PrimaryKey, AutoInc] public ulong ScheduledId; + public ScheduleAt ScheduledAt; + public ulong GroupId; + } + + private static void Enqueue(ReducerContext ctx, ulong groupId) => ctx.Db.DeleteJob.Insert(new DeleteJob + { + ScheduledAt = new ScheduleAt.Time(ctx.Timestamp + new TimeDuration { Microseconds = 1_000 }), + GroupId = groupId, + }); + + [Reducer] + public static void SeedGroup(ReducerContext ctx, ulong groupId, uint count) + { + for (uint offset = 0; offset < count; offset++) ctx.Db.WorkItem.Insert(new WorkItem { Id = groupId * 100 + offset, GroupId = groupId }); + } + + [Reducer] + public static void RequestDelete(ReducerContext ctx, ulong groupId) => Enqueue(ctx, groupId); + + [Reducer] + public static void RunDeleteBatch(ReducerContext ctx, DeleteJob job) + { + foreach (var row in ctx.Db.WorkItem.GroupId.Filter(job.GroupId).Take(2).ToList()) ctx.Db.WorkItem.Id.Delete(row.Id); + if (ctx.Db.WorkItem.GroupId.Filter(job.GroupId).Any()) Enqueue(ctx, job.GroupId); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/answers/rust.rs new file mode 100644 index 00000000000..d4418cc5539 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/answers/rust.rs @@ -0,0 +1,53 @@ +use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; +use std::time::Duration; + +#[table(accessor = work_item, public)] +pub struct WorkItem { + #[primary_key] + pub id: u64, + #[index(btree)] + pub group_id: u64, +} + +#[table(accessor = delete_job, scheduled(run_delete_batch))] +pub struct DeleteJob { + #[primary_key] + #[auto_inc] + pub scheduled_id: u64, + pub scheduled_at: ScheduleAt, + pub group_id: u64, +} + +fn enqueue(ctx: &ReducerContext, group_id: u64) { + ctx.db.delete_job().insert(DeleteJob { + scheduled_id: 0, + scheduled_at: ScheduleAt::Time(ctx.timestamp + Duration::from_millis(1)), + group_id, + }); +} + +#[reducer] +pub fn seed_group(ctx: &ReducerContext, group_id: u64, count: u32) { + for offset in 0..count { + ctx.db.work_item().insert(WorkItem { + id: group_id * 100 + offset as u64, + group_id, + }); + } +} + +#[reducer] +pub fn request_delete(ctx: &ReducerContext, group_id: u64) { + enqueue(ctx, group_id); +} + +#[reducer] +pub fn run_delete_batch(ctx: &ReducerContext, job: DeleteJob) { + let rows: Vec<_> = ctx.db.work_item().group_id().filter(job.group_id).take(2).collect(); + for row in rows { + ctx.db.work_item().id().delete(row.id); + } + if ctx.db.work_item().group_id().filter(job.group_id).next().is_some() { + enqueue(ctx, job.group_id); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/answers/typescript.ts new file mode 100644 index 00000000000..ab06c4599a4 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/answers/typescript.ts @@ -0,0 +1,37 @@ +import { ScheduleAt } from 'spacetimedb'; +import { schema, table, t } from 'spacetimedb/server'; +import type { InferSchema, ReducerCtx } from 'spacetimedb/server'; + +const workItem = table( + { name: 'work_item', public: true }, + { id: t.u64().primaryKey(), groupId: t.u64().index('btree') } +); +const deleteJob = table( + { name: 'delete_job', scheduled: (): any => runDeleteBatch }, + { scheduledId: t.u64().primaryKey().autoInc(), scheduledAt: t.scheduleAt(), groupId: t.u64() } +); +const spacetimedb = schema({ workItem, deleteJob }); +export default spacetimedb; +type Ctx = ReducerCtx>; + +function enqueue(ctx: Ctx, groupId: bigint) { + ctx.db.deleteJob.insert({ + scheduledId: 0n, + scheduledAt: ScheduleAt.time(ctx.timestamp.microsSinceUnixEpoch + 1_000n), + groupId, + }); +} + +export const seed_group = spacetimedb.reducer({ groupId: t.u64(), count: t.u32() }, (ctx, { groupId, count }) => { + for (let offset = 0; offset < count; offset++) ctx.db.workItem.insert({ id: groupId * 100n + BigInt(offset), groupId }); +}); +export const request_delete = spacetimedb.reducer({ groupId: t.u64() }, (ctx, { groupId }) => enqueue(ctx, groupId)); +export const runDeleteBatch = spacetimedb.reducer({ timer: deleteJob.rowType }, (ctx, { timer }) => { + let deleted = 0; + for (const row of ctx.db.workItem.groupId.filter(timer.groupId)) { + if (deleted === 2) break; + ctx.db.workItem.id.delete(row.id); + deleted++; + } + if (ctx.db.workItem.groupId.filter(timer.groupId)[Symbol.iterator]().next().done === false) enqueue(ctx, timer.groupId); +}); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/spec.rs new file mode 100644 index 00000000000..1bde4f8755c --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/spec.rs @@ -0,0 +1,40 @@ +use crate::eval::defaults::{ + default_schema_parity_scorers, make_eventually_sql_count_scorer, make_reducer_call_both_scorer, +}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec}; +use serde_json::json; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_reducer_call_both_scorer( + host_url, + file!(), + route_tag, + "seed_group", + vec![json!(7), json!(5)], + "seed_group", + )); + scorers.push(make_reducer_call_both_scorer( + host_url, + file!(), + route_tag, + "request_delete", + vec![json!(7)], + "request_delete", + )); + let table = table_name("work_item", lang); + let group_id = ident("group_id", casing_for_lang(lang)); + scorers.push(make_eventually_sql_count_scorer( + host_url, + file!(), + route_tag, + format!("SELECT COUNT(*) AS n FROM {table} WHERE {group_id}=7"), + 0, + "scheduled_batches_finish", + Duration::from_secs(10), + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/csharp.txt new file mode 100644 index 00000000000..94180f3182d --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/csharp.txt @@ -0,0 +1,20 @@ +Write a SpacetimeDB backend module in C# that deletes a logical group in scheduled batches of at most two rows. + +TABLES +- WorkItem (public, accessor WorkItem) + - Fields: + - Id: ulong (primary key) + - GroupId: ulong (btree index) +- DeleteJob (private, scheduled for RunDeleteBatch, accessor DeleteJob) + - Fields: + - ScheduledId: ulong (primary key, auto-increment) + - ScheduledAt: ScheduleAt + - GroupId: ulong + +REDUCERS +- SeedGroup(groupId: ulong, count: uint) + - Assign deterministic unique Id values to the inserted rows +- RequestDelete(groupId: ulong) +- RunDeleteBatch(job: DeleteJob) (scheduled reducer) + - Delete at most two WorkItem rows matching the scheduled GroupId + - If matching rows remain, enqueue another one-shot DeleteJob diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/rust.txt new file mode 100644 index 00000000000..c88990defec --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/rust.txt @@ -0,0 +1,20 @@ +Write a SpacetimeDB backend module in Rust that deletes a logical group in scheduled batches of at most two rows. + +TABLES +- work_item (public), struct WorkItem + - Fields: + - id: u64 (primary key) + - group_id: u64 (btree index) +- delete_job (private, scheduled for run_delete_batch), struct DeleteJob + - Fields: + - scheduled_id: u64 (primary key, auto-increment) + - scheduled_at: ScheduleAt + - group_id: u64 + +REDUCERS +- seed_group(group_id: u64, count: u32) + - Assign deterministic unique id values to the inserted rows +- request_delete(group_id: u64) +- run_delete_batch(job: DeleteJob) (scheduled reducer) + - Delete at most two work_item rows matching the scheduled group_id + - If matching rows remain, enqueue another one-shot delete_job diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/typescript.txt new file mode 100644 index 00000000000..e34652dfbd1 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_058_batched_delete/tasks/typescript.txt @@ -0,0 +1,20 @@ +Write a SpacetimeDB backend module in TypeScript that deletes a logical group in scheduled batches of at most two rows. + +TABLES +- work_item (public) + - Fields: + - id: u64 (primary key) + - groupId: u64 (btree index) +- delete_job (private, scheduled for runDeleteBatch) + - Fields: + - scheduledId: u64 (primary key, auto-increment) + - scheduledAt: ScheduleAt + - groupId: u64 + +REDUCERS +- seed_group(groupId: u64, count: u32) + - Assign deterministic unique id values to the inserted rows +- request_delete(groupId: u64) +- runDeleteBatch(timer: delete_job row) (scheduled reducer) + - Delete at most two work_item rows matching the scheduled groupId + - If matching rows remain, enqueue another one-shot delete_job diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/answers/csharp.cs new file mode 100644 index 00000000000..2c27bcb4159 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/answers/csharp.cs @@ -0,0 +1,20 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "GeneratedValue", Public = true)] + public partial struct GeneratedValue + { + [PrimaryKey, AutoInc] public ulong Id; + public Timestamp CreatedAt; + public long RandomValue; + } + + [Reducer] + public static void Generate(ReducerContext ctx) => ctx.Db.GeneratedValue.Insert(new GeneratedValue + { + Id = 0, + CreatedAt = ctx.Timestamp, + RandomValue = ctx.Rng.NextInt64(1, long.MaxValue), + }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/answers/rust.rs new file mode 100644 index 00000000000..50baa498033 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/answers/rust.rs @@ -0,0 +1,20 @@ +use spacetimedb::{reducer, table, ReducerContext, Table, Timestamp}; + +#[table(accessor = generated_value, public)] +pub struct GeneratedValue { + #[primary_key] + #[auto_inc] + pub id: u64, + pub created_at: Timestamp, + pub random_value: u64, +} + +#[reducer] +pub fn generate(ctx: &ReducerContext) { + let random_value: u64 = ctx.random(); + ctx.db.generated_value().insert(GeneratedValue { + id: 0, + created_at: ctx.timestamp, + random_value: random_value.max(1), + }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/answers/typescript.ts new file mode 100644 index 00000000000..92b487c6964 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/answers/typescript.ts @@ -0,0 +1,16 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const generatedValue = table( + { name: 'generated_value', public: true }, + { id: t.u64().primaryKey().autoInc(), createdAt: t.timestamp(), randomValue: t.i64() } +); +const spacetimedb = schema({ generatedValue }); +export default spacetimedb; + +export const generate = spacetimedb.reducer(ctx => { + ctx.db.generatedValue.insert({ + id: 0n, + createdAt: ctx.timestamp, + randomValue: BigInt(ctx.random.integerInRange(1, Number.MAX_SAFE_INTEGER)), + }); +}); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/spec.rs new file mode 100644 index 00000000000..af961a7c99a --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/spec.rs @@ -0,0 +1,60 @@ +use crate::eval::defaults::{ + default_schema_parity_scorers, make_reducer_sql_count_scorer, make_sql_distinct_rows_scorer, + make_sql_output_excludes_scorer, +}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec, ReducerSqlCountConfig}; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + let table = table_name("generated_value", lang); + let casing = casing_for_lang(lang); + scorers.push(make_reducer_sql_count_scorer( + host_url, + ReducerSqlCountConfig { + src_file: file!(), + route_tag, + reducer: "generate".into(), + args: vec![], + sql_count_query: format!("SELECT COUNT(*) AS n FROM {table}"), + expected_count: 1, + id_str: "generated_value_recorded", + timeout: Duration::from_secs(10), + }, + )); + scorers.push(make_reducer_sql_count_scorer( + host_url, + ReducerSqlCountConfig { + src_file: file!(), + route_tag, + reducer: "generate".into(), + args: vec![], + sql_count_query: format!("SELECT COUNT(*) AS n FROM {table}"), + expected_count: 2, + id_str: "second_generated_value_recorded", + timeout: Duration::from_secs(10), + }, + )); + let random_value = ident("random_value", casing); + scorers.push(make_sql_distinct_rows_scorer( + host_url, + file!(), + route_tag, + format!("SELECT {random_value} FROM {table}"), + 2, + "successive_random_values_differ", + Duration::from_secs(10), + )); + let created_at = ident("created_at", casing); + scorers.push(make_sql_output_excludes_scorer( + host_url, + file!(), + route_tag, + format!("SELECT {created_at} FROM {table}"), + vec!["1970-01-01T00:00:00+00:00".into()], + "context_timestamp_recorded", + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/csharp.txt new file mode 100644 index 00000000000..e0d32d485f9 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/csharp.txt @@ -0,0 +1,14 @@ +Write a SpacetimeDB backend module in C# that records deterministic reducer time and randomness. + +TABLE +- GeneratedValue (public, accessor GeneratedValue) + - Fields: + - Id: ulong (primary key, auto-increment) + - CreatedAt: Timestamp + - RandomValue: long + +REDUCERS +- Generate() + - Store ctx.Timestamp in CreatedAt + - Store a random long from ctx.Rng in RandomValue + - Do not use a host clock or a separately seeded Random diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/rust.txt new file mode 100644 index 00000000000..ee479b62124 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/rust.txt @@ -0,0 +1,14 @@ +Write a SpacetimeDB backend module in Rust that records deterministic reducer time and randomness. + +TABLE +- generated_value (public), struct GeneratedValue + - Fields: + - id: u64 (primary key, auto-increment) + - created_at: Timestamp + - random_value: u64 + +REDUCERS +- generate() + - Store the reducer context timestamp in created_at + - Store a u64 from the reducer context RNG in random_value + - Do not use a host clock or a separately seeded RNG diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/typescript.txt new file mode 100644 index 00000000000..6d3d105d6e2 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_059_deterministic_context/tasks/typescript.txt @@ -0,0 +1,14 @@ +Write a SpacetimeDB backend module in TypeScript that records deterministic reducer time and randomness. + +TABLE +- generated_value (public) + - Fields: + - id: u64 (primary key, auto-increment) + - createdAt: Timestamp + - randomValue: i64 + +REDUCERS +- generate() + - Store ctx.timestamp in createdAt + - Store a random integer from ctx.random in randomValue + - Do not use Date, Math.random, or a separately seeded RNG diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/answers/csharp.cs new file mode 100644 index 00000000000..709ad50df32 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/answers/csharp.cs @@ -0,0 +1,16 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "CommandResult", Public = true)] + public partial struct CommandResult + { + [PrimaryKey] public string RequestId; + public bool Success; + public string Message; + } + + [Reducer] + public static void RunCommand(ReducerContext ctx, string requestId, int value) => + ctx.Db.CommandResult.Insert(new CommandResult { RequestId = requestId, Success = true, Message = $"value={value}" }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/answers/rust.rs new file mode 100644 index 00000000000..146e5eefbd3 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/answers/rust.rs @@ -0,0 +1,18 @@ +use spacetimedb::{reducer, table, ReducerContext, Table}; + +#[table(accessor = command_result, public)] +pub struct CommandResult { + #[primary_key] + pub request_id: String, + pub success: bool, + pub message: String, +} + +#[reducer] +pub fn run_command(ctx: &ReducerContext, request_id: String, value: i32) { + ctx.db.command_result().insert(CommandResult { + request_id, + success: true, + message: format!("value={value}"), + }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/answers/typescript.ts new file mode 100644 index 00000000000..784561fc821 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/answers/typescript.ts @@ -0,0 +1,17 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const commandResult = table( + { name: 'command_result', public: true }, + { requestId: t.string().primaryKey(), success: t.bool(), message: t.string() } +); +const spacetimedb = schema({ commandResult }); +export default spacetimedb; + +export const run_command = spacetimedb.reducer( + { requestId: t.string(), value: t.i32() }, + (ctx, { requestId, value }) => ctx.db.commandResult.insert({ + requestId, + success: true, + message: `value=${value}`, + }) +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/spec.rs new file mode 100644 index 00000000000..5b5bf42e027 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/spec.rs @@ -0,0 +1,28 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_data_parity_scorer}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec, ReducerDataParityConfig, SqlBuilder}; +use serde_json::json; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + let sql = SqlBuilder::new(casing_for_lang(lang)); + let table = table_name("command_result", lang); + let columns = sql.cols(&["request_id", "success", "message"]).join(", "); + let request_id = ident("request_id", casing_for_lang(lang)); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "run_command".into(), + args: vec![json!("req-1"), json!(7)], + select_query: format!("SELECT {columns} FROM {table} WHERE {request_id}='req-1'"), + id_str: "result_row_written", + collapse_ws: true, + timeout: Duration::from_secs(10), + }, + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/tasks/csharp.txt new file mode 100644 index 00000000000..b104afbfed6 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/tasks/csharp.txt @@ -0,0 +1,14 @@ +Write a SpacetimeDB backend module in C# that writes reducer results to a table. + +TABLE +- CommandResult (public, accessor CommandResult) + - Fields: + - RequestId: string (primary key) + - Success: bool + - Message: string + +REDUCERS +- RunCommand(requestId: string, value: int) + - Reducers do not return application data + - Insert a CommandResult row with Success set to true + - Set Message to "value=" using the supplied value diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/tasks/rust.txt new file mode 100644 index 00000000000..4f59c30455c --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/tasks/rust.txt @@ -0,0 +1,14 @@ +Write a SpacetimeDB backend module in Rust that writes reducer results to a table. + +TABLE +- command_result (public), struct CommandResult + - Fields: + - request_id: String (primary key) + - success: bool + - message: String + +REDUCERS +- run_command(request_id: String, value: i32) + - Reducers do not return application data + - Insert a command_result row with success set to true + - Set message to "value=" using the supplied value diff --git a/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/tasks/typescript.txt new file mode 100644 index 00000000000..011e5797d96 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/reducers/t_060_reducer_result_table/tasks/typescript.txt @@ -0,0 +1,14 @@ +Write a SpacetimeDB backend module in TypeScript that writes reducer results to a table. + +TABLE +- command_result (public) + - Fields: + - requestId: string (primary key) + - success: boolean + - message: string + +REDUCERS +- run_command(requestId: string, value: i32) + - Reducers do not return application data + - Insert a command_result row with success set to true + - Set message to "value=" using the supplied value diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/answers/csharp.cs new file mode 100644 index 00000000000..2a89d17f3ce --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/answers/csharp.cs @@ -0,0 +1,20 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "CollectionOwner", Public = true)] + public partial struct CollectionOwner + { + [PrimaryKey, AutoInc] public ulong Id; + public string Name; + } + + [Table(Accessor = "ChildItem", Public = true)] + [SpacetimeDB.Index.BTree(Accessor = "by_owner", Columns = new[] { nameof(OwnerId) })] + public partial struct ChildItem + { + [PrimaryKey, AutoInc] public ulong Id; + public ulong OwnerId; + public string Value; + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/answers/rust.rs new file mode 100644 index 00000000000..97a2b33446c --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/answers/rust.rs @@ -0,0 +1,22 @@ +use spacetimedb::table; + +#[table(accessor = collection_owner, public)] +pub struct CollectionOwner { + #[primary_key] + #[auto_inc] + pub id: u64, + pub name: String, +} + +#[table( + accessor = child_item, + public, + index(accessor = by_owner, btree(columns = [owner_id])) +)] +pub struct ChildItem { + #[primary_key] + #[auto_inc] + pub id: u64, + pub owner_id: u64, + pub value: String, +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/answers/typescript.ts new file mode 100644 index 00000000000..c3933734b99 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/answers/typescript.ts @@ -0,0 +1,24 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const collectionOwner = table( + { name: 'collection_owner', public: true }, + { + id: t.u64().primaryKey().autoInc(), + name: t.string(), + } +); + +const childItem = table( + { + name: 'child_item', + public: true, + indexes: [{ accessor: 'byOwner', algorithm: 'btree', columns: ['ownerId'] }], + }, + { + id: t.u64().primaryKey().autoInc(), + ownerId: t.u64(), + value: t.string(), + } +); + +export default schema({ collectionOwner, childItem }); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/spec.rs new file mode 100644 index 00000000000..e5466e2ccd7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/spec.rs @@ -0,0 +1,8 @@ +use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + default_schema_parity_scorers(host_url, file!(), route_tag) + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/tasks/csharp.txt new file mode 100644 index 00000000000..4d2038b695a --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/tasks/csharp.txt @@ -0,0 +1,13 @@ +Write a SpacetimeDB backend module in C# that models an unbounded child collection with normalized tables. + +TABLES +- CollectionOwner (public, accessor CollectionOwner) + - Id: ulong (primary key, auto increment) + - Name: string +- ChildItem (public, accessor ChildItem) + - Id: ulong (primary key, auto increment) + - OwnerId: ulong + - Value: string + - index by_owner: btree(OwnerId) + +Do not store child items in an array on CollectionOwner. No reducers are required. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/tasks/rust.txt new file mode 100644 index 00000000000..91a4b60a0b2 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/tasks/rust.txt @@ -0,0 +1,13 @@ +Write a SpacetimeDB backend module in Rust that models an unbounded child collection with normalized tables. + +TABLES +- collection_owner (public), struct CollectionOwner + - id: u64 (primary key, auto_inc) + - name: String +- child_item (public), struct ChildItem + - id: u64 (primary key, auto_inc) + - owner_id: u64 + - value: String + - index by_owner: btree(owner_id) + +Do not store child items in an array on CollectionOwner. No reducers are required. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/tasks/typescript.txt new file mode 100644 index 00000000000..2ef2b0935a8 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_047_normalized_collection/tasks/typescript.txt @@ -0,0 +1,13 @@ +Write a SpacetimeDB backend module in TypeScript that models an unbounded child collection with normalized tables. + +TABLES +- collection_owner (public) + - id: u64 (primary key, auto increment) + - name: string +- child_item (public) + - id: u64 (primary key, auto increment) + - ownerId: u64 + - value: string + - index byOwner: btree(ownerId) + +Do not store child items in an array on collection_owner. No reducers are required. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/answers/csharp.cs new file mode 100644 index 00000000000..98978397b1f --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/answers/csharp.cs @@ -0,0 +1,22 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "UserProfile", Public = true)] + public partial struct UserProfile + { + [PrimaryKey] public Identity Identity; + public string DisplayName; + public Timestamp CreatedAt; + } + + [Table(Accessor = "ConnectionPresence", Public = true)] + [SpacetimeDB.Index.BTree(Accessor = "by_user", Columns = new[] { nameof(UserIdentity) })] + public partial struct ConnectionPresence + { + [PrimaryKey] public ConnectionId ConnectionId; + public Identity UserIdentity; + public Timestamp LastHeartbeat; + public bool Online; + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/answers/rust.rs new file mode 100644 index 00000000000..0bf35e7ff3a --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/answers/rust.rs @@ -0,0 +1,22 @@ +use spacetimedb::{table, ConnectionId, Identity, Timestamp}; + +#[table(accessor = user_profile, public)] +pub struct UserProfile { + #[primary_key] + pub identity: Identity, + pub display_name: String, + pub created_at: Timestamp, +} + +#[table( + accessor = connection_presence, + public, + index(accessor = by_user, btree(columns = [user_identity])) +)] +pub struct ConnectionPresence { + #[primary_key] + pub connection_id: ConnectionId, + pub user_identity: Identity, + pub last_heartbeat: Timestamp, + pub online: bool, +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/answers/typescript.ts new file mode 100644 index 00000000000..babde8b0f53 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/answers/typescript.ts @@ -0,0 +1,26 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const userProfile = table( + { name: 'user_profile', public: true }, + { + identity: t.identity().primaryKey(), + displayName: t.string(), + createdAt: t.timestamp(), + } +); + +const connectionPresence = table( + { + name: 'connection_presence', + public: true, + indexes: [{ accessor: 'byUser', algorithm: 'btree', columns: ['userIdentity'] }], + }, + { + connectionId: t.connectionId().primaryKey(), + userIdentity: t.identity(), + lastHeartbeat: t.timestamp(), + online: t.bool(), + } +); + +export default schema({ userProfile, connectionPresence }); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/spec.rs new file mode 100644 index 00000000000..e5466e2ccd7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/spec.rs @@ -0,0 +1,8 @@ +use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + default_schema_parity_scorers(host_url, file!(), route_tag) + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/tasks/csharp.txt new file mode 100644 index 00000000000..cea83835fb2 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/tasks/csharp.txt @@ -0,0 +1,15 @@ +Write a SpacetimeDB backend module in C# that separates stable profile data from frequently updated connection presence. + +TABLES +- UserProfile (public, accessor UserProfile) + - Identity: Identity (primary key) + - DisplayName: string + - CreatedAt: Timestamp +- ConnectionPresence (public, accessor ConnectionPresence) + - ConnectionId: ConnectionId (primary key) + - UserIdentity: Identity + - LastHeartbeat: Timestamp + - Online: bool + - index by_user: btree(UserIdentity) + +Do not put LastHeartbeat or Online on UserProfile. No reducers are required. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/tasks/rust.txt new file mode 100644 index 00000000000..51ffd285a05 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/tasks/rust.txt @@ -0,0 +1,15 @@ +Write a SpacetimeDB backend module in Rust that separates stable profile data from frequently updated connection presence. + +TABLES +- user_profile (public), struct UserProfile + - identity: Identity (primary key) + - display_name: String + - created_at: Timestamp +- connection_presence (public), struct ConnectionPresence + - connection_id: ConnectionId (primary key) + - user_identity: Identity + - last_heartbeat: Timestamp + - online: bool + - index by_user: btree(user_identity) + +Do not put last_heartbeat or online on UserProfile. No reducers are required. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/tasks/typescript.txt new file mode 100644 index 00000000000..2d21d61606d --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_048_heartbeat_isolation/tasks/typescript.txt @@ -0,0 +1,15 @@ +Write a SpacetimeDB backend module in TypeScript that separates stable profile data from frequently updated connection presence. + +TABLES +- user_profile (public) + - identity: Identity (primary key) + - displayName: string + - createdAt: Timestamp +- connection_presence (public) + - connectionId: ConnectionId (primary key) + - userIdentity: Identity + - lastHeartbeat: Timestamp + - online: boolean + - index byUser: btree(userIdentity) + +Do not put lastHeartbeat or online on user_profile. No reducers are required. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/answers/csharp.cs new file mode 100644 index 00000000000..36bb8b1fc67 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/answers/csharp.cs @@ -0,0 +1,30 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "BlobRecord", Public = true)] + [SpacetimeDB.Index.BTree(Accessor = "by_owner", Columns = new[] { nameof(Owner) })] + public partial struct BlobRecord + { + [PrimaryKey, AutoInc] public ulong Id; + public Identity Owner; + public string Filename; + public string MimeType; + public ulong Size; + public List Data; + } + + [Reducer] + public static void StoreBlob(ReducerContext ctx, string filename, string mimeType, List data) + { + ctx.Db.BlobRecord.Insert(new BlobRecord + { + Id = 0, + Owner = ctx.Sender, + Filename = filename, + MimeType = mimeType, + Size = (ulong)data.Count, + Data = data, + }); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/answers/rust.rs new file mode 100644 index 00000000000..031b3645072 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/answers/rust.rs @@ -0,0 +1,29 @@ +use spacetimedb::{reducer, table, Identity, ReducerContext, Table}; + +#[table( + accessor = blob_record, + public, + index(accessor = by_owner, btree(columns = [owner])) +)] +pub struct BlobRecord { + #[primary_key] + #[auto_inc] + pub id: u64, + pub owner: Identity, + pub filename: String, + pub mime_type: String, + pub size: u64, + pub data: Vec, +} + +#[reducer] +pub fn store_blob(ctx: &ReducerContext, filename: String, mime_type: String, data: Vec) { + ctx.db.blob_record().insert(BlobRecord { + id: 0, + owner: ctx.sender(), + filename, + mime_type, + size: data.len() as u64, + data, + }); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/answers/typescript.ts new file mode 100644 index 00000000000..e6982c6745e --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/answers/typescript.ts @@ -0,0 +1,34 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const blobRecord = table( + { + name: 'blob_record', + public: true, + indexes: [{ accessor: 'byOwner', algorithm: 'btree', columns: ['owner'] }], + }, + { + id: t.u64().primaryKey().autoInc(), + owner: t.identity(), + filename: t.string(), + mimeType: t.string(), + size: t.u64(), + data: t.array(t.u8()), + } +); + +const spacetimedb = schema({ blobRecord }); +export default spacetimedb; + +export const store_blob = spacetimedb.reducer( + { filename: t.string(), mimeType: t.string(), data: t.array(t.u8()) }, + (ctx, { filename, mimeType, data }) => { + ctx.db.blobRecord.insert({ + id: 0n, + owner: ctx.sender, + filename, + mimeType, + size: BigInt(data.length), + data, + }); + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/spec.rs new file mode 100644 index 00000000000..3161c7ce1df --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/spec.rs @@ -0,0 +1,34 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_data_parity_scorer}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec, ReducerDataParityConfig, SqlBuilder}; +use serde_json::json; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + let sql = SqlBuilder::new(casing_for_lang(lang)); + let table = table_name("blob_record", lang); + let columns = sql.cols(&["filename", "mime_type", "size", "data"]).join(", "); + let filename = ident("filename", casing_for_lang(lang)); + let select_query = format!("SELECT {columns} FROM {table} WHERE {filename}='eval.bin'"); + + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "store_blob".into(), + args: vec![ + json!("eval.bin"), + json!("application/octet-stream"), + json!([0, 1, 2, 127, 128, 255]), + ], + select_query, + id_str: "binary_round_trip", + collapse_ws: true, + timeout: Duration::from_secs(10), + }, + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/csharp.txt new file mode 100644 index 00000000000..687052ddd10 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/csharp.txt @@ -0,0 +1,17 @@ +Write a SpacetimeDB backend module in C# that stores binary content and its metadata. + +TABLE +- BlobRecord (public, accessor BlobRecord) + - Fields: + - Id: ulong (primary key, auto-increment) + - Owner: Identity (btree index by_owner) + - Filename: string + - MimeType: string + - Size: ulong + - Data: List + +REDUCERS +- StoreBlob(filename: string, mimeType: string, data: List) + - Store the sender as Owner + - Derive Size from the byte count + - Insert the bytes unchanged diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/rust.txt new file mode 100644 index 00000000000..9126ebc5b62 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/rust.txt @@ -0,0 +1,17 @@ +Write a SpacetimeDB backend module in Rust that stores binary content and its metadata. + +TABLE +- blob_record (public), struct BlobRecord + - Fields: + - id: u64 (primary key, auto-increment) + - owner: Identity (btree index by_owner) + - filename: String + - mime_type: String + - size: u64 + - data: Vec + +REDUCERS +- store_blob(filename: String, mime_type: String, data: Vec) + - Store the sender as owner + - Derive size from the byte length + - Insert the bytes unchanged diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/typescript.txt new file mode 100644 index 00000000000..13dd98d2208 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_049_binary_storage/tasks/typescript.txt @@ -0,0 +1,17 @@ +Write a SpacetimeDB backend module in TypeScript that stores binary content and its metadata. + +TABLE +- blob_record (public) + - Fields: + - id: u64 (primary key, auto-increment) + - owner: Identity (btree index byOwner) + - filename: string + - mimeType: string + - size: u64 + - data: u8 array + +REDUCERS +- store_blob(filename: string, mimeType: string, data: u8 array) + - Store ctx.sender as owner + - Derive size from the byte length + - Insert the bytes unchanged diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/answers/csharp.cs new file mode 100644 index 00000000000..368f0499e92 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/answers/csharp.cs @@ -0,0 +1,29 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Organization", Public = true)] + public partial struct Organization + { + [PrimaryKey, AutoInc] public ulong Id; + public string Name; + } + + [Table(Accessor = "Department", Public = true)] + [SpacetimeDB.Index.BTree(Accessor = "by_organization", Columns = new[] { nameof(OrganizationId) })] + public partial struct Department + { + [PrimaryKey, AutoInc] public ulong Id; + public ulong OrganizationId; + public string Name; + } + + [Table(Accessor = "Employee", Public = true)] + [SpacetimeDB.Index.BTree(Accessor = "by_department", Columns = new[] { nameof(DepartmentId) })] + public partial struct Employee + { + [PrimaryKey, AutoInc] public ulong Id; + public ulong DepartmentId; + public string Name; + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/answers/rust.rs new file mode 100644 index 00000000000..d072ad87eac --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/answers/rust.rs @@ -0,0 +1,35 @@ +use spacetimedb::table; + +#[table(accessor = organization, public)] +pub struct Organization { + #[primary_key] + #[auto_inc] + pub id: u64, + pub name: String, +} + +#[table( + accessor = department, + public, + index(accessor = by_organization, btree(columns = [organization_id])) +)] +pub struct Department { + #[primary_key] + #[auto_inc] + pub id: u64, + pub organization_id: u64, + pub name: String, +} + +#[table( + accessor = employee, + public, + index(accessor = by_department, btree(columns = [department_id])) +)] +pub struct Employee { + #[primary_key] + #[auto_inc] + pub id: u64, + pub department_id: u64, + pub name: String, +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/answers/typescript.ts new file mode 100644 index 00000000000..ea21bdeb647 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/answers/typescript.ts @@ -0,0 +1,34 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const organization = table( + { name: 'organization', public: true }, + { id: t.u64().primaryKey().autoInc(), name: t.string() } +); + +const department = table( + { + name: 'department', + public: true, + indexes: [{ accessor: 'byOrganization', algorithm: 'btree', columns: ['organizationId'] }], + }, + { + id: t.u64().primaryKey().autoInc(), + organizationId: t.u64(), + name: t.string(), + } +); + +const employee = table( + { + name: 'employee', + public: true, + indexes: [{ accessor: 'byDepartment', algorithm: 'btree', columns: ['departmentId'] }], + }, + { + id: t.u64().primaryKey().autoInc(), + departmentId: t.u64(), + name: t.string(), + } +); + +export default schema({ organization, department, employee }); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/spec.rs new file mode 100644 index 00000000000..e5466e2ccd7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/spec.rs @@ -0,0 +1,8 @@ +use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + default_schema_parity_scorers(host_url, file!(), route_tag) + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/tasks/csharp.txt new file mode 100644 index 00000000000..0c1d3a28f1f --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/tasks/csharp.txt @@ -0,0 +1,18 @@ +Write a SpacetimeDB backend module in C# that normalizes organization data into three related tables. + +TABLES +- Organization (public, accessor Organization) + - Id: ulong (primary key, auto increment) + - Name: string +- Department (public, accessor Department) + - Id: ulong (primary key, auto increment) + - OrganizationId: ulong + - Name: string + - index by_organization: btree(OrganizationId) +- Employee (public, accessor Employee) + - Id: ulong (primary key, auto increment) + - DepartmentId: ulong + - Name: string + - index by_department: btree(DepartmentId) + +Do not use nested arrays. No reducers are required. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/tasks/rust.txt new file mode 100644 index 00000000000..88d85ac39d5 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/tasks/rust.txt @@ -0,0 +1,18 @@ +Write a SpacetimeDB backend module in Rust that normalizes organization data into three related tables. + +TABLES +- organization (public), struct Organization + - id: u64 (primary key, auto_inc) + - name: String +- department (public), struct Department + - id: u64 (primary key, auto_inc) + - organization_id: u64 + - name: String + - index by_organization: btree(organization_id) +- employee (public), struct Employee + - id: u64 (primary key, auto_inc) + - department_id: u64 + - name: String + - index by_department: btree(department_id) + +Do not use nested arrays. No reducers are required. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/tasks/typescript.txt new file mode 100644 index 00000000000..6258b64389b --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_050_normalized_schema/tasks/typescript.txt @@ -0,0 +1,18 @@ +Write a SpacetimeDB backend module in TypeScript that normalizes organization data into three related tables. + +TABLES +- organization (public) + - id: u64 (primary key, auto increment) + - name: string +- department (public) + - id: u64 (primary key, auto increment) + - organizationId: u64 + - name: string + - index byOrganization: btree(organizationId) +- employee (public) + - id: u64 (primary key, auto increment) + - departmentId: u64 + - name: string + - index byDepartment: btree(departmentId) + +Do not use nested arrays. No reducers are required. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/answers/csharp.cs new file mode 100644 index 00000000000..0fe6e1675eb --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/answers/csharp.cs @@ -0,0 +1,53 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Category", Public = true)] + public partial struct Category + { + [PrimaryKey] public ulong Id; + public string Slug; + } + + [Table(Accessor = "Product", Public = true)] + [SpacetimeDB.Index.BTree(Accessor = "by_category", Columns = new[] { nameof(CategoryId) })] + [SpacetimeDB.Index.BTree(Accessor = "by_category_slug", Columns = new[] { nameof(CategorySlug) })] + public partial struct Product + { + [PrimaryKey] public ulong Id; + public ulong CategoryId; + public string CategorySlug; + public string Name; + } + + [Reducer] + public static void CreateCategory(ReducerContext ctx, ulong id, string slug) + { + ctx.Db.Category.Insert(new Category { Id = id, Slug = slug }); + } + + [Reducer] + public static void CreateProduct(ReducerContext ctx, ulong id, ulong categoryId, string name) + { + var category = ctx.Db.Category.Id.Find(categoryId) ?? throw new Exception("category not found"); + ctx.Db.Product.Insert(new Product + { + Id = id, + CategoryId = categoryId, + CategorySlug = category.Slug, + Name = name, + }); + } + + [Reducer] + public static void RenameCategory(ReducerContext ctx, ulong id, string newSlug) + { + var category = ctx.Db.Category.Id.Find(id) ?? throw new Exception("category not found"); + ctx.Db.Category.Id.Update(category with { Slug = newSlug }); + + foreach (var product in ctx.Db.Product.by_category.Filter(id)) + { + ctx.Db.Product.Id.Update(product with { CategorySlug = newSlug }); + } + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/answers/rust.rs new file mode 100644 index 00000000000..8122437ebc9 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/answers/rust.rs @@ -0,0 +1,62 @@ +use spacetimedb::{reducer, table, ReducerContext, Table}; + +#[table(accessor = category, public)] +pub struct Category { + #[primary_key] + pub id: u64, + pub slug: String, +} + +#[table( + accessor = product, + public, + index(accessor = by_category, btree(columns = [category_id])), + index(accessor = by_category_slug, btree(columns = [category_slug])) +)] +pub struct Product { + #[primary_key] + pub id: u64, + pub category_id: u64, + pub category_slug: String, + pub name: String, +} + +#[reducer] +pub fn create_category(ctx: &ReducerContext, id: u64, slug: String) { + ctx.db.category().insert(Category { id, slug }); +} + +#[reducer] +pub fn create_product(ctx: &ReducerContext, id: u64, category_id: u64, name: String) -> Result<(), String> { + let category = ctx + .db + .category() + .id() + .find(category_id) + .ok_or_else(|| "category not found".to_string())?; + ctx.db.product().insert(Product { + id, + category_id, + category_slug: category.slug, + name, + }); + Ok(()) +} + +#[reducer] +pub fn rename_category(ctx: &ReducerContext, id: u64, new_slug: String) -> Result<(), String> { + let mut category = ctx + .db + .category() + .id() + .find(id) + .ok_or_else(|| "category not found".to_string())?; + category.slug = new_slug.clone(); + ctx.db.category().id().update(category); + + for mut product in ctx.db.product().by_category().filter(id) { + product.category_slug = new_slug.clone(); + ctx.db.product().id().update(product); + } + Ok(()) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/answers/typescript.ts new file mode 100644 index 00000000000..85d8088d52a --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/answers/typescript.ts @@ -0,0 +1,52 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const category = table( + { name: 'category', public: true }, + { id: t.u64().primaryKey(), slug: t.string() } +); + +const product = table( + { + name: 'product', + public: true, + indexes: [ + { accessor: 'byCategory', algorithm: 'btree', columns: ['categoryId'] }, + { accessor: 'byCategorySlug', algorithm: 'btree', columns: ['categorySlug'] }, + ], + }, + { + id: t.u64().primaryKey(), + categoryId: t.u64(), + categorySlug: t.string(), + name: t.string(), + } +); + +const spacetimedb = schema({ category, product }); +export default spacetimedb; + +export const create_category = spacetimedb.reducer( + { id: t.u64(), slug: t.string() }, + (ctx, { id, slug }) => ctx.db.category.insert({ id, slug }) +); + +export const create_product = spacetimedb.reducer( + { id: t.u64(), categoryId: t.u64(), name: t.string() }, + (ctx, { id, categoryId, name }) => { + const found = ctx.db.category.id.find(categoryId); + if (!found) throw new Error('category not found'); + ctx.db.product.insert({ id, categoryId, categorySlug: found.slug, name }); + } +); + +export const rename_category = spacetimedb.reducer( + { id: t.u64(), newSlug: t.string() }, + (ctx, { id, newSlug }) => { + const found = ctx.db.category.id.find(id); + if (!found) throw new Error('category not found'); + ctx.db.category.id.update({ ...found, slug: newSlug }); + for (const existing of ctx.db.product.byCategory.filter(id)) { + ctx.db.product.id.update({ ...existing, categorySlug: newSlug }); + } + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/spec.rs new file mode 100644 index 00000000000..710e155e5e0 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/spec.rs @@ -0,0 +1,47 @@ +use crate::eval::defaults::{ + default_schema_parity_scorers, make_reducer_call_both_scorer, make_reducer_data_parity_scorer, +}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec, ReducerDataParityConfig, SqlBuilder}; +use serde_json::json; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_reducer_call_both_scorer( + host_url, + file!(), + route_tag, + "create_category", + vec![json!(1), json!("old-slug")], + "create_category", + )); + scorers.push(make_reducer_call_both_scorer( + host_url, + file!(), + route_tag, + "create_product", + vec![json!(10), json!(1), json!("Widget")], + "create_product", + )); + + let sql = SqlBuilder::new(casing_for_lang(lang)); + let table = table_name("product", lang); + let columns = sql.cols(&["id", "category_id", "category_slug", "name"]).join(", "); + let id = ident("id", casing_for_lang(lang)); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "rename_category".into(), + args: vec![json!(1), json!("new-slug")], + select_query: format!("SELECT {columns} FROM {table} WHERE {id}=10"), + id_str: "denormalized_value_stays_synchronized", + collapse_ws: true, + timeout: Duration::from_secs(10), + }, + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/tasks/csharp.txt new file mode 100644 index 00000000000..9e7dac73cd5 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/tasks/csharp.txt @@ -0,0 +1,21 @@ +Write a SpacetimeDB backend module in C# that maintains an indexed denormalized category slug on products. + +TABLES +- Category (public, accessor Category) + - Fields: + - Id: ulong (primary key) + - Slug: string +- Product (public, accessor Product) + - Fields: + - Id: ulong (primary key) + - CategoryId: ulong (btree index by_category) + - CategorySlug: string (btree index by_category_slug) + - Name: string + +REDUCERS +- CreateCategory(id: ulong, slug: string) +- CreateProduct(id: ulong, categoryId: ulong, name: string) + - Copy the current Category Slug into CategorySlug +- RenameCategory(id: ulong, newSlug: string) + - Update the Category Slug + - Update every related Product so CategorySlug stays synchronized diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/tasks/rust.txt new file mode 100644 index 00000000000..1aea0d4e82b --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/tasks/rust.txt @@ -0,0 +1,21 @@ +Write a SpacetimeDB backend module in Rust that maintains an indexed denormalized category slug on products. + +TABLES +- category (public), struct Category + - Fields: + - id: u64 (primary key) + - slug: String +- product (public), struct Product + - Fields: + - id: u64 (primary key) + - category_id: u64 (btree index by_category) + - category_slug: String (btree index by_category_slug) + - name: String + +REDUCERS +- create_category(id: u64, slug: String) +- create_product(id: u64, category_id: u64, name: String) + - Copy the current category slug into category_slug +- rename_category(id: u64, new_slug: String) + - Update the category slug + - Update every related product so category_slug stays synchronized diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/tasks/typescript.txt new file mode 100644 index 00000000000..08113e51758 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_051_denormalized_index/tasks/typescript.txt @@ -0,0 +1,21 @@ +Write a SpacetimeDB backend module in TypeScript that maintains an indexed denormalized category slug on products. + +TABLES +- category (public) + - Fields: + - id: u64 (primary key) + - slug: string +- product (public) + - Fields: + - id: u64 (primary key) + - categoryId: u64 (btree index byCategory) + - categorySlug: string (btree index byCategorySlug) + - name: string + +REDUCERS +- create_category(id: u64, slug: string) +- create_product(id: u64, categoryId: u64, name: string) + - Copy the current category slug into categorySlug +- rename_category(id: u64, newSlug: string) + - Update the category slug + - Update every related product so categorySlug stays synchronized diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/answers/csharp.cs new file mode 100644 index 00000000000..7f761292915 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/answers/csharp.cs @@ -0,0 +1,30 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Parent", Public = true)] + public partial struct Parent + { + [PrimaryKey, AutoInc] public ulong Id; + public string Name; + } + + [Table(Accessor = "Child", Public = true)] + [SpacetimeDB.Index.BTree(Accessor = "by_parent", Columns = new[] { nameof(ParentId) })] + public partial struct Child + { + [PrimaryKey, AutoInc] public ulong Id; + public ulong ParentId; + public string Name; + } + + [Reducer] + public static void CreateFamily(ReducerContext ctx, string parentName, List childNames) + { + var parent = ctx.Db.Parent.Insert(new Parent { Id = 0, Name = parentName }); + foreach (var name in childNames) + { + ctx.Db.Child.Insert(new Child { Id = 0, ParentId = parent.Id, Name = name }); + } + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/answers/rust.rs new file mode 100644 index 00000000000..298ee10457a --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/answers/rust.rs @@ -0,0 +1,37 @@ +use spacetimedb::{reducer, table, ReducerContext, Table}; + +#[table(accessor = parent, public)] +pub struct Parent { + #[primary_key] + #[auto_inc] + pub id: u64, + pub name: String, +} + +#[table( + accessor = child, + public, + index(accessor = by_parent, btree(columns = [parent_id])) +)] +pub struct Child { + #[primary_key] + #[auto_inc] + pub id: u64, + pub parent_id: u64, + pub name: String, +} + +#[reducer] +pub fn create_family(ctx: &ReducerContext, parent_name: String, child_names: Vec) { + let parent = ctx.db.parent().insert(Parent { + id: 0, + name: parent_name, + }); + for name in child_names { + ctx.db.child().insert(Child { + id: 0, + parent_id: parent.id, + name, + }); + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/answers/typescript.ts new file mode 100644 index 00000000000..33603dcbd56 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/answers/typescript.ts @@ -0,0 +1,28 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const parent = table( + { name: 'parent', public: true }, + { id: t.u64().primaryKey().autoInc(), name: t.string() } +); + +const child = table( + { + name: 'child', + public: true, + indexes: [{ accessor: 'byParent', algorithm: 'btree', columns: ['parentId'] }], + }, + { id: t.u64().primaryKey().autoInc(), parentId: t.u64(), name: t.string() } +); + +const spacetimedb = schema({ parent, child }); +export default spacetimedb; + +export const create_family = spacetimedb.reducer( + { parentName: t.string(), childNames: t.array(t.string()) }, + (ctx, { parentName, childNames }) => { + const insertedParent = ctx.db.parent.insert({ id: 0n, name: parentName }); + for (const name of childNames) { + ctx.db.child.insert({ id: 0n, parentId: insertedParent.id, name }); + } + } +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/spec.rs new file mode 100644 index 00000000000..0bc0798dd09 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/spec.rs @@ -0,0 +1,34 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_sql_count_scorer}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec, ReducerSqlCountConfig}; +use serde_json::json; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + let casing = casing_for_lang(lang); + let parent = table_name("parent", lang); + let child = table_name("child", lang); + let parent_id = ident("parent_id", casing); + let id = ident("id", casing); + let name = ident("name", casing); + let sql = format!( + "SELECT COUNT(*) AS n FROM {child} JOIN {parent} ON {child}.{parent_id}={parent}.{id} WHERE {parent}.{name}='Taylor'" + ); + + scorers.push(make_reducer_sql_count_scorer( + host_url, + ReducerSqlCountConfig { + src_file: file!(), + route_tag, + reducer: "create_family".into(), + args: vec![json!("Taylor"), json!(["Avery", "Jordan"])], + sql_count_query: sql, + expected_count: 2, + id_str: "children_reference_inserted_parent", + timeout: Duration::from_secs(10), + }, + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/csharp.txt new file mode 100644 index 00000000000..2c79b8c0fc0 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/csharp.txt @@ -0,0 +1,16 @@ +Write a SpacetimeDB backend module in C# that safely creates related rows with an auto-incremented parent ID. + +TABLES +- Parent (public, accessor Parent) + - Fields: + - Id: ulong (primary key, auto-increment) + - Name: string +- Child (public, accessor Child) + - Fields: + - Id: ulong (primary key, auto-increment) + - ParentId: ulong (btree index by_parent) + - Name: string + +REDUCERS +- CreateFamily(parentName: string, childNames: List) + - Insert the Parent, then insert every Child using the Id actually assigned to that Parent diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/rust.txt new file mode 100644 index 00000000000..7351cfd0827 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/rust.txt @@ -0,0 +1,16 @@ +Write a SpacetimeDB backend module in Rust that safely creates related rows with an auto-incremented parent ID. + +TABLES +- parent (public), struct Parent + - Fields: + - id: u64 (primary key, auto-increment) + - name: String +- child (public), struct Child + - Fields: + - id: u64 (primary key, auto-increment) + - parent_id: u64 (btree index by_parent) + - name: String + +REDUCERS +- create_family(parent_name: String, child_names: Vec) + - Insert the parent, then insert every child using the ID actually assigned to that parent diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/typescript.txt new file mode 100644 index 00000000000..334dfaad17c --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_052_autoinc_reference/tasks/typescript.txt @@ -0,0 +1,16 @@ +Write a SpacetimeDB backend module in TypeScript that safely creates related rows with an auto-incremented parent ID. + +TABLES +- parent (public) + - Fields: + - id: u64 (primary key, auto-increment) + - name: string +- child (public) + - Fields: + - id: u64 (primary key, auto-increment) + - parentId: u64 (btree index byParent) + - name: string + +REDUCERS +- create_family(parentName: string, childNames: string array) + - Insert the parent, then insert every child using the ID actually assigned to that parent diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/answers/csharp.cs new file mode 100644 index 00000000000..457349fbdf1 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/answers/csharp.cs @@ -0,0 +1,21 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Widget", Public = true)] + public partial struct Widget + { + [PrimaryKey] public ulong Id; + public string Name; + [Default(true)] public bool Enabled; + } + + [Reducer] + public static void Seed(ReducerContext ctx) + { + ctx.Db.Widget.Insert(new Widget { Id = 1, Name = "legacy", Enabled = true }); + } + + [Reducer] + public static void Touch(ReducerContext ctx) { } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/answers/rust.rs new file mode 100644 index 00000000000..76963b7a165 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/answers/rust.rs @@ -0,0 +1,22 @@ +use spacetimedb::{ReducerContext, Table}; + +#[spacetimedb::table(accessor = widget, public)] +pub struct Widget { + #[primary_key] + id: u64, + name: String, + #[default(true)] + enabled: bool, +} + +#[spacetimedb::reducer] +pub fn seed(ctx: &ReducerContext) { + ctx.db.widget().insert(Widget { + id: 1, + name: "legacy".into(), + enabled: true, + }); +} + +#[spacetimedb::reducer] +pub fn touch(_ctx: &ReducerContext) {} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/answers/typescript.ts new file mode 100644 index 00000000000..836945ca178 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/answers/typescript.ts @@ -0,0 +1,15 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const widget = table( + { name: 'widget', public: true }, + { id: t.u64().primaryKey(), name: t.string(), enabled: t.bool().default(true) } +); + +const spacetimedb = schema({ widget }); +export default spacetimedb; + +export const seed = spacetimedb.reducer(ctx => { + ctx.db.widget.insert({ id: 1n, name: 'legacy', enabled: true }); +}); + +export const touch = spacetimedb.reducer(_ctx => {}); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/setup/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/setup/csharp.cs new file mode 100644 index 00000000000..316091c1af2 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/setup/csharp.cs @@ -0,0 +1,20 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Widget", Public = true)] + public partial struct Widget + { + [PrimaryKey] public ulong Id; + public string Name; + } + + [Reducer] + public static void Seed(ReducerContext ctx) + { + ctx.Db.Widget.Insert(new Widget { Id = 1, Name = "legacy" }); + } + + [Reducer] + public static void Touch(ReducerContext ctx) { } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/setup/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/setup/rust.rs new file mode 100644 index 00000000000..221f8362303 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/setup/rust.rs @@ -0,0 +1,19 @@ +use spacetimedb::{ReducerContext, Table}; + +#[spacetimedb::table(accessor = widget, public)] +pub struct Widget { + #[primary_key] + id: u64, + name: String, +} + +#[spacetimedb::reducer] +pub fn seed(ctx: &ReducerContext) { + ctx.db.widget().insert(Widget { + id: 1, + name: "legacy".into(), + }); +} + +#[spacetimedb::reducer] +pub fn touch(_ctx: &ReducerContext) {} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/setup/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/setup/typescript.ts new file mode 100644 index 00000000000..efb842f3499 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/setup/typescript.ts @@ -0,0 +1,15 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const widget = table( + { name: 'widget', public: true }, + { id: t.u64().primaryKey(), name: t.string() } +); + +const spacetimedb = schema({ widget }); +export default spacetimedb; + +export const seed = spacetimedb.reducer(ctx => { + ctx.db.widget.insert({ id: 1n, name: 'legacy' }); +}); + +export const touch = spacetimedb.reducer(_ctx => {}); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/spec.rs new file mode 100644 index 00000000000..332980dcb89 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/spec.rs @@ -0,0 +1,28 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_data_parity_scorer}; +use crate::eval::{casing_for_lang, ident, table_name, BenchmarkSpec, ReducerDataParityConfig}; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + let widget = table_name("widget", lang); + let casing = casing_for_lang(lang); + let id = ident("id", casing); + let name = ident("name", casing); + let enabled = ident("enabled", casing); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "touch".into(), + args: vec![], + select_query: format!("SELECT {id}, {name}, {enabled} FROM {widget}"), + collapse_ws: true, + timeout: Duration::from_secs(10), + id_str: "existing_row_backfilled", + }, + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/csharp.txt new file mode 100644 index 00000000000..8499bad93c9 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/csharp.txt @@ -0,0 +1,15 @@ +Update the existing SpacetimeDB backend module in C# to add a migration-safe default value. + +TABLE +- Widget (public, accessor Widget) + - Preserve the existing table and fields + - Keep Id: ulong as the primary key + - Add Enabled: bool as the trailing field + - Give Enabled a default value of true + +REDUCERS +- Preserve Seed and Touch without renaming them +- Update new Widget values to set Enabled to true + +MIGRATION +- A compatible republish must backfill existing rows diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/rust.txt new file mode 100644 index 00000000000..37ecc05516f --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/rust.txt @@ -0,0 +1,15 @@ +Update the existing SpacetimeDB backend module in Rust to add a migration-safe default value. + +TABLE +- widget (public), struct Widget + - Preserve the existing table and fields + - Keep id: u64 as the primary key + - Add enabled: bool as the trailing field + - Give enabled a default value of true + +REDUCERS +- Preserve seed and touch without renaming them +- Update new Widget values to set enabled to true + +MIGRATION +- A compatible republish must backfill existing rows diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/typescript.txt new file mode 100644 index 00000000000..61d1e32ef14 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_053_default_values/tasks/typescript.txt @@ -0,0 +1,15 @@ +Update the existing SpacetimeDB backend module in TypeScript to add a migration-safe default value. + +TABLE +- widget (public) + - Preserve the existing table and fields + - Keep id: u64 as the primary key + - Add enabled: bool as the trailing field + - Give enabled a default value of true + +REDUCERS +- Preserve seed and touch without renaming them +- Update new widget values to set enabled to true + +MIGRATION +- A compatible republish must backfill existing rows diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/answers/csharp.cs new file mode 100644 index 00000000000..34b49f90ae5 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/answers/csharp.cs @@ -0,0 +1,17 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "SpecialValue", Public = true)] + public partial struct SpecialValue + { + [PrimaryKey] public ulong Id; + public Uuid Uuid; + public ConnectionId ConnectionId; + public TimeDuration Duration; + public U128 Unsigned128; + public I128 Signed128; + public U256 Unsigned256; + public I256 Signed256; + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/answers/rust.rs new file mode 100644 index 00000000000..f247846178c --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/answers/rust.rs @@ -0,0 +1,17 @@ +use spacetimedb::{ + sats::{i256, u256}, + table, ConnectionId, TimeDuration, Uuid, +}; + +#[table(accessor = special_value, public)] +pub struct SpecialValue { + #[primary_key] + pub id: u64, + pub uuid: Uuid, + pub connection_id: ConnectionId, + pub duration: TimeDuration, + pub unsigned_128: u128, + pub signed_128: i128, + pub unsigned_256: u256, + pub signed_256: i256, +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/answers/typescript.ts new file mode 100644 index 00000000000..6ff811f7303 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/answers/typescript.ts @@ -0,0 +1,17 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const specialValue = table( + { name: 'special_value', public: true }, + { + id: t.u64().primaryKey(), + uuid: t.uuid(), + connectionId: t.connectionId(), + duration: t.timeDuration(), + unsigned128: t.u128(), + signed128: t.i128(), + unsigned256: t.u256(), + signed256: t.i256(), + } +); + +export default schema({ specialValue }); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/spec.rs new file mode 100644 index 00000000000..e5466e2ccd7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/spec.rs @@ -0,0 +1,8 @@ +use crate::eval::defaults::default_schema_parity_scorers; +use crate::eval::BenchmarkSpec; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |_lang, route_tag, host_url| { + default_schema_parity_scorers(host_url, file!(), route_tag) + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/csharp.txt new file mode 100644 index 00000000000..2da04668d5e --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/csharp.txt @@ -0,0 +1,15 @@ +Write a SpacetimeDB backend module in C# defining one public table with special value types. + +TABLE +- SpecialValue (public, accessor SpecialValue) + - Fields: + - Id: ulong (primary key) + - Uuid: Uuid + - ConnectionId: ConnectionId + - Duration: TimeDuration + - Unsigned128: U128 + - Signed128: I128 + - Unsigned256: U256 + - Signed256: I256 + +No reducers are required. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/rust.txt new file mode 100644 index 00000000000..7264c4157f4 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/rust.txt @@ -0,0 +1,15 @@ +Write a SpacetimeDB backend module in Rust defining one public table with special value types. + +TABLE +- special_value (public), struct SpecialValue + - Fields: + - id: u64 (primary key) + - uuid: Uuid + - connection_id: ConnectionId + - duration: TimeDuration + - unsigned_128: u128 + - signed_128: i128 + - unsigned_256: u256 + - signed_256: i256 + +No reducers are required. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/typescript.txt new file mode 100644 index 00000000000..d2866eb8bcd --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/tables/t_054_special_types/tasks/typescript.txt @@ -0,0 +1,15 @@ +Write a SpacetimeDB backend module in TypeScript defining one public table with special value types. + +TABLE +- special_value (public) + - Fields: + - id: u64 (primary key) + - uuid: Uuid + - connectionId: ConnectionId + - duration: TimeDuration + - unsigned128: u128 + - signed128: i128 + - unsigned256: u256 + - signed256: i256 + +No reducers are required. diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/answers/csharp.cs new file mode 100644 index 00000000000..af579b9000d --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/answers/csharp.cs @@ -0,0 +1,18 @@ +using SpacetimeDB; +public static partial class Module +{ + [Table(Accessor = "Customer", Public = true)] public partial struct Customer { [PrimaryKey] public ulong Id; public string Name; } + [Table(Accessor = "Purchase", Public = true)] public partial struct Purchase { [PrimaryKey] public ulong Id; [SpacetimeDB.Index.BTree] public ulong CustomerId; } + [Table(Accessor = "LineItem", Public = true)] public partial struct LineItem { [PrimaryKey] public ulong Id; [SpacetimeDB.Index.BTree] public ulong PurchaseId; public string Sku; [SpacetimeDB.Index.BTree] public bool Visible; } + [SpacetimeDB.Type] public partial struct OrderLineDetail { public ulong LineId; public string CustomerName; public string Sku; } + [Reducer] public static void Seed(ReducerContext ctx) { ctx.Db.Customer.Insert(new Customer { Id = 1, Name = "Ada" }); ctx.Db.Purchase.Insert(new Purchase { Id = 10, CustomerId = 1 }); ctx.Db.LineItem.Insert(new LineItem { Id = 100, PurchaseId = 10, Sku = "SKU-1", Visible = true }); } + [SpacetimeDB.View(Accessor = "OrderLineDetail", Public = true)] + public static IEnumerable OrderLineDetails(AnonymousViewContext ctx) + { + foreach (var line in ctx.Db.LineItem.Visible.Filter(true)) { + var purchase = ctx.Db.Purchase.Id.Find(line.PurchaseId); if (purchase is null) continue; + var customer = ctx.Db.Customer.Id.Find(purchase.Value.CustomerId); if (customer is null) continue; + yield return new OrderLineDetail { LineId = line.Id, CustomerName = customer.Value.Name, Sku = line.Sku }; + } + } +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/answers/rust.rs new file mode 100644 index 00000000000..a8760d3d8eb --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/answers/rust.rs @@ -0,0 +1,61 @@ +use spacetimedb::{reducer, table, view, AnonymousViewContext, ReducerContext, SpacetimeType, Table}; +#[table(accessor = customer, public)] +pub struct Customer { + #[primary_key] + pub id: u64, + pub name: String, +} +#[table(accessor = purchase, public)] +pub struct Purchase { + #[primary_key] + pub id: u64, + #[index(btree)] + pub customer_id: u64, +} +#[table(accessor = line_item, public)] +pub struct LineItem { + #[primary_key] + pub id: u64, + #[index(btree)] + pub purchase_id: u64, + pub sku: String, + #[index(btree)] + pub visible: bool, +} +#[derive(SpacetimeType)] +pub struct OrderLineDetail { + pub line_id: u64, + pub customer_name: String, + pub sku: String, +} +#[reducer] +pub fn seed(ctx: &ReducerContext) { + ctx.db.customer().insert(Customer { + id: 1, + name: "Ada".into(), + }); + ctx.db.purchase().insert(Purchase { id: 10, customer_id: 1 }); + ctx.db.line_item().insert(LineItem { + id: 100, + purchase_id: 10, + sku: "SKU-1".into(), + visible: true, + }); +} +#[view(accessor = order_line_detail, public)] +pub fn order_line_detail(ctx: &AnonymousViewContext) -> Vec { + ctx.db + .line_item() + .visible() + .filter(true) + .filter_map(|line| { + let purchase = ctx.db.purchase().id().find(line.purchase_id)?; + let customer = ctx.db.customer().id().find(purchase.customer_id)?; + Some(OrderLineDetail { + line_id: line.id, + customer_name: customer.name, + sku: line.sku, + }) + }) + .collect() +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/answers/typescript.ts new file mode 100644 index 00000000000..bdcf65523ef --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/answers/typescript.ts @@ -0,0 +1,10 @@ +import { schema, table, t } from 'spacetimedb/server'; +const customer = table({ name: 'customer', public: true }, { id: t.u64().primaryKey(), name: t.string() }); +const purchase = table({ name: 'purchase', public: true }, { id: t.u64().primaryKey(), customerId: t.u64().index('btree') }); +const lineItem = table({ name: 'line_item', public: true }, { id: t.u64().primaryKey(), purchaseId: t.u64().index('btree'), sku: t.string(), visible: t.bool().index('btree') }); +const OrderLineDetail = t.row('ThreeTableJoinRow', { lineId: t.u64(), customerName: t.string(), sku: t.string() }); +const spacetimedb = schema({ customer, purchase, lineItem }); export default spacetimedb; +export const seed = spacetimedb.reducer(ctx => { ctx.db.customer.insert({ id: 1n, name: 'Ada' }); ctx.db.purchase.insert({ id: 10n, customerId: 1n }); ctx.db.lineItem.insert({ id: 100n, purchaseId: 10n, sku: 'SKU-1', visible: true }); }); +export const order_line_detail = spacetimedb.anonymousView({ name: 'order_line_detail', public: true }, t.array(OrderLineDetail), ctx => { + const rows: Array<{ lineId: bigint; customerName: string; sku: string }> = []; for (const line of ctx.db.lineItem.visible.filter(true)) { const purchase = ctx.db.purchase.id.find(line.purchaseId); if (!purchase) continue; const customer = ctx.db.customer.id.find(purchase.customerId); if (customer) rows.push({ lineId: line.id, customerName: customer.name, sku: line.sku }); } return rows; +}); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/spec.rs new file mode 100644 index 00000000000..8eaee51e5b4 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/spec.rs @@ -0,0 +1,24 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_data_parity_scorer}; +use crate::eval::{table_name, BenchmarkSpec, ReducerDataParityConfig}; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + let view = table_name("order_line_detail", lang); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "seed".into(), + args: vec![], + select_query: format!("SELECT * FROM {view}"), + id_str: "three_table_join", + collapse_ws: true, + timeout: Duration::from_secs(10), + }, + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/csharp.txt new file mode 100644 index 00000000000..9fd5ab763e7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/csharp.txt @@ -0,0 +1,29 @@ +Write a SpacetimeDB backend module in C# that defines a public view joining three normalized tables. + +TABLES +- Customer (public, accessor Customer) + - Fields: + - Id: ulong (primary key) + - Name: string +- Purchase (public, accessor Purchase) + - Fields: + - Id: ulong (primary key) + - CustomerId: ulong (btree index) +- LineItem (public, accessor LineItem) + - Fields: + - Id: ulong (primary key) + - PurchaseId: ulong (btree index) + - Sku: string + - Visible: bool (btree index) + +REDUCERS +- Seed() + - Insert one complete visible Customer -> Purchase -> LineItem chain + - Use Customer Id 1 named "Ada" + - Use Purchase Id 10 for Customer 1 + - Use visible LineItem Id 100 for Purchase 10 with Sku "SKU-1" + +VIEW +- OrderLineDetail (anonymous procedural, public) + - Returns rows with LineId: ulong, CustomerName: string, and Sku: string + - Return only rows connected through matching Customer, Purchase, and LineItem IDs diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/rust.txt new file mode 100644 index 00000000000..cd4cf43c491 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/rust.txt @@ -0,0 +1,29 @@ +Write a SpacetimeDB backend module in Rust that defines a public view joining three normalized tables. + +TABLES +- customer (public), struct Customer + - Fields: + - id: u64 (primary key) + - name: String +- purchase (public), struct Purchase + - Fields: + - id: u64 (primary key) + - customer_id: u64 (btree index) +- line_item (public), struct LineItem + - Fields: + - id: u64 (primary key) + - purchase_id: u64 (btree index) + - sku: String + - visible: bool (btree index) + +REDUCERS +- seed() + - Insert one complete visible customer -> purchase -> line_item chain + - Use customer id 1 named "Ada" + - Use purchase id 10 for customer 1 + - Use visible line_item id 100 for purchase 10 with sku "SKU-1" + +VIEW +- order_line_detail (anonymous procedural, public) + - Returns rows with line_id: u64, customer_name: String, and sku: String + - Return only rows connected through matching customer, purchase, and line-item IDs diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/typescript.txt new file mode 100644 index 00000000000..f930598b039 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_061_three_table_join/tasks/typescript.txt @@ -0,0 +1,29 @@ +Write a SpacetimeDB backend module in TypeScript that defines a public view joining three normalized tables. + +TABLES +- customer (public) + - Fields: + - id: u64 (primary key) + - name: string +- purchase (public) + - Fields: + - id: u64 (primary key) + - customerId: u64 (btree index) +- line_item (public) + - Fields: + - id: u64 (primary key) + - purchaseId: u64 (btree index) + - sku: string + - visible: bool (btree index) + +REDUCERS +- seed() + - Insert one complete visible customer -> purchase -> line_item chain + - Use customer id 1 named "Ada" + - Use purchase id 10 for customer 1 + - Use visible line_item id 100 for purchase 10 with sku "SKU-1" + +VIEW +- order_line_detail (anonymous procedural, public) + - Returns rows with lineId: u64, customerName: string, and sku: string + - Return only rows connected through matching customer, purchase, and line-item IDs diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/answers/csharp.cs new file mode 100644 index 00000000000..64ea861edfb --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/answers/csharp.cs @@ -0,0 +1,17 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Member", Public = true)] public partial struct Member { [PrimaryKey] public ulong Id; public string Name; } + [Table(Accessor = "Eligibility", Public = true)] public partial struct Eligibility { [PrimaryKey] public ulong Id; [SpacetimeDB.Index.BTree] public ulong MemberId; } + [Reducer] + public static void Seed(ReducerContext ctx) + { + ctx.Db.Member.Insert(new Member { Id = 1, Name = "Ada" }); + ctx.Db.Member.Insert(new Member { Id = 2, Name = "Grace" }); + ctx.Db.Eligibility.Insert(new Eligibility { Id = 10, MemberId = 1 }); + } + [SpacetimeDB.View(Accessor = "EligibleMember", Public = true)] + public static IQuery EligibleMember(ViewContext ctx) => + ctx.From.Eligibility().RightSemijoin(ctx.From.Member(), (eligibility, member) => eligibility.MemberId.Eq(member.Id)); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/answers/rust.rs new file mode 100644 index 00000000000..b0324b642d5 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/answers/rust.rs @@ -0,0 +1,37 @@ +use spacetimedb::{reducer, table, view, Query, ReducerContext, Table, ViewContext}; + +#[table(accessor = member, public)] +pub struct Member { + #[primary_key] + pub id: u64, + pub name: String, +} +#[table(accessor = eligibility, public)] +pub struct Eligibility { + #[primary_key] + pub id: u64, + #[index(btree)] + pub member_id: u64, +} + +#[reducer] +pub fn seed(ctx: &ReducerContext) { + ctx.db.member().insert(Member { + id: 1, + name: "Ada".into(), + }); + ctx.db.member().insert(Member { + id: 2, + name: "Grace".into(), + }); + ctx.db.eligibility().insert(Eligibility { id: 10, member_id: 1 }); +} + +#[view(accessor = eligible_member, public)] +pub fn eligible_member(ctx: &ViewContext) -> impl Query { + ctx.from + .eligibility() + .right_semijoin(ctx.from.member(), |eligibility, member| { + eligibility.member_id.eq(member.id) + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/answers/typescript.ts new file mode 100644 index 00000000000..aa887f103e8 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/answers/typescript.ts @@ -0,0 +1,14 @@ +import { schema, table, t } from 'spacetimedb/server'; + +const member = table({ name: 'member', public: true }, { id: t.u64().primaryKey(), name: t.string() }); +const eligibility = table({ name: 'eligibility', public: true }, { id: t.u64().primaryKey(), memberId: t.u64().index('btree') }); +const spacetimedb = schema({ member, eligibility }); +export default spacetimedb; +export const seed = spacetimedb.reducer(ctx => { + ctx.db.member.insert({ id: 1n, name: 'Ada' }); ctx.db.member.insert({ id: 2n, name: 'Grace' }); + ctx.db.eligibility.insert({ id: 10n, memberId: 1n }); +}); +export const eligible_member = spacetimedb.view( + { name: 'eligible_member', public: true }, t.array(member.rowType), + ctx => ctx.from.eligibility.rightSemijoin(ctx.from.member, (eligibility, member) => eligibility.memberId.eq(member.id)) +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/spec.rs new file mode 100644 index 00000000000..c9387e35e2c --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/spec.rs @@ -0,0 +1,24 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_data_parity_scorer}; +use crate::eval::{table_name, BenchmarkSpec, ReducerDataParityConfig}; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + let view = table_name("eligible_member", lang); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "seed".into(), + args: vec![], + select_query: format!("SELECT * FROM {view}"), + id_str: "semijoin_intersection", + collapse_ws: true, + timeout: Duration::from_secs(10), + }, + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/csharp.txt new file mode 100644 index 00000000000..4655bb596ce --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/csharp.txt @@ -0,0 +1,21 @@ +Write a SpacetimeDB backend module in C# that defines a query-builder semijoin view. + +TABLES +- Member (public, accessor Member) + - Fields: + - Id: ulong (primary key) + - Name: string +- Eligibility (public, accessor Eligibility) + - Fields: + - Id: ulong (primary key) + - MemberId: ulong (btree index) + +REDUCERS +- Seed() + - Insert Member 1 named "Ada" and Member 2 named "Grace" + - Insert Eligibility Id 10 for only Member 1 + +VIEW +- EligibleMember (public, non-anonymous query-builder) + - Use a semijoin + - Return members only when a matching Eligibility row exists diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/rust.txt new file mode 100644 index 00000000000..cfe7ab28e21 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/rust.txt @@ -0,0 +1,21 @@ +Write a SpacetimeDB backend module in Rust that defines a query-builder semijoin view. + +TABLES +- member (public), struct Member + - Fields: + - id: u64 (primary key) + - name: String +- eligibility (public), struct Eligibility + - Fields: + - id: u64 (primary key) + - member_id: u64 (btree index) + +REDUCERS +- seed() + - Insert member 1 named "Ada" and member 2 named "Grace" + - Insert eligibility id 10 for only member 1 + +VIEW +- eligible_member (public, non-anonymous query-builder) + - Use a semijoin + - Return members only when a matching eligibility row exists diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/typescript.txt new file mode 100644 index 00000000000..8c0716686c9 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_062_semijoin_intersection/tasks/typescript.txt @@ -0,0 +1,21 @@ +Write a SpacetimeDB backend module in TypeScript that defines a query-builder semijoin view. + +TABLES +- member (public) + - Fields: + - id: u64 (primary key) + - name: string +- eligibility (public) + - Fields: + - id: u64 (primary key) + - memberId: u64 (btree index) + +REDUCERS +- seed() + - Insert member 1 named "Ada" and member 2 named "Grace" + - Insert eligibility id 10 for only member 1 + +VIEW +- eligible_member (public, non-anonymous query-builder) + - Use a semijoin + - Return members only when a matching eligibility row exists diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/answers/csharp.cs new file mode 100644 index 00000000000..0a051a85ca4 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/answers/csharp.cs @@ -0,0 +1,27 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Event", Public = true)] + public partial struct Event + { + [PrimaryKey] public ulong Id; + [SpacetimeDB.Index.BTree] public Timestamp OccurredAt; + public string Label; + } + + [Reducer] + public static void Seed(ReducerContext ctx) + { + long[] times = [100, 200, 300, 400, 500]; + for (var index = 0; index < times.Length; index++) + { + var micros = times[index]; + ctx.Db.Event.Insert(new Event { Id = (ulong)index + 1, OccurredAt = new Timestamp(micros), Label = $"event-{micros}" }); + } + } + + [SpacetimeDB.View(Accessor = "WindowEvent", Public = true)] + public static IEnumerable WindowEvent(AnonymousViewContext ctx) => + ctx.Db.Event.OccurredAt.Filter((new Timestamp(200), new Timestamp(400))); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/answers/rust.rs new file mode 100644 index 00000000000..86a8ce431b9 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/answers/rust.rs @@ -0,0 +1,28 @@ +use spacetimedb::{reducer, table, view, AnonymousViewContext, ReducerContext, Table, Timestamp}; + +#[table(accessor = event, public)] +pub struct Event { + #[primary_key] + pub id: u64, + #[index(btree)] + pub occurred_at: Timestamp, + pub label: String, +} + +#[reducer] +pub fn seed(ctx: &ReducerContext) { + for (id, micros) in [100, 200, 300, 400, 500].into_iter().enumerate() { + ctx.db.event().insert(Event { + id: id as u64 + 1, + occurred_at: Timestamp::from_micros_since_unix_epoch(micros), + label: format!("event-{micros}"), + }); + } +} + +#[view(accessor = window_event, public)] +pub fn window_event(ctx: &AnonymousViewContext) -> Vec { + let start = Timestamp::from_micros_since_unix_epoch(200); + let end = Timestamp::from_micros_since_unix_epoch(400); + ctx.db.event().occurred_at().filter(start..=end).collect() +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/answers/typescript.ts new file mode 100644 index 00000000000..251fcbb0597 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/answers/typescript.ts @@ -0,0 +1,26 @@ +import { Timestamp } from 'spacetimedb'; +import { Range, schema, table, t } from 'spacetimedb/server'; + +const event = table({ name: 'event', public: true }, { + id: t.u64().primaryKey(), + occurredAt: t.timestamp().index('btree'), + label: t.string(), +}); + +const spacetimedb = schema({ event }); +export default spacetimedb; + +export const seed = spacetimedb.reducer(ctx => { + [100n, 200n, 300n, 400n, 500n].forEach((micros, index) => { + ctx.db.event.insert({ id: BigInt(index + 1), occurredAt: new Timestamp(micros), label: `event-${micros}` }); + }); +}); + +export const window_event = spacetimedb.anonymousView( + { name: 'window_event', public: true }, + t.array(event.rowType), + ctx => Array.from(ctx.db.event.occurredAt.filter(new Range( + { tag: 'included', value: new Timestamp(200n) }, + { tag: 'included', value: new Timestamp(400n) }, + ))) +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/spec.rs new file mode 100644 index 00000000000..cde104ab4d4 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/spec.rs @@ -0,0 +1,24 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_data_parity_scorer}; +use crate::eval::{table_name, BenchmarkSpec, ReducerDataParityConfig}; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + let view = table_name("window_event", lang); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "seed".into(), + args: vec![], + select_query: format!("SELECT * FROM {view}"), + id_str: "inclusive_timestamp_window", + collapse_ws: true, + timeout: Duration::from_secs(10), + }, + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/csharp.txt new file mode 100644 index 00000000000..f7f96c90858 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/csharp.txt @@ -0,0 +1,20 @@ +Write a SpacetimeDB backend module in C# that defines a procedural view over an inclusive timestamp range. + +TABLE +- Event (public, accessor Event) + - Fields: + - Id: ulong (primary key) + - OccurredAt: Timestamp (btree index) + - Label: string + +REDUCERS +- Seed() + - Insert events at 100, 200, 300, 400, and 500 microseconds after the Unix epoch + - Assign Ids 1 through 5 in timestamp order + - Set each Label to "event-{microseconds}", such as "event-100" + +VIEW +- WindowEvent (anonymous procedural, public) + - Use the OccurredAt index + - Return the inclusive range from 200 through 400 microseconds + - Return rows in ascending OccurredAt order diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/rust.txt new file mode 100644 index 00000000000..be8b847f940 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/rust.txt @@ -0,0 +1,20 @@ +Write a SpacetimeDB backend module in Rust that defines a procedural view over an inclusive timestamp range. + +TABLE +- event (public), struct Event + - Fields: + - id: u64 (primary key) + - occurred_at: Timestamp (btree index) + - label: String + +REDUCERS +- seed() + - Insert events at 100, 200, 300, 400, and 500 microseconds after the Unix epoch + - Assign ids 1 through 5 in timestamp order + - Set each label to "event-{microseconds}", such as "event-100" + +VIEW +- window_event (anonymous procedural, public) + - Use the occurred_at index + - Return the inclusive range from 200 through 400 microseconds + - Return rows in ascending occurred_at order diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/typescript.txt new file mode 100644 index 00000000000..f6dd5b43631 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_063_timestamp_window/tasks/typescript.txt @@ -0,0 +1,20 @@ +Write a SpacetimeDB backend module in TypeScript that defines a procedural view over an inclusive timestamp range. + +TABLE +- event (public) + - Fields: + - id: u64 (primary key) + - occurredAt: Timestamp (btree index) + - label: string + +REDUCERS +- seed() + - Insert events at 100, 200, 300, 400, and 500 microseconds after the Unix epoch + - Assign ids 1 through 5 in timestamp order + - Set each label to "event-{microseconds}", such as "event-100" + +VIEW +- window_event (anonymous procedural, public) + - Use the occurredAt index + - Return the inclusive range from 200 through 400 microseconds + - Return rows in ascending occurredAt order diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/answers/csharp.cs new file mode 100644 index 00000000000..60aeb5c3d82 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/answers/csharp.cs @@ -0,0 +1,15 @@ +using SpacetimeDB; +public static partial class Module +{ + [Table(Accessor = "Content", Public = true)] public partial struct Content { [PrimaryKey] public ulong Id; [SpacetimeDB.Index.BTree] public string Category; public bool Active; public int Score; } + [Reducer] + public static void Seed(ReducerContext ctx) + { + ctx.Db.Content.Insert(new Content { Id = 1, Category = "news", Active = true, Score = 20 }); + ctx.Db.Content.Insert(new Content { Id = 2, Category = "news", Active = false, Score = 20 }); + ctx.Db.Content.Insert(new Content { Id = 3, Category = "news", Active = true, Score = 5 }); + ctx.Db.Content.Insert(new Content { Id = 4, Category = "sports", Active = true, Score = 20 }); + } + [SpacetimeDB.View(Accessor = "FeaturedContent", Public = true)] + public static IEnumerable FeaturedContent(AnonymousViewContext ctx) => ctx.Db.Content.Category.Filter("news").Where(row => row.Active && row.Score >= 10); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/answers/rust.rs new file mode 100644 index 00000000000..f4337a01fe7 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/answers/rust.rs @@ -0,0 +1,50 @@ +use spacetimedb::{reducer, table, view, AnonymousViewContext, ReducerContext, Table}; +#[table(accessor = content, public)] +pub struct Content { + #[primary_key] + pub id: u64, + #[index(btree)] + pub category: String, + pub active: bool, + pub score: i32, +} +#[reducer] +pub fn seed(ctx: &ReducerContext) { + for row in [ + Content { + id: 1, + category: "news".into(), + active: true, + score: 20, + }, + Content { + id: 2, + category: "news".into(), + active: false, + score: 20, + }, + Content { + id: 3, + category: "news".into(), + active: true, + score: 5, + }, + Content { + id: 4, + category: "sports".into(), + active: true, + score: 20, + }, + ] { + ctx.db.content().insert(row); + } +} +#[view(accessor = featured_content, public)] +pub fn featured_content(ctx: &AnonymousViewContext) -> Vec { + ctx.db + .content() + .category() + .filter(&"news".to_string()) + .filter(|row| row.active && row.score >= 10) + .collect() +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/answers/typescript.ts new file mode 100644 index 00000000000..a7d8093aa81 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/answers/typescript.ts @@ -0,0 +1,13 @@ +import { schema, table, t } from 'spacetimedb/server'; +const content = table({ name: 'content', public: true }, { id: t.u64().primaryKey(), category: t.string().index('btree'), active: t.bool(), score: t.i32() }); +const spacetimedb = schema({ content }); export default spacetimedb; +export const seed = spacetimedb.reducer(ctx => { + ctx.db.content.insert({ id: 1n, category: 'news', active: true, score: 20 }); + ctx.db.content.insert({ id: 2n, category: 'news', active: false, score: 20 }); + ctx.db.content.insert({ id: 3n, category: 'news', active: true, score: 5 }); + ctx.db.content.insert({ id: 4n, category: 'sports', active: true, score: 20 }); +}); +export const featured_content = spacetimedb.anonymousView( + { name: 'featured_content', public: true }, t.array(content.rowType), + ctx => Array.from(ctx.db.content.category.filter('news')).filter(row => row.active && row.score >= 10) +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/spec.rs new file mode 100644 index 00000000000..3df3dbfd85e --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/spec.rs @@ -0,0 +1,24 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_data_parity_scorer}; +use crate::eval::{table_name, BenchmarkSpec, ReducerDataParityConfig}; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + let view = table_name("featured_content", lang); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "seed".into(), + args: vec![], + select_query: format!("SELECT * FROM {view}"), + id_str: "indexed_candidates_are_filtered", + collapse_ws: true, + timeout: Duration::from_secs(10), + }, + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/csharp.txt new file mode 100644 index 00000000000..83c69688658 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/csharp.txt @@ -0,0 +1,21 @@ +Write a SpacetimeDB backend module in C# that defines a procedural view using an index and additional filters. + +TABLE +- Content (public, accessor Content) + - Fields: + - Id: ulong (primary key) + - Category: string (btree index) + - Active: bool + - Score: int + +REDUCERS +- Seed() + - Insert Id 1 with Category "news", Active true, and Score 20 + - Insert Id 2 with Category "news", Active false, and Score 20 + - Insert Id 3 with Category "news", Active true, and Score 5 + - Insert Id 4 with Category "sports", Active true, and Score 20 + +VIEW +- FeaturedContent (anonymous procedural, public) + - First use the Category index for "news" + - Then return only Active rows with Score at least 10 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/rust.txt new file mode 100644 index 00000000000..2e30de3af72 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/rust.txt @@ -0,0 +1,21 @@ +Write a SpacetimeDB backend module in Rust that defines a procedural view using an index and additional filters. + +TABLE +- content (public), struct Content + - Fields: + - id: u64 (primary key) + - category: String (btree index) + - active: bool + - score: i32 + +REDUCERS +- seed() + - Insert id 1 with category "news", active true, and score 20 + - Insert id 2 with category "news", active false, and score 20 + - Insert id 3 with category "news", active true, and score 5 + - Insert id 4 with category "sports", active true, and score 20 + +VIEW +- featured_content (anonymous procedural, public) + - First use the category index for "news" + - Then return only active rows with score at least 10 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/typescript.txt new file mode 100644 index 00000000000..4ec5613aae5 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_064_index_and_filter/tasks/typescript.txt @@ -0,0 +1,21 @@ +Write a SpacetimeDB backend module in TypeScript that defines a procedural view using an index and additional filters. + +TABLE +- content (public) + - Fields: + - id: u64 (primary key) + - category: string (btree index) + - active: boolean + - score: i32 + +REDUCERS +- seed() + - Insert id 1 with category "news", active true, and score 20 + - Insert id 2 with category "news", active false, and score 20 + - Insert id 3 with category "news", active true, and score 5 + - Insert id 4 with category "sports", active true, and score 20 + +VIEW +- featured_content (anonymous procedural, public) + - First use the category index for "news" + - Then return only active rows with score at least 10 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/answers/csharp.cs new file mode 100644 index 00000000000..989a4fc7abe --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/answers/csharp.cs @@ -0,0 +1,54 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "Sale", Public = true)] + public partial struct Sale { [PrimaryKey] public ulong Id; public string Category; public long Amount; } + + [Table(Accessor = "CategoryTotal", Public = true)] + public partial struct CategoryTotal { [PrimaryKey] public string Category; public long TotalAmount; public ulong SaleCount; } + + private static void AddToTotal(ReducerContext ctx, string category, long amount) + { + var total = ctx.Db.CategoryTotal.Category.Find(category); + if (total is null) ctx.Db.CategoryTotal.Insert(new CategoryTotal { Category = category, TotalAmount = amount, SaleCount = 1 }); + else { var row = total.Value; row.TotalAmount += amount; row.SaleCount += 1; ctx.Db.CategoryTotal.Category.Update(row); } + } + + private static void RemoveFromTotal(ReducerContext ctx, string category, long amount) + { + var row = ctx.Db.CategoryTotal.Category.Find(category) ?? throw new InvalidOperationException("missing category total"); + if (row.SaleCount == 1) ctx.Db.CategoryTotal.Category.Delete(category); + else { row.TotalAmount -= amount; row.SaleCount -= 1; ctx.Db.CategoryTotal.Category.Update(row); } + } + + private static void UpsertSale(ReducerContext ctx, Sale sale) + { + var old = ctx.Db.Sale.Id.Find(sale.Id); + if (old is null) ctx.Db.Sale.Insert(sale); + else { RemoveFromTotal(ctx, old.Value.Category, old.Value.Amount); ctx.Db.Sale.Id.Update(sale); } + AddToTotal(ctx, sale.Category, sale.Amount); + } + + private static void DeleteSale(ReducerContext ctx, ulong id) + { + var old = ctx.Db.Sale.Id.Find(id); + if (old is null) return; + ctx.Db.Sale.Id.Delete(id); + RemoveFromTotal(ctx, old.Value.Category, old.Value.Amount); + } + + [Reducer] + public static void Exercise(ReducerContext ctx) + { + UpsertSale(ctx, new Sale { Id = 1, Category = "books", Amount = 10 }); + UpsertSale(ctx, new Sale { Id = 2, Category = "books", Amount = 20 }); + UpsertSale(ctx, new Sale { Id = 2, Category = "books", Amount = 25 }); + UpsertSale(ctx, new Sale { Id = 3, Category = "games", Amount = 40 }); + DeleteSale(ctx, 3); + DeleteSale(ctx, 1); + } + + [SpacetimeDB.View(Accessor = "CategorySummary", Public = true)] + public static IQuery CategorySummary(ViewContext ctx) => ctx.From.CategoryTotal(); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/answers/rust.rs new file mode 100644 index 00000000000..8e18db470bf --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/answers/rust.rs @@ -0,0 +1,111 @@ +use spacetimedb::{reducer, table, view, Query, ReducerContext, Table, ViewContext}; + +#[table(accessor = sale, public)] +pub struct Sale { + #[primary_key] + pub id: u64, + pub category: String, + pub amount: i64, +} + +#[table(accessor = category_total, public)] +pub struct CategoryTotal { + #[primary_key] + pub category: String, + pub total_amount: i64, + pub sale_count: u64, +} + +fn add_to_total(ctx: &ReducerContext, category: &String, amount: i64) { + if let Some(mut total) = ctx.db.category_total().category().find(category) { + total.total_amount += amount; + total.sale_count += 1; + ctx.db.category_total().category().update(total); + } else { + ctx.db.category_total().insert(CategoryTotal { + category: category.clone(), + total_amount: amount, + sale_count: 1, + }); + } +} + +fn remove_from_total(ctx: &ReducerContext, category: &String, amount: i64) { + let mut total = ctx + .db + .category_total() + .category() + .find(category) + .expect("missing category total"); + if total.sale_count == 1 { + ctx.db.category_total().category().delete(category); + } else { + total.total_amount -= amount; + total.sale_count -= 1; + ctx.db.category_total().category().update(total); + } +} + +fn upsert_sale(ctx: &ReducerContext, sale: Sale) { + let id = sale.id; + let category = sale.category.clone(); + let amount = sale.amount; + if let Some(old) = ctx.db.sale().id().find(sale.id) { + remove_from_total(ctx, &old.category, old.amount); + ctx.db.sale().id().update(sale); + } else { + ctx.db.sale().insert(sale); + } + debug_assert!(ctx.db.sale().id().find(id).is_some()); + add_to_total(ctx, &category, amount); +} + +fn delete_sale(ctx: &ReducerContext, id: u64) { + if let Some(old) = ctx.db.sale().id().find(id) { + ctx.db.sale().id().delete(id); + remove_from_total(ctx, &old.category, old.amount); + } +} + +#[reducer] +pub fn exercise(ctx: &ReducerContext) { + upsert_sale( + ctx, + Sale { + id: 1, + category: "books".into(), + amount: 10, + }, + ); + upsert_sale( + ctx, + Sale { + id: 2, + category: "books".into(), + amount: 20, + }, + ); + upsert_sale( + ctx, + Sale { + id: 2, + category: "books".into(), + amount: 25, + }, + ); + upsert_sale( + ctx, + Sale { + id: 3, + category: "games".into(), + amount: 40, + }, + ); + delete_sale(ctx, 3); + delete_sale(ctx, 1); +} + +#[view(accessor = category_summary, public)] +pub fn category_summary(ctx: &ViewContext) -> impl Query { + ctx.from.category_total() +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/answers/typescript.ts new file mode 100644 index 00000000000..d3542e12993 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/answers/typescript.ts @@ -0,0 +1,53 @@ +import { schema, table, t } from 'spacetimedb/server'; +import type { InferSchema, ReducerCtx } from 'spacetimedb/server'; + +const sale = table({ name: 'sale', public: true }, { + id: t.u64().primaryKey(), category: t.string(), amount: t.i64(), +}); +const categoryTotal = table({ name: 'category_total', public: true }, { + category: t.string().primaryKey(), totalAmount: t.i64(), saleCount: t.u64(), +}); +const spacetimedb = schema({ sale, categoryTotal }); +export default spacetimedb; + +type Ctx = ReducerCtx>; + +function addToTotal(ctx: Ctx, category: string, amount: bigint) { + const total = ctx.db.categoryTotal.category.find(category); + if (total) ctx.db.categoryTotal.category.update({ ...total, totalAmount: total.totalAmount + amount, saleCount: total.saleCount + 1n }); + else ctx.db.categoryTotal.insert({ category, totalAmount: amount, saleCount: 1n }); +} + +function removeFromTotal(ctx: Ctx, category: string, amount: bigint) { + const total = ctx.db.categoryTotal.category.find(category); + if (!total) throw new Error('missing category total'); + if (total.saleCount === 1n) ctx.db.categoryTotal.category.delete(category); + else ctx.db.categoryTotal.category.update({ ...total, totalAmount: total.totalAmount - amount, saleCount: total.saleCount - 1n }); +} + +function upsertSale(ctx: Ctx, row: { id: bigint; category: string; amount: bigint }) { + const old = ctx.db.sale.id.find(row.id); + if (old) { removeFromTotal(ctx, old.category, old.amount); ctx.db.sale.id.update(row); } + else ctx.db.sale.insert(row); + addToTotal(ctx, row.category, row.amount); +} + +function deleteSale(ctx: Ctx, id: bigint) { + const old = ctx.db.sale.id.find(id); + if (!old) return; + ctx.db.sale.id.delete(id); + removeFromTotal(ctx, old.category, old.amount); +} + +export const exercise = spacetimedb.reducer(ctx => { + upsertSale(ctx, { id: 1n, category: 'books', amount: 10n }); + upsertSale(ctx, { id: 2n, category: 'books', amount: 20n }); + upsertSale(ctx, { id: 2n, category: 'books', amount: 25n }); + upsertSale(ctx, { id: 3n, category: 'games', amount: 40n }); + deleteSale(ctx, 3n); + deleteSale(ctx, 1n); +}); + +export const category_summary = spacetimedb.view( + { name: 'category_summary', public: true }, t.array(categoryTotal.rowType), ctx => ctx.from.categoryTotal +); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/spec.rs new file mode 100644 index 00000000000..4939ac6682d --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/spec.rs @@ -0,0 +1,25 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_data_parity_scorer}; +use crate::eval::{casing_for_lang, table_name, BenchmarkSpec, ReducerDataParityConfig, SqlBuilder}; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + let sql = SqlBuilder::new(casing_for_lang(lang)); + let columns = sql.cols(&["category", "total_amount", "sale_count"]).join(", "); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "exercise".into(), + args: vec![], + select_query: format!("SELECT {columns} FROM {}", table_name("category_summary", lang)), + id_str: "aggregate_is_synchronized", + collapse_ws: true, + timeout: Duration::from_secs(10), + }, + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/csharp.txt new file mode 100644 index 00000000000..0be092a9636 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/csharp.txt @@ -0,0 +1,27 @@ +Write a SpacetimeDB backend module in C# that maintains and exposes a materialized aggregate. + +TABLES +- Sale (public, accessor Sale) + - Fields: + - Id: ulong (primary key) + - Category: string + - Amount: long +- CategoryTotal (public, accessor CategoryTotal) + - Fields: + - Category: string (primary key) + - TotalAmount: long + - SaleCount: ulong + +REDUCERS +- Exercise() + - This is the only exported reducer + - Keep CategoryTotal transactionally synchronized when a Sale is inserted, changed, moved between categories, or deleted + - Insert books Sale rows with Ids 1 and 2 and Amounts 10 and 20 + - Update the second books Sale to 25 + - Insert and then delete a games Sale with Id 3 + - Delete the first books Sale + +VIEW +- CategorySummary (public, non-anonymous query-builder) + - Expose all CategoryTotal rows + - After Exercise, the only row is books with TotalAmount 25 and SaleCount 1 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/rust.txt new file mode 100644 index 00000000000..29043396e57 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/rust.txt @@ -0,0 +1,27 @@ +Write a SpacetimeDB backend module in Rust that maintains and exposes a materialized aggregate. + +TABLES +- sale (public), struct Sale + - Fields: + - id: u64 (primary key) + - category: String + - amount: i64 +- category_total (public), struct CategoryTotal + - Fields: + - category: String (primary key) + - total_amount: i64 + - sale_count: u64 + +REDUCERS +- exercise() + - This is the only exported reducer + - Keep category_total transactionally synchronized when a sale is inserted, changed, moved between categories, or deleted + - Insert books sales with ids 1 and 2 and amounts 10 and 20 + - Update the second books sale to 25 + - Insert and then delete a games sale with id 3 + - Delete the first books sale + +VIEW +- category_summary (public, non-anonymous query-builder) + - Expose all category_total rows + - After exercise, the only row is books with total_amount 25 and sale_count 1 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/typescript.txt new file mode 100644 index 00000000000..4f0d4616c77 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_065_materialized_aggregate/tasks/typescript.txt @@ -0,0 +1,27 @@ +Write a SpacetimeDB backend module in TypeScript that maintains and exposes a materialized aggregate. + +TABLES +- sale (public) + - Fields: + - id: u64 (primary key) + - category: string + - amount: i64 +- category_total (public) + - Fields: + - category: string (primary key) + - totalAmount: i64 + - saleCount: u64 + +REDUCERS +- exercise() + - This is the only exported reducer + - Keep category_total transactionally synchronized when a sale is inserted, changed, moved between categories, or deleted + - Insert books sales with ids 1 and 2 and amounts 10 and 20 + - Update the second books sale to 25 + - Insert and then delete a games sale with id 3 + - Delete the first books sale + +VIEW +- category_summary (public, non-anonymous query-builder) + - Expose all category_total rows + - After exercise, the only row is books with totalAmount 25 and saleCount 1 diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/answers/csharp.cs new file mode 100644 index 00000000000..b6694a6ac78 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/answers/csharp.cs @@ -0,0 +1,8 @@ +using SpacetimeDB; +public static partial class Module +{ + [Table(Accessor = "Ticket", Public = true)] public partial struct Ticket { [PrimaryKey] public ulong Id; [SpacetimeDB.Index.BTree] public string Status; public string Title; } + [Reducer] public static void Seed(ReducerContext ctx) { ctx.Db.Ticket.Insert(new Ticket { Id = 1, Status = "open", Title = "A" }); ctx.Db.Ticket.Insert(new Ticket { Id = 2, Status = "closed", Title = "B" }); } + [SpacetimeDB.View(Accessor = "OpenTicket", Public = true)] + public static IQuery OpenTicket(ViewContext ctx) => ctx.From.Ticket().Where(ticket => ticket.Status.Eq("open")); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/answers/rust.rs new file mode 100644 index 00000000000..d607333d84d --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/answers/rust.rs @@ -0,0 +1,26 @@ +use spacetimedb::{reducer, table, view, Query, ReducerContext, Table, ViewContext}; +#[table(accessor = ticket, public)] +pub struct Ticket { + #[primary_key] + pub id: u64, + #[index(btree)] + pub status: String, + pub title: String, +} +#[reducer] +pub fn seed(ctx: &ReducerContext) { + ctx.db.ticket().insert(Ticket { + id: 1, + status: "open".into(), + title: "A".into(), + }); + ctx.db.ticket().insert(Ticket { + id: 2, + status: "closed".into(), + title: "B".into(), + }); +} +#[view(accessor = open_ticket, public)] +pub fn open_ticket(ctx: &ViewContext) -> impl Query { + ctx.from.ticket().filter(|ticket| ticket.status.eq("open".to_string())) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/answers/typescript.ts new file mode 100644 index 00000000000..1ed6045b4c6 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/answers/typescript.ts @@ -0,0 +1,5 @@ +import { schema, table, t } from 'spacetimedb/server'; +const ticket = table({ name: 'ticket', public: true }, { id: t.u64().primaryKey(), status: t.string().index('btree'), title: t.string() }); +const spacetimedb = schema({ ticket }); export default spacetimedb; +export const seed = spacetimedb.reducer(ctx => { ctx.db.ticket.insert({ id: 1n, status: 'open', title: 'A' }); ctx.db.ticket.insert({ id: 2n, status: 'closed', title: 'B' }); }); +export const open_ticket = spacetimedb.view({ name: 'open_ticket', public: true }, t.array(ticket.rowType), ctx => ctx.from.ticket.where(ticket => ticket.status.eq('open'))); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/spec.rs new file mode 100644 index 00000000000..d49c659b8fd --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/spec.rs @@ -0,0 +1,24 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_data_parity_scorer}; +use crate::eval::{table_name, BenchmarkSpec, ReducerDataParityConfig}; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + let view = table_name("open_ticket", lang); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "seed".into(), + args: vec![], + select_query: format!("SELECT * FROM {view}"), + id_str: "query_builder_filter", + collapse_ws: true, + timeout: Duration::from_secs(10), + }, + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/csharp.txt new file mode 100644 index 00000000000..1a95f8affc0 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/csharp.txt @@ -0,0 +1,18 @@ +Write a SpacetimeDB backend module in C# that defines a query-builder filter view. + +TABLE +- Ticket (public, accessor Ticket) + - Fields: + - Id: ulong (primary key) + - Status: string (btree index) + - Title: string + +REDUCERS +- Seed() + - Insert open Ticket Id 1 titled "A" + - Insert closed Ticket Id 2 titled "B" + +VIEW +- OpenTicket (public, non-anonymous query-builder) + - Return the query-builder filter for Status "open" + - Do not materialize rows manually diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/rust.txt new file mode 100644 index 00000000000..80e07a7546a --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/rust.txt @@ -0,0 +1,18 @@ +Write a SpacetimeDB backend module in Rust that defines a query-builder filter view. + +TABLE +- ticket (public), struct Ticket + - Fields: + - id: u64 (primary key) + - status: String (btree index) + - title: String + +REDUCERS +- seed() + - Insert open ticket id 1 titled "A" + - Insert closed ticket id 2 titled "B" + +VIEW +- open_ticket (public, non-anonymous query-builder) + - Return the query-builder filter for status "open" + - Do not materialize rows manually diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/typescript.txt new file mode 100644 index 00000000000..07872178aa0 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_066_query_builder_view/tasks/typescript.txt @@ -0,0 +1,18 @@ +Write a SpacetimeDB backend module in TypeScript that defines a query-builder filter view. + +TABLE +- ticket (public) + - Fields: + - id: u64 (primary key) + - status: string (btree index) + - title: string + +REDUCERS +- seed() + - Insert open ticket id 1 titled "A" + - Insert closed ticket id 2 titled "B" + +VIEW +- open_ticket (public, non-anonymous query-builder) + - Return the query-builder filter for status "open" + - Do not materialize rows manually diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/answers/csharp.cs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/answers/csharp.cs new file mode 100644 index 00000000000..a86a546b493 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/answers/csharp.cs @@ -0,0 +1,23 @@ +using SpacetimeDB; + +public static partial class Module +{ + [Table(Accessor = "SourceRow", Public = true)] + public partial struct SourceRow + { + [PrimaryKey] public ulong Id; + public string Value; + [SpacetimeDB.Index.BTree] public bool Visible; + } + + [Reducer] + public static void Seed(ReducerContext ctx) + { + ctx.Db.SourceRow.Insert(new SourceRow { Id = 1, Value = "shown", Visible = true }); + ctx.Db.SourceRow.Insert(new SourceRow { Id = 2, Value = "hidden", Visible = false }); + } + + [SpacetimeDB.View(Accessor = "SourceView", Public = true, PrimaryKey = nameof(SourceRow.Id))] + public static IEnumerable SourceView(AnonymousViewContext ctx) => + ctx.Db.SourceRow.Visible.Filter(true); +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/answers/rust.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/answers/rust.rs new file mode 100644 index 00000000000..3115c51ddd2 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/answers/rust.rs @@ -0,0 +1,29 @@ +use spacetimedb::{table, view, AnonymousViewContext, ReducerContext, Table}; + +#[table(accessor = source_row, public)] +pub struct SourceRow { + #[primary_key] + pub id: u64, + pub value: String, + #[index(btree)] + pub visible: bool, +} + +#[spacetimedb::reducer] +pub fn seed(ctx: &ReducerContext) { + ctx.db.source_row().insert(SourceRow { + id: 1, + value: "shown".into(), + visible: true, + }); + ctx.db.source_row().insert(SourceRow { + id: 2, + value: "hidden".into(), + visible: false, + }); +} + +#[view(accessor = source_view, public, primary_key = id)] +pub fn source_view(ctx: &AnonymousViewContext) -> Vec { + ctx.db.source_row().visible().filter(true).collect() +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/answers/typescript.ts b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/answers/typescript.ts new file mode 100644 index 00000000000..7bfffd801fb --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/answers/typescript.ts @@ -0,0 +1,9 @@ +import { schema, table, t } from 'spacetimedb/server'; +const SourceViewRow = t.row('SourceViewRow', { id: t.u64().primaryKey(), value: t.string(), visible: t.bool().index('btree') }); +const sourceRow = table({ name: 'source_row', public: true }, SourceViewRow); +const spacetimedb = schema({ sourceRow }); export default spacetimedb; +export const seed = spacetimedb.reducer(ctx => { + ctx.db.sourceRow.insert({ id: 1n, value: 'shown', visible: true }); + ctx.db.sourceRow.insert({ id: 2n, value: 'hidden', visible: false }); +}); +export const source_view = spacetimedb.anonymousView({ name: 'source_view', public: true }, t.array(SourceViewRow), ctx => Array.from(ctx.db.sourceRow.visible.filter(true))); diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/spec.rs b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/spec.rs new file mode 100644 index 00000000000..778ba91fefb --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/spec.rs @@ -0,0 +1,23 @@ +use crate::eval::defaults::{default_schema_parity_scorers, make_reducer_data_parity_scorer}; +use crate::eval::{table_name, BenchmarkSpec, ReducerDataParityConfig}; +use std::time::Duration; + +pub fn spec() -> BenchmarkSpec { + BenchmarkSpec::from_tasks_auto(file!(), |lang, route_tag, host_url| { + let mut scorers = default_schema_parity_scorers(host_url, file!(), route_tag); + scorers.push(make_reducer_data_parity_scorer( + host_url, + ReducerDataParityConfig { + src_file: file!(), + route_tag, + reducer: "seed".into(), + args: vec![], + select_query: format!("SELECT * FROM {}", table_name("source_view", lang)), + collapse_ws: true, + timeout: Duration::from_secs(10), + id_str: "visible_rows_only", + }, + )); + scorers + }) +} diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/csharp.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/csharp.txt new file mode 100644 index 00000000000..f0917addc3f --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/csharp.txt @@ -0,0 +1,18 @@ +Write a SpacetimeDB backend module in C# that defines a procedural view with a primary key. + +TABLE +- SourceRow (public, accessor SourceRow) + - Fields: + - Id: ulong (primary key) + - Value: string + - Visible: bool (btree index) + +REDUCERS +- Seed() + - Insert row 1 with Value "shown" and Visible true + - Insert row 2 with Value "hidden" and Visible false + +VIEW +- SourceView (anonymous procedural, public) + - Return visible SourceRow rows + - Explicitly declare Id as the view primary key diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/rust.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/rust.txt new file mode 100644 index 00000000000..a1e0797c3bd --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/rust.txt @@ -0,0 +1,18 @@ +Write a SpacetimeDB backend module in Rust that defines a procedural view with a primary key. + +TABLE +- source_row (public), struct SourceRow + - Fields: + - id: u64 (primary key) + - value: String + - visible: bool (btree index) + +REDUCERS +- seed() + - Insert row 1 with value "shown" and visible true + - Insert row 2 with value "hidden" and visible false + +VIEW +- source_view (anonymous procedural, public) + - Return visible source_row rows + - Explicitly declare id as the view primary key diff --git a/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/typescript.txt b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/typescript.txt new file mode 100644 index 00000000000..b328e4ac382 --- /dev/null +++ b/tools/xtask-llm-benchmark/src/benchmarks/views/t_067_view_primary_key/tasks/typescript.txt @@ -0,0 +1,18 @@ +Write a SpacetimeDB backend module in TypeScript that defines a procedural view with a primary key. + +TABLE +- source_row (public) + - Fields: + - id: u64 (primary key) + - value: string + - visible: boolean (btree index) + +REDUCERS +- seed() + - Insert row 1 with value "shown" and visible true + - Insert row 2 with value "hidden" and visible false + +VIEW +- source_view (anonymous procedural, public) + - Return visible source_row rows + - The returned row type declares id as its primary key diff --git a/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs b/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs index 6ec030a49e8..be76423286d 100644 --- a/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs +++ b/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs @@ -13,7 +13,8 @@ use tokio::runtime::Runtime; use xtask_llm_benchmark::api::ApiClient; use xtask_llm_benchmark::bench::bench_route_concurrency; use xtask_llm_benchmark::bench::runner::{ - build_goldens_only_for_lang, ensure_goldens_built_once, run_selected_or_all_for_model_async_for_lang, + delete_databases, ensure_goldens_built_once, run_selected_or_all_for_model_async_for_lang, + validate_goldens_for_lang, }; use xtask_llm_benchmark::bench::types::{BenchRunContext, RouteRun, RunConfig, RunOutcome}; use xtask_llm_benchmark::context::constants::ALL_MODES; @@ -100,7 +101,7 @@ struct RunArgs { #[arg(long, conflicts_with = "goldens_only")] hash_only: bool, - /// Build/publish goldens only (skip LLM calls) + /// Build/publish and self-score goldens (skip LLM calls) #[arg(long, conflicts_with = "hash_only")] goldens_only: bool, @@ -230,7 +231,14 @@ fn cmd_run(args: RunArgs) -> Result<()> { fn run_benchmarks(args: RunArgs) -> Result<()> { let dry_run = args.dry_run; let local_analysis = args.local_analysis; - let dry_run_id = dry_run.then(|| chrono::Utc::now().format("%Y-%m-%d_%H%M%S").to_string()); + let dry_run_id = dry_run.then(|| { + format!( + "{}-{}-{}", + chrono::Utc::now().format("%Y-%m-%d_%H%M%S_%3f"), + args.lang.as_str(), + std::process::id() + ) + }); let should_fetch_remote_routes = should_fetch_remote_routes(&args); let needs_api_client = should_fetch_remote_routes || !dry_run; @@ -279,7 +287,7 @@ fn run_benchmarks(args: RunArgs) -> Result<()> { eprintln!("[warn] failed to upload task catalog: {e}"); } - let RuntimeInit { runtime, guard } = initialize_runtime(config.hash_only)?; + let RuntimeInit { runtime, mut guard } = initialize_runtime(config.hash_only)?; config.host = guard.as_ref().map(|g| g.host_url.clone()); @@ -295,25 +303,17 @@ fn run_benchmarks(args: RunArgs) -> Result<()> { if config.goldens_only { let rt = runtime.as_ref().expect("runtime required for --goldens-only"); - rt.block_on(build_goldens_only_for_lang( + rt.block_on(validate_goldens_for_lang( config.host.clone(), &bench_root, config.lang, selectors_ref, ))?; - println!("[{}] goldens-only build complete", config.lang.as_str()); + println!("[{}] goldens-only validation complete", config.lang.as_str()); return Ok(()); } let llm_provider = if !config.goldens_only && !config.hash_only { - let rt = runtime.as_ref().expect("failed to initialize runtime for goldens"); - rt.block_on(ensure_goldens_built_once( - config.host.clone(), - &bench_root, - config.lang, - selectors_ref, - ))?; - let provider = make_provider_from_env()?; let rt = runtime.as_ref().expect("failed to initialize runtime for preflight"); let routes = filter_routes(&config); @@ -326,14 +326,21 @@ fn run_benchmarks(args: RunArgs) -> Result<()> { let mut all_outcomes: Vec = Vec::new(); for mode in modes { - let outcomes = run_mode_benchmarks( + let result = run_mode_benchmarks( &mode, config.lang, &config, &bench_root, runtime.as_ref(), llm_provider.as_ref(), - )?; + ); + let outcomes = match result { + Ok(outcomes) => outcomes, + Err(error) => { + report_server_status(guard.as_mut()); + return Err(error); + } + }; all_outcomes.extend(outcomes); } @@ -358,6 +365,17 @@ fn run_benchmarks(args: RunArgs) -> Result<()> { Ok(()) } +fn report_server_status(guard: Option<&mut SpacetimeDbGuard>) { + let Some(guard) = guard else { + return; + }; + match guard.child.try_wait() { + Ok(Some(status)) => eprintln!("[server] local SpacetimeDB exited unexpectedly: {status}"), + Ok(None) => eprintln!("[server] local SpacetimeDB is still running after benchmark failure"), + Err(error) => eprintln!("[server] failed to read local SpacetimeDB exit status: {error}"), + } +} + /* ------------------------------ analyze ------------------------------ */ fn cmd_analyze(args: AnalyzeArgs) -> Result<()> { @@ -756,6 +774,10 @@ async fn run_many_routes_for_mode( async move { println!("\u{2192} running {}", route.display_name); + let golden_scope = xtask_llm_benchmark::bench::run_scope_tag(mode, route.vendor.slug(), &route.api_model); + let golden_databases = + ensure_goldens_built_once(host.clone(), bench_root, lang, selectors, &golden_scope).await?; + let per = BenchRunContext { bench_root, mode, @@ -772,7 +794,16 @@ async fn run_many_routes_for_mode( dry_run_id, }; - let outcomes = run_selected_or_all_for_model_async_for_lang(&per).await?; + let run_result = run_selected_or_all_for_model_async_for_lang(&per).await; + let cleanup_result = delete_databases(golden_databases).await; + let outcomes = match (run_result, cleanup_result) { + (Ok(outcomes), Ok(())) => outcomes, + (Err(error), Ok(())) => return Err(error), + (Ok(_), Err(error)) => return Err(error.context("failed to delete golden databases")), + (Err(error), Err(cleanup_error)) => { + return Err(error.context(format!("also failed to delete golden databases: {cleanup_error:#}"))) + } + }; Ok::<_, anyhow::Error>(RouteRun { route_name: route.display_name.to_string(), diff --git a/tools/xtask-llm-benchmark/src/eval/defaults.rs b/tools/xtask-llm-benchmark/src/eval/defaults.rs index ea40f7a8b93..f78fd6c1b5b 100644 --- a/tools/xtask-llm-benchmark/src/eval/defaults.rs +++ b/tools/xtask-llm-benchmark/src/eval/defaults.rs @@ -1,7 +1,8 @@ -use crate::bench::utils::sanitize_db_name; +use crate::bench::utils::{golden_db_name, sanitize_db_name}; use crate::eval::scorers::{ - ReducerCallBothScorer, ReducerDataParityScorer, ReducerSqlCountScorer, SchemaParityScorer, Scorer, - SqlCountOnlyScorer, SqlExecBothScorer, + CallOutputParityScorer, EventuallySqlCountScorer, HttpRouteCase, HttpRouteParityScorer, ReducerCallBothScorer, + ReducerDataParityScorer, ReducerSqlCountScorer, SchemaParityScorer, Scorer, SqlCountOnlyScorer, + SqlDistinctRowsScorer, SqlExecBothScorer, SqlOutputExcludesScorer, }; use crate::eval::{derive_cat_task_from_file, ReducerDataParityConfig, ReducerSqlCountConfig}; use std::time::Duration; @@ -9,7 +10,7 @@ use std::time::Duration; pub fn default_schema_parity_scorers(host_url: &str, src_file: &str, route_tag: &str) -> Vec> { let (cat, task) = derive_cat_task_from_file(src_file); - let golden_db = sanitize_db_name(&format!("{}-{}-golden", cat, task)); + let golden_db = golden_db_name(&cat, &task, route_tag); let llm_db = sanitize_db_name(&format!("{}-{}-{}-llm", cat, task, route_tag)); vec![Box::new(SchemaParityScorer { @@ -58,9 +59,51 @@ pub fn make_sql_count_only_scorer( }) } +pub fn make_sql_distinct_rows_scorer( + host_url: &str, + src_file: &str, + route_tag: &str, + sql: impl Into, + expected: usize, + id_str: &'static str, + timeout: Duration, +) -> Box { + let (cat, task) = derive_cat_task_from_file(src_file); + let llm_db = sanitize_db_name(&format!("{}-{}-{}-llm", cat, task, route_tag)); + Box::new(SqlDistinctRowsScorer { + server: host_url.to_string(), + db: llm_db, + sql: sql.into(), + expected, + timeout, + id_str, + }) +} + +pub fn make_eventually_sql_count_scorer( + host_url: &str, + src_file: &str, + route_tag: &str, + sql: impl Into, + expected: i64, + id_str: &'static str, + timeout: Duration, +) -> Box { + let (cat, task) = derive_cat_task_from_file(src_file); + let llm_db = sanitize_db_name(&format!("{}-{}-{}-llm", cat, task, route_tag)); + Box::new(EventuallySqlCountScorer { + server: host_url.to_string(), + db: llm_db, + sql: sql.into(), + expected, + timeout, + id_str, + }) +} + pub fn make_reducer_data_parity_scorer(host_url: &str, cfg: ReducerDataParityConfig<'_>) -> Box { let (cat, task) = derive_cat_task_from_file(cfg.src_file); - let golden_db = sanitize_db_name(&format!("{}-{}-golden", cat, task)); + let golden_db = golden_db_name(&cat, &task, cfg.route_tag); let llm_db = sanitize_db_name(&format!("{}-{}-{}-llm", cat, task, cfg.route_tag)); Box::new(ReducerDataParityScorer { @@ -85,7 +128,7 @@ pub fn make_sql_exec_both_scorer( timeout: Duration, ) -> Box { let (cat, task) = derive_cat_task_from_file(src_file); - let golden_db = sanitize_db_name(&format!("{}-{}-golden", cat, task)); + let golden_db = golden_db_name(&cat, &task, route_tag); let llm_db = sanitize_db_name(&format!("{}-{}-{}-llm", cat, task, route_tag)); Box::new(SqlExecBothScorer { @@ -105,9 +148,40 @@ pub fn make_reducer_call_both_scorer( reducer: &str, args: Vec, id_str: &'static str, +) -> Box { + make_reducer_call_both_scorer_with_attempts(host_url, src_file, route_tag, reducer, args, id_str, 1) +} + +pub fn make_sql_output_excludes_scorer( + host_url: &str, + src_file: &str, + route_tag: &str, + sql: impl Into, + excluded: Vec, + id_str: &'static str, +) -> Box { + let (cat, task) = derive_cat_task_from_file(src_file); + let llm_db = sanitize_db_name(&format!("{}-{}-{}-llm", cat, task, route_tag)); + Box::new(SqlOutputExcludesScorer { + server: host_url.to_string(), + db: llm_db, + sql: sql.into(), + excluded, + id_str, + }) +} + +pub fn make_reducer_call_both_scorer_with_attempts( + host_url: &str, + src_file: &str, + route_tag: &str, + reducer: &str, + args: Vec, + id_str: &'static str, + attempts: usize, ) -> Box { let (cat, task) = derive_cat_task_from_file(src_file); - let golden_db = sanitize_db_name(&format!("{}-{}-golden", cat, task)); + let golden_db = golden_db_name(&cat, &task, route_tag); let llm_db = sanitize_db_name(&format!("{}-{}-{}-llm", cat, task, route_tag)); Box::new(ReducerCallBothScorer { @@ -116,6 +190,71 @@ pub fn make_reducer_call_both_scorer( llm_db, reducer: reducer.to_string(), args, + attempts, id_str, }) as Box } + +pub fn make_call_output_parity_scorer( + host_url: &str, + src_file: &str, + route_tag: &str, + function: &str, + args: Vec, + id_str: &'static str, +) -> Box { + make_call_output_parity_scorer_with_attempts(host_url, src_file, route_tag, function, args, id_str, 1) +} + +pub fn make_call_output_parity_scorer_with_attempts( + host_url: &str, + src_file: &str, + route_tag: &str, + function: &str, + args: Vec, + id_str: &'static str, + attempts: usize, +) -> Box { + let (cat, task) = derive_cat_task_from_file(src_file); + let golden_db = golden_db_name(&cat, &task, route_tag); + let llm_db = sanitize_db_name(&format!("{}-{}-{}-llm", cat, task, route_tag)); + Box::new(CallOutputParityScorer { + server: host_url.to_string(), + golden_db, + llm_db, + function: function.to_string(), + args, + collapse_ws: true, + attempts, + id_str, + }) +} + +pub fn make_http_route_parity_scorer( + host_url: &str, + src_file: &str, + route_tag: &str, + cases: Vec<(&str, &str, Option<&str>)>, + compare_content_type: bool, + id_str: &'static str, +) -> Box { + let (cat, task) = derive_cat_task_from_file(src_file); + let golden_db = golden_db_name(&cat, &task, route_tag); + let llm_db = sanitize_db_name(&format!("{}-{}-{}-llm", cat, task, route_tag)); + Box::new(HttpRouteParityScorer { + server: host_url.to_string(), + golden_db, + llm_db, + id_str, + compare_content_type, + timeout: Duration::from_secs(10), + cases: cases + .into_iter() + .map(|(method, path, body)| HttpRouteCase { + method: method.to_string(), + path: path.to_string(), + body: body.map(str::to_string), + }) + .collect(), + }) +} diff --git a/tools/xtask-llm-benchmark/src/eval/scorers.rs b/tools/xtask-llm-benchmark/src/eval/scorers.rs index 2b1c850a562..a38e23b7308 100644 --- a/tools/xtask-llm-benchmark/src/eval/scorers.rs +++ b/tools/xtask-llm-benchmark/src/eval/scorers.rs @@ -1,9 +1,9 @@ use crate::bench::utils::debug_llm_verbose; -use crate::eval::{normalize, sql_exec, ScoreDetails}; +use crate::eval::utils::{run_with_timeout, sql_exec_with_timeout}; +use crate::eval::{normalize, spacetime_command, ScoreDetails}; use serde_json::{json, Value}; use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; -use std::process::{Command, Stdio}; use std::time::{Duration, Instant}; use std::{io, thread}; @@ -20,6 +20,67 @@ pub struct SchemaParityScorer { pub id_str: &'static str, } +#[derive(Debug, Default, PartialEq, Eq)] +struct SchemaSnapshot { + tables: BTreeMap>, + reducers: BTreeSet, + exports: BTreeSet, + row_level_security: BTreeSet, +} + +#[derive(Debug, Default)] +struct SchemaNames { + tables: BTreeMap, + functions: BTreeMap, +} + +impl SchemaNames { + fn from_schema(schema: &Value) -> Self { + let Some(sections) = schema.get("sections") else { + return Self::default(); + }; + let mut names = Self::default(); + + if let Some(entries) = sections + .as_array() + .into_iter() + .flatten() + .find_map(|section| section.get("ExplicitNames")) + .and_then(|names| names.get("entries")) + .and_then(Value::as_array) + { + for entry in entries { + if let Some(mapping) = entry.get("Table") { + insert_name_mapping(&mut names.tables, mapping); + } else if let Some(mapping) = entry.get("Function") { + insert_name_mapping(&mut names.functions, mapping); + } + } + } + names + } + + fn table(&self, name: String) -> String { + self.canonical(name, &self.tables) + } + + fn function(&self, name: String) -> String { + self.canonical(name, &self.functions) + } + + fn canonical(&self, name: String, explicit: &BTreeMap) -> String { + explicit.get(&name).cloned().unwrap_or(name) + } +} + +fn insert_name_mapping(names: &mut BTreeMap, mapping: &Value) { + let source = schema_name(mapping.get("source_name")); + let canonical = schema_name(mapping.get("canonical_name")); + if !source.is_empty() && !canonical.is_empty() { + names.insert(source, canonical); + } +} + impl Scorer for SchemaParityScorer { fn id(&self) -> &'static str { self.id_str @@ -44,12 +105,14 @@ impl Scorer for SchemaParityScorer { } } - let (tables_a, reducers_a) = extract_schema(&golden); - let (tables_b, reducers_b) = extract_schema(&llm); + let schema_a = extract_schema(&golden); + let schema_b = extract_schema(&llm); - let tables_diff = diff_maps(&tables_a, &tables_b); - let reducers_diff = diff_sets(&reducers_a, &reducers_b); - let pass = tables_diff.is_null() && reducers_diff.is_null(); + let tables_diff = diff_maps(&schema_a.tables, &schema_b.tables); + let reducers_diff = diff_sets(&schema_a.reducers, &schema_b.reducers); + let exports_diff = diff_sets(&schema_a.exports, &schema_b.exports); + let rls_diff = diff_sets(&schema_a.row_level_security, &schema_b.row_level_security); + let pass = tables_diff.is_null() && reducers_diff.is_null() && exports_diff.is_null() && rls_diff.is_null(); ScoreDetails { pass, @@ -60,8 +123,12 @@ impl Scorer for SchemaParityScorer { "llm_db": self.llm_db, "tables_equal": tables_diff.is_null(), "reducers_equal": reducers_diff.is_null(), + "exports_equal": exports_diff.is_null(), + "row_level_security_equal": rls_diff.is_null(), "tables_diff": tables_diff, "reducers_diff": reducers_diff, + "exports_diff": exports_diff, + "row_level_security_diff": rls_diff, }), } } @@ -69,82 +136,378 @@ impl Scorer for SchemaParityScorer { /* helpers */ -fn run_with_timeout(mut cmd: Command, cwd: &Path, timeout: Duration) -> io::Result<(i32, Vec, Vec)> { - let mut child = cmd - .current_dir(cwd) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()?; - let start = Instant::now(); - loop { - if let Some(status) = child.try_wait()? { - let out = child.wait_with_output()?; - let code = status.code().unwrap_or(-1); - return Ok((code, out.stdout, out.stderr)); - } - if start.elapsed() > timeout { - let _ = child.kill(); - return Err(io::Error::new(io::ErrorKind::TimedOut, "process timeout")); - } - thread::sleep(Duration::from_millis(30)); - } +fn describe_db(server: &str, db: &str, timeout: Duration) -> io::Result { + thread::scope(|scope| { + scope + .spawn(|| describe_db_blocking(server, db, timeout)) + .join() + .map_err(|_| io::Error::other("schema request thread panicked"))? + }) } -fn describe_db(server: &str, db: &str, timeout: Duration) -> io::Result { - let mut cmd = Command::new("spacetime"); - cmd.arg("describe") - .arg("--json") - .arg("-s") - .arg(server) - .arg("-y") - .arg(db); - let (code, out, err) = run_with_timeout(cmd, Path::new("."), timeout)?; - if code != 0 { - return Err(io::Error::other(format!( - "describe failed: {}", - String::from_utf8_lossy(&err) - ))); +fn describe_db_blocking(server: &str, db: &str, timeout: Duration) -> io::Result { + let url = format!( + "{}/v1/database/{}/schema", + server.trim_end_matches('/'), + urlencoding::encode(db) + ); + let client = reqwest::blocking::Client::builder() + .timeout(timeout) + .build() + .map_err(io::Error::other)?; + let response = client + .get(url) + .query(&[("version", "10")]) + .send() + .map_err(io::Error::other)?; + let status = response.status(); + if !status.is_success() { + let body = response.text().unwrap_or_default(); + return Err(io::Error::other(format!("schema request failed ({status}): {body}"))); } - let v: Value = serde_json::from_slice(&out).map_err(|e| io::Error::other(format!("parse json: {}", e)))?; - Ok(v) + response.json().map_err(io::Error::other) } -fn extract_schema(v: &Value) -> (BTreeMap>, BTreeSet) { - let mut tables: BTreeMap> = BTreeMap::new(); - let mut reducers: BTreeSet = BTreeSet::new(); +fn extract_schema(v: &Value) -> SchemaSnapshot { + let mut schema = SchemaSnapshot::default(); + let names = SchemaNames::from_schema(v); + let typespace = v.get("typespace").or_else(|| section(v, "Typespace")); + let types = typespace + .and_then(|value| value.get("types")) + .and_then(Value::as_array) + .map(Vec::as_slice) + .unwrap_or_default(); - if let Some(ts) = v.get("tables").and_then(|x| x.as_array()) { + if let Some(ts) = schema_array(v, "tables", "Tables") { for t in ts { - let name = t.get("name").and_then(|x| x.as_str()).unwrap_or("").to_string(); + let name = names.table(schema_name(t.get("source_name").or_else(|| t.get("name")))); let mut cols = BTreeMap::new(); - if let Some(cs) = t.get("columns").and_then(|x| x.as_array()) { + + // Older CLI descriptions put columns directly on the table. Keep + // accepting that shape while also reading the current typespace + // representation. + let legacy_columns = t.get("columns").and_then(Value::as_array); + let current_columns = t + .get("product_type_ref") + .and_then(Value::as_u64) + .and_then(|idx| types.get(idx as usize)) + .and_then(|ty| ty.pointer("/Product/elements")) + .and_then(Value::as_array); + + if let Some(cs) = legacy_columns.or(current_columns) { for c in cs { - let cname = c.get("name").and_then(|x| x.as_str()).unwrap_or("").to_string(); - let cty = c.get("type").and_then(|x| x.as_str()).unwrap_or("").to_string(); + let cname = schema_name(c.get("name")); + let cty = c + .get("type") + .and_then(Value::as_str) + .map(str::to_owned) + .or_else(|| { + c.get("algebraic_type") + .map(|ty| canonical_type(ty, types, &mut BTreeSet::new()).to_string()) + }) + .unwrap_or_default(); cols.insert(cname, cty); } } - tables.insert(name, cols); + + let column_names: Vec = current_columns + .or(legacy_columns) + .into_iter() + .flatten() + .map(|column| schema_name(column.get("name"))) + .collect(); + + insert_schema_property( + &mut cols, + "primary_key", + column_list(t.get("primary_key"), &column_names), + ); + insert_schema_property(&mut cols, "indexes", normalize_indexes(t.get("indexes"), &column_names)); + insert_schema_property( + &mut cols, + "constraints", + normalize_constraints(t.get("constraints"), &column_names), + ); + insert_schema_property( + &mut cols, + "sequences", + normalize_sequences(t.get("sequences"), &column_names), + ); + insert_schema_property(&mut cols, "schedule", canonical_value(t.get("schedule"))); + insert_schema_property(&mut cols, "table_type", canonical_value(t.get("table_type"))); + insert_schema_property(&mut cols, "table_access", canonical_value(t.get("table_access"))); + insert_schema_property(&mut cols, "default_values", canonical_value(t.get("default_values"))); + insert_schema_property(&mut cols, "is_event", canonical_value(t.get("is_event"))); + schema.tables.insert(name, cols); } } - if let Some(rs) = v.get("reducers").and_then(|x| x.as_array()) { + if let Some(rs) = schema_array(v, "reducers", "Reducers") { for r in rs { - let name = r.get("name").and_then(|x| x.as_str()).unwrap_or(""); - let sig = if let Some(args) = r.get("args").and_then(|x| x.as_array()) { - let tys: Vec = args + let name = names.function(schema_name(r.get("source_name").or_else(|| r.get("name")))); + let mut sig = format!("{}({})", name, parameter_types(r, types).join(",")); + append_field(&mut sig, "visibility", r.get("visibility")); + append_type(&mut sig, "ok", r.get("ok_return_type"), types); + append_type(&mut sig, "err", r.get("err_return_type"), types); + append_field(&mut sig, "lifecycle", r.get("lifecycle")); + schema.reducers.insert(sig); + } + } + + for export in v.get("misc_exports").and_then(Value::as_array).into_iter().flatten() { + if let Some(procedure) = export.get("Procedure") { + schema + .exports + .insert(function_export("procedure", procedure, types, &names)); + } else if let Some(view) = export.get("View") { + schema.exports.insert(view_export(view, types, &names)); + } else if let Some(default) = export.get("ColumnDefaultValue") { + schema + .exports + .insert(format!("column_default:{}", canonical_value(Some(default)))); + } + } + + for procedure in schema_array(v, "procedures", "Procedures").into_iter().flatten() { + schema + .exports + .insert(function_export("procedure", procedure, types, &names)); + } + for view in schema_array(v, "views", "Views").into_iter().flatten() { + schema.exports.insert(view_export(view, types, &names)); + } + insert_canonical_exports(&mut schema.exports, v, "schedules", "Schedules", "schedule"); + insert_canonical_exports( + &mut schema.exports, + v, + "life_cycle_reducers", + "LifeCycleReducers", + "lifecycle", + ); + insert_http_routes(&mut schema.exports, v); + insert_canonical_exports( + &mut schema.exports, + v, + "view_primary_keys", + "ViewPrimaryKeys", + "view_primary_key", + ); + + for rule in schema_array(v, "row_level_security", "RowLevelSecurity") + .into_iter() + .flatten() + { + schema.row_level_security.insert(canonical_value(Some(rule))); + } + + schema +} + +fn section<'a>(v: &'a Value, name: &str) -> Option<&'a Value> { + v.get("sections") + .and_then(Value::as_array) + .into_iter() + .flatten() + .find_map(|item| item.get(name)) +} + +fn schema_array<'a>(v: &'a Value, direct_name: &str, section_name: &str) -> Option<&'a Vec> { + v.get(direct_name) + .or_else(|| section(v, section_name)) + .and_then(Value::as_array) +} + +fn parameter_types(function: &Value, types: &[Value]) -> Vec { + function + .pointer("/params/elements") + .and_then(Value::as_array) + .or_else(|| function.get("args").and_then(Value::as_array)) + .into_iter() + .flatten() + .map(|param| { + param + .get("algebraic_type") + .map(|ty| canonical_type(ty, types, &mut BTreeSet::new()).to_string()) + .or_else(|| param.get("type").and_then(Value::as_str).map(str::to_owned)) + .unwrap_or_default() + }) + .collect() +} + +fn append_field(signature: &mut String, label: &str, value: Option<&Value>) { + if let Some(value) = value { + signature.push_str(&format!("|{label}={}", canonical_value(Some(value)))); + } +} + +fn append_type(signature: &mut String, label: &str, value: Option<&Value>, types: &[Value]) { + if let Some(value) = value { + signature.push_str(&format!( + "|{label}={}", + canonical_type(value, types, &mut BTreeSet::new()) + )); + } +} + +fn function_export(kind: &str, function: &Value, types: &[Value], names: &SchemaNames) -> String { + let name = names.function(schema_name( + function.get("source_name").or_else(|| function.get("name")), + )); + let mut signature = format!("{kind}:{name}({})", parameter_types(function, types).join(",")); + append_type(&mut signature, "return", function.get("return_type"), types); + append_field(&mut signature, "visibility", function.get("visibility")); + signature +} + +fn view_export(view: &Value, types: &[Value], names: &SchemaNames) -> String { + let mut signature = function_export("view", view, types, names); + append_field(&mut signature, "public", view.get("is_public")); + append_field(&mut signature, "anonymous", view.get("is_anonymous")); + signature +} + +fn insert_canonical_exports( + exports: &mut BTreeSet, + schema: &Value, + direct_name: &str, + section_name: &str, + kind: &str, +) { + for export in schema_array(schema, direct_name, section_name).into_iter().flatten() { + exports.insert(format!("{kind}:{}", canonical_value(Some(export)))); + } +} + +fn insert_http_routes(exports: &mut BTreeSet, schema: &Value) { + for route in schema_array(schema, "http_routes", "HttpRoutes").into_iter().flatten() { + let mut route = route.clone(); + if let Some(route) = route.as_object_mut() { + route.remove("handler_function"); + } + exports.insert(format!("http_route:{}", canonical_value(Some(&route)))); + } +} + +fn schema_name(value: Option<&Value>) -> String { + value + .and_then(Value::as_str) + .or_else(|| value.and_then(|value| value.get("some")).and_then(Value::as_str)) + .unwrap_or("") + .to_owned() +} + +fn canonical_type(value: &Value, types: &[Value], visiting: &mut BTreeSet) -> Value { + if let Some(idx) = value.get("Ref").and_then(Value::as_u64).map(|idx| idx as usize) { + if !visiting.insert(idx) { + return json!({ "recursive_ref": idx }); + } + let resolved = types + .get(idx) + .map(|value| canonical_type(value, types, visiting)) + .unwrap_or_else(|| json!({ "missing_ref": idx })); + visiting.remove(&idx); + return resolved; + } + + match value { + Value::Array(values) => Value::Array( + values + .iter() + .map(|value| canonical_type(value, types, visiting)) + .collect(), + ), + Value::Object(values) => Value::Object( + values + .iter() + .map(|(key, value)| (key.clone(), canonical_type(value, types, visiting))) + .collect(), + ), + _ => value.clone(), + } +} + +fn column_name(value: &Value, columns: &[String]) -> String { + value + .as_u64() + .and_then(|idx| columns.get(idx as usize)) + .cloned() + .unwrap_or_else(|| value.to_string()) +} + +fn column_list(value: Option<&Value>, columns: &[String]) -> String { + value + .and_then(Value::as_array) + .map(|values| { + values + .iter() + .map(|value| column_name(value, columns)) + .collect::>() + .join(",") + }) + .unwrap_or_default() +} + +fn normalize_indexes(value: Option<&Value>, columns: &[String]) -> String { + let mut indexes = BTreeSet::new(); + for index in value.and_then(Value::as_array).into_iter().flatten() { + let Some(algorithm) = index.get("algorithm").and_then(Value::as_object) else { + continue; + }; + for (kind, indexed_columns) in algorithm { + let normalized_columns = match indexed_columns { + Value::Array(values) => values .iter() - .map(|a| a.get("type").and_then(|x| x.as_str()).unwrap_or("").to_string()) - .collect(); - format!("{}({})", name, tys.join(",")) - } else { - format!("{}()", name) + .map(|value| column_name(value, columns)) + .collect::>() + .join(","), + value => column_name(value, columns), }; - reducers.insert(sig); + indexes.insert(format!("{kind}({normalized_columns})")); } } + indexes.into_iter().collect::>().join(";") +} - (tables, reducers) +fn normalize_constraints(value: Option<&Value>, columns: &[String]) -> String { + let mut constraints = BTreeSet::new(); + for constraint in value.and_then(Value::as_array).into_iter().flatten() { + let Some(data) = constraint.get("data").and_then(Value::as_object) else { + continue; + }; + for (kind, detail) in data { + let normalized_columns = column_list(detail.get("columns"), columns); + constraints.insert(format!("{kind}({normalized_columns})")); + } + } + constraints.into_iter().collect::>().join(";") +} + +fn normalize_sequences(value: Option<&Value>, columns: &[String]) -> String { + let mut sequences = BTreeSet::new(); + for sequence in value.and_then(Value::as_array).into_iter().flatten() { + let column = sequence + .get("column") + .map(|value| column_name(value, columns)) + .unwrap_or_default(); + let increment = canonical_value(sequence.get("increment")); + let start = canonical_value(sequence.get("start")); + let min = canonical_value(sequence.get("min_value")); + let max = canonical_value(sequence.get("max_value")); + sequences.insert(format!( + "{column}:increment={increment}:start={start}:min={min}:max={max}" + )); + } + sequences.into_iter().collect::>().join(";") +} + +fn canonical_value(value: Option<&Value>) -> String { + value.map(Value::to_string).unwrap_or_default() +} + +fn insert_schema_property(columns: &mut BTreeMap, name: &str, value: String) { + if !value.is_empty() { + columns.insert(format!("@{name}"), value); + } } fn diff_maps(a: &BTreeMap>, b: &BTreeMap>) -> Value { @@ -197,7 +560,17 @@ fn err_details(phase: &str, e: io::Error) -> ScoreDetails { /* reducer/sql helpers */ pub fn call_reducer_json_out(db: &str, reducer: &str, args: &[Value], host: Option<&str>) -> Result { - let mut cmd = Command::new("spacetime"); + call_reducer_json_out_with_timeout(db, reducer, args, host, Duration::from_secs(30)) +} + +fn call_reducer_json_out_with_timeout( + db: &str, + reducer: &str, + args: &[Value], + host: Option<&str>, + timeout: Duration, +) -> Result { + let mut cmd = spacetime_command(); cmd.arg("call").arg(db).arg(reducer); for v in args { @@ -212,28 +585,28 @@ pub fn call_reducer_json_out(db: &str, reducer: &str, args: &[Value], host: Opti eprintln!("[dbg] spacetime call: {:?}", cmd); } - let out = cmd - .output() - .map_err(|e| format!("failed to spawn spacetime call: {e}"))?; + let (code, stdout, stderr) = run_with_timeout(cmd, Path::new("."), timeout) + .map_err(|e| format!("spacetime call failed or timed out: {e}"))?; if debug_llm_verbose() { eprintln!( "[dbg] spacetime call exit={} stdout:\n{}\n-- stderr:\n{}\n", - out.status.code().unwrap_or(-1), - String::from_utf8_lossy(&out.stdout), - String::from_utf8_lossy(&out.stderr) + code, + String::from_utf8_lossy(&stdout), + String::from_utf8_lossy(&stderr) ); } - if !out.status.success() { - return Err(format!( - "spacetime call failed:\n{}", - String::from_utf8_lossy(&out.stderr) - )); + if code != 0 { + return Err(format!("spacetime call failed:\n{}", String::from_utf8_lossy(&stderr))); } - Ok(String::from_utf8_lossy(&out.stdout).to_string()) + Ok(String::from_utf8_lossy(&stdout).to_string()) } pub fn sql_raw(db: &str, query: &str, host: Option<&str>) -> Result { - let mut cmd = Command::new("spacetime"); + sql_raw_with_timeout(db, query, host, Duration::from_secs(30)) +} + +fn sql_raw_with_timeout(db: &str, query: &str, host: Option<&str>, timeout: Duration) -> Result { + let mut cmd = spacetime_command(); cmd.arg("sql").arg(db).arg(query); if let Some(h) = host { cmd.arg("--server").arg(h); @@ -243,28 +616,53 @@ pub fn sql_raw(db: &str, query: &str, host: Option<&str>) -> Result, timeout: Duration) -> Result { + let mut cmd = spacetime_command(); + cmd.arg("sql").arg(db).arg(query).args(["--format", "json"]); + if let Some(h) = host { + cmd.arg("--server").arg(h); + } + + let (code, stdout, stderr) = run_with_timeout(cmd, Path::new("."), timeout) + .map_err(|e| format!("spacetime sql failed or timed out: {e}"))?; + if code != 0 { + return Err(format!("spacetime sql failed:\n{}", String::from_utf8_lossy(&stderr))); + } + serde_json::from_slice(&stdout).map_err(|error| format!("invalid JSON SQL output: {error}")) +} + +fn distinct_sql_row_count(output: &Value) -> Result { + let rows = output + .as_array() + .and_then(|statements| statements.first()) + .and_then(|statement| statement.get("rows")) + .and_then(Value::as_array) + .ok_or_else(|| "JSON SQL output is missing rows".to_string())?; + Ok(rows.iter().map(Value::to_string).collect::>().len()) } pub fn sql_count(db: &str, query: &str, host: Option<&str>) -> Result { - let mut cmd = Command::new("spacetime"); + sql_count_with_timeout(db, query, host, Duration::from_secs(30)) +} + +fn sql_count_with_timeout(db: &str, query: &str, host: Option<&str>, timeout: Duration) -> Result { + let mut cmd = spacetime_command(); cmd.arg("sql").arg(db).arg(query); if let Some(h) = host { cmd.arg("--server").arg(h); @@ -274,24 +672,20 @@ pub fn sql_count(db: &str, query: &str, host: Option<&str>) -> Result() { if debug_llm_verbose() { @@ -329,7 +723,8 @@ impl Scorer for ReducerSqlEqualsScorer { self.reducer, self.args, self.db, self.server ); } - let call_res = call_reducer_json_out(&self.db, &self.reducer, &self.args, Some(&self.server)); + let call_res = + call_reducer_json_out_with_timeout(&self.db, &self.reducer, &self.args, Some(&self.server), self.timeout); if let Err(e) = call_res { return ScoreDetails { pass: false, @@ -341,7 +736,7 @@ impl Scorer for ReducerSqlEqualsScorer { if debug_llm_verbose() { eprintln!("[dbg] ReducerSqlEqualsScorer: running sql: {}", self.sql); } - match sql_raw(&self.db, &self.sql, Some(&self.server)) { + match sql_raw_with_timeout(&self.db, &self.sql, Some(&self.server), self.timeout) { Ok(out) => { let actual = normalize(&out, self.collapse_ws); let expected = normalize(&self.expected, self.collapse_ws); @@ -398,7 +793,8 @@ impl Scorer for ReducerSqlCountScorer { self.reducer, self.args, self.db, self.server ); } - let call = call_reducer_json_out(&self.db, &self.reducer, &self.args, Some(&self.server)); + let call = + call_reducer_json_out_with_timeout(&self.db, &self.reducer, &self.args, Some(&self.server), self.timeout); if let Err(e) = call { return ScoreDetails { pass: false, @@ -410,7 +806,7 @@ impl Scorer for ReducerSqlCountScorer { if debug_llm_verbose() { eprintln!("[dbg] ReducerSqlCountScorer: running sql: {}", self.sql); } - match sql_count(&self.db, &self.sql, Some(&self.server)) { + match sql_count_with_timeout(&self.db, &self.sql, Some(&self.server), self.timeout) { Ok(n) => { let pass = n == self.expected; if debug_llm_verbose() { @@ -456,14 +852,26 @@ impl Scorer for ReducerDataParityScorer { ); } - if let Err(e) = call_reducer_json_out(&self.golden_db, &self.reducer, &self.args, Some(&self.server)) { + if let Err(e) = call_reducer_json_out_with_timeout( + &self.golden_db, + &self.reducer, + &self.args, + Some(&self.server), + self.timeout, + ) { return ScoreDetails { pass: false, partial: 0.0, notes: json!({"phase":"call_reducer_golden","error":e}), }; } - if let Err(e) = call_reducer_json_out(&self.llm_db, &self.reducer, &self.args, Some(&self.server)) { + if let Err(e) = call_reducer_json_out_with_timeout( + &self.llm_db, + &self.reducer, + &self.args, + Some(&self.server), + self.timeout, + ) { return ScoreDetails { pass: false, partial: 0.0, @@ -474,7 +882,7 @@ impl Scorer for ReducerDataParityScorer { if debug_llm_verbose() { eprintln!("[dbg] query for parity: {}", self.query); } - let g = match sql_raw(&self.golden_db, &self.query, Some(&self.server)) { + let g = match sql_raw_with_timeout(&self.golden_db, &self.query, Some(&self.server), self.timeout) { Ok(s) => s, Err(e) => { return ScoreDetails { @@ -484,7 +892,7 @@ impl Scorer for ReducerDataParityScorer { } } }; - let l = match sql_raw(&self.llm_db, &self.query, Some(&self.server)) { + let l = match sql_raw_with_timeout(&self.llm_db, &self.query, Some(&self.server), self.timeout) { Ok(s) => s, Err(e) => { return ScoreDetails { @@ -532,12 +940,77 @@ pub struct SqlCountOnlyScorer { pub id_str: &'static str, } +pub struct SqlDistinctRowsScorer { + pub server: String, + pub db: String, + pub sql: String, + pub expected: usize, + pub timeout: Duration, + pub id_str: &'static str, +} + +pub struct SqlOutputExcludesScorer { + pub server: String, + pub db: String, + pub sql: String, + pub excluded: Vec, + pub id_str: &'static str, +} + +pub struct EventuallySqlCountScorer { + pub server: String, + pub db: String, + pub sql: String, + pub expected: i64, + pub timeout: Duration, + pub id_str: &'static str, +} + +impl Scorer for EventuallySqlCountScorer { + fn id(&self) -> &'static str { + self.id_str + } + + fn score(&self, _llm_output: &str) -> ScoreDetails { + let started = Instant::now(); + loop { + let remaining = self.timeout.saturating_sub(started.elapsed()); + if remaining.is_zero() { + return ScoreDetails { + pass: false, + partial: 0.0, + notes: json!({ "sql": self.sql, "expected": self.expected, "last": { "error": "timeout" } }), + }; + } + let last = match sql_count_with_timeout(&self.db, &self.sql, Some(&self.server), remaining) { + Ok(actual) if actual == self.expected => { + return ScoreDetails { + pass: true, + partial: 1.0, + notes: json!({ "sql": self.sql, "expected": self.expected, "actual": actual }), + }; + } + Ok(actual) => json!({ "actual": actual }), + Err(error) => json!({ "error": error }), + }; + if started.elapsed() >= self.timeout { + return ScoreDetails { + pass: false, + partial: 0.0, + notes: json!({ "sql": self.sql, "expected": self.expected, "last": last }), + }; + } + thread::sleep(Duration::from_millis(50)); + } + } +} + impl Scorer for SqlCountOnlyScorer { fn id(&self) -> &'static str { self.id_str } fn score(&self, _llm_output: &str) -> ScoreDetails { - match sql_count(&self.db, &self.sql, Some(&self.server)) { + match sql_count_with_timeout(&self.db, &self.sql, Some(&self.server), self.timeout) { Ok(n) => { let pass = n == self.expected; ScoreDetails { @@ -555,6 +1028,70 @@ impl Scorer for SqlCountOnlyScorer { } } +impl Scorer for SqlDistinctRowsScorer { + fn id(&self) -> &'static str { + self.id_str + } + + fn score(&self, _llm_output: &str) -> ScoreDetails { + let output = match sql_json_with_timeout(&self.db, &self.sql, Some(&self.server), self.timeout) { + Ok(output) => output, + Err(error) => { + return ScoreDetails { + pass: false, + partial: 0.0, + notes: json!({ "phase": "sql", "error": error }), + } + } + }; + match distinct_sql_row_count(&output) { + Ok(actual) => { + let pass = actual == self.expected; + ScoreDetails { + pass, + partial: if pass { 1.0 } else { 0.0 }, + notes: json!({ "sql": self.sql, "expected": self.expected, "actual": actual }), + } + } + Err(error) => ScoreDetails { + pass: false, + partial: 0.0, + notes: json!({ "phase": "parse_sql", "error": error, "output": output }), + }, + } + } +} + +impl Scorer for SqlOutputExcludesScorer { + fn id(&self) -> &'static str { + self.id_str + } + + fn score(&self, _llm_output: &str) -> ScoreDetails { + match sql_raw(&self.db, &self.sql, Some(&self.server)) { + Ok(output) => { + let found = self + .excluded + .iter() + .filter(|value| output.contains(value.as_str())) + .cloned() + .collect::>(); + let pass = found.is_empty(); + ScoreDetails { + pass, + partial: if pass { 1.0 } else { 0.0 }, + notes: json!({ "sql": self.sql, "excluded": self.excluded, "found": found, "actual": output }), + } + } + Err(error) => ScoreDetails { + pass: false, + partial: 0.0, + notes: json!({ "phase": "sql", "error": error }), + }, + } + } +} + pub struct SqlExecBothScorer { pub server: String, pub golden_db: String, @@ -576,14 +1113,14 @@ impl Scorer for SqlExecBothScorer { self.sql, self.golden_db, self.llm_db, self.server ); } - if let Err(e) = sql_exec(&self.golden_db, &self.sql, Some(&self.server)) { + if let Err(e) = sql_exec_with_timeout(&self.golden_db, &self.sql, Some(&self.server), self.timeout) { return ScoreDetails { pass: false, partial: 0.0, notes: json!({ "phase":"sql_golden", "error": e, "sql": self.sql }), }; } - if let Err(e) = sql_exec(&self.llm_db, &self.sql, Some(&self.server)) { + if let Err(e) = sql_exec_with_timeout(&self.llm_db, &self.sql, Some(&self.server), self.timeout) { return ScoreDetails { pass: false, partial: 0.0, @@ -607,9 +1144,190 @@ pub struct ReducerCallBothScorer { pub llm_db: String, pub reducer: String, pub args: Vec, + pub attempts: usize, + pub id_str: &'static str, +} + +pub struct CallOutputParityScorer { + pub server: String, + pub golden_db: String, + pub llm_db: String, + pub function: String, + pub args: Vec, + pub collapse_ws: bool, + pub attempts: usize, pub id_str: &'static str, } +fn call_with_retries( + db: &str, + function: &str, + args: &[Value], + server: &str, + attempts: usize, +) -> Result { + let attempts = attempts.max(1); + let mut last_error = None; + for attempt in 0..attempts { + match call_reducer_json_out(db, function, args, Some(server)) { + Ok(output) => return Ok(output), + Err(error) => last_error = Some(error), + } + if attempt + 1 < attempts { + thread::sleep(Duration::from_millis(250)); + } + } + Err(last_error.expect("at least one call attempt must run")) +} + +pub struct HttpRouteCase { + pub method: String, + pub path: String, + pub body: Option, +} + +pub struct HttpRouteParityScorer { + pub server: String, + pub golden_db: String, + pub llm_db: String, + pub cases: Vec, + pub compare_content_type: bool, + pub timeout: Duration, + pub id_str: &'static str, +} + +fn call_http_route( + server: &str, + db: &str, + case: &HttpRouteCase, + timeout: Duration, +) -> Result<(u16, String, String), String> { + let server = server.trim_end_matches('/').to_string(); + let db = db.to_string(); + let method = case.method.clone(); + let path = case.path.clone(); + let body = case.body.clone(); + std::thread::spawn(move || { + let runtime = tokio::runtime::Runtime::new().map_err(|error| error.to_string())?; + runtime.block_on(async move { + let method = reqwest::Method::from_bytes(method.as_bytes()).map_err(|error| error.to_string())?; + let url = format!("{server}/v1/database/{db}/route{path}"); + let client = reqwest::Client::builder() + .timeout(timeout) + .build() + .map_err(|error| error.to_string())?; + let mut request = client.request(method, url); + if let Some(body) = body { + request = request.header("content-type", "text/plain").body(body); + } + let response = request.send().await.map_err(|error| error.to_string())?; + let status = response.status().as_u16(); + let content_type = response + .headers() + .get("content-type") + .and_then(|value| value.to_str().ok()) + .unwrap_or("") + .to_string(); + let body = response.text().await.map_err(|error| error.to_string())?; + Ok((status, content_type, body)) + }) + }) + .join() + .map_err(|_| "HTTP route worker panicked".to_string())? +} + +fn http_route_results_equal( + golden_results: &[(u16, String, String)], + llm_results: &[(u16, String, String)], + compare_content_type: bool, +) -> bool { + golden_results.len() == llm_results.len() + && golden_results + .iter() + .zip(llm_results) + .all(|(golden, llm)| golden.0 == llm.0 && golden.2 == llm.2 && (!compare_content_type || golden.1 == llm.1)) +} + +impl Scorer for HttpRouteParityScorer { + fn id(&self) -> &'static str { + self.id_str + } + + fn score(&self, _llm_output: &str) -> ScoreDetails { + let mut golden_results = Vec::new(); + let mut llm_results = Vec::new(); + for case in &self.cases { + match call_http_route(&self.server, &self.golden_db, case, self.timeout) { + Ok(result) => golden_results.push(result), + Err(error) => { + return ScoreDetails { + pass: false, + partial: 0.0, + notes: json!({ "phase": "http_golden", "error": error }), + } + } + } + match call_http_route(&self.server, &self.llm_db, case, self.timeout) { + Ok(result) => llm_results.push(result), + Err(error) => { + return ScoreDetails { + pass: false, + partial: 0.0, + notes: json!({ "phase": "http_llm", "error": error }), + } + } + } + } + let pass = http_route_results_equal(&golden_results, &llm_results, self.compare_content_type); + ScoreDetails { + pass, + partial: if pass { 1.0 } else { 0.0 }, + notes: json!({ + "golden": golden_results, + "llm": llm_results, + "compared_content_type": self.compare_content_type, + }), + } + } +} + +impl Scorer for CallOutputParityScorer { + fn id(&self) -> &'static str { + self.id_str + } + + fn score(&self, _llm_output: &str) -> ScoreDetails { + let golden = match call_with_retries(&self.golden_db, &self.function, &self.args, &self.server, self.attempts) { + Ok(output) => output, + Err(error) => { + return ScoreDetails { + pass: false, + partial: 0.0, + notes: json!({ "phase": "call_golden", "function": self.function, "error": error }), + } + } + }; + let llm = match call_with_retries(&self.llm_db, &self.function, &self.args, &self.server, self.attempts) { + Ok(output) => output, + Err(error) => { + return ScoreDetails { + pass: false, + partial: 0.0, + notes: json!({ "phase": "call_llm", "function": self.function, "error": error }), + } + } + }; + let golden = normalize(&golden, self.collapse_ws); + let llm = normalize(&llm, self.collapse_ws); + let pass = golden == llm; + ScoreDetails { + pass, + partial: if pass { 1.0 } else { 0.0 }, + notes: json!({ "function": self.function, "golden": golden, "llm": llm }), + } + } +} + impl Scorer for ReducerCallBothScorer { fn id(&self) -> &'static str { self.id_str @@ -622,14 +1340,14 @@ impl Scorer for ReducerCallBothScorer { self.reducer, self.args, self.golden_db, self.llm_db, self.server ); } - if let Err(e) = call_reducer_json_out(&self.golden_db, &self.reducer, &self.args, Some(&self.server)) { + if let Err(e) = call_with_retries(&self.golden_db, &self.reducer, &self.args, &self.server, self.attempts) { return ScoreDetails { pass: false, partial: 0.0, notes: json!({ "phase":"call_reducer_golden", "error": e, "reducer": self.reducer }), }; } - if let Err(e) = call_reducer_json_out(&self.llm_db, &self.reducer, &self.args, Some(&self.server)) { + if let Err(e) = call_with_retries(&self.llm_db, &self.reducer, &self.args, &self.server, self.attempts) { return ScoreDetails { pass: false, partial: 0.0, @@ -646,3 +1364,338 @@ impl Scorer for ReducerCallBothScorer { } } } + +#[cfg(test)] +mod tests { + use super::*; + use spacetimedb_lib::db::raw_def::v10::{MethodOrAny, RawModuleDefV10Builder}; + use spacetimedb_lib::db::raw_def::v9::Lifecycle; + use spacetimedb_lib::sats::serde::SerdeWrapper; + use spacetimedb_lib::sats::{AlgebraicType, ProductType}; + + #[test] + fn counts_distinct_rows_in_json_sql_output() { + let output = json!([{ + "rows": [[1], [2], [1]], + "schema": { "elements": [] }, + "total_duration_micros": 1 + }]); + + assert_eq!(distinct_sql_row_count(&output).unwrap(), 2); + } + + fn current_schema(include_owner_index: bool) -> Value { + let mut indexes = vec![json!({ "algorithm": { "BTree": [0] } })]; + if include_owner_index { + indexes.push(json!({ "algorithm": { "BTree": [1] } })); + } + json!({ + "typespace": { + "types": [{ + "Product": { + "elements": [ + { "name": { "some": "id" }, "algebraic_type": { "U64": [] } }, + { "name": { "some": "owner_id" }, "algebraic_type": { "U64": [] } } + ] + } + }] + }, + "tables": [{ + "name": "child_item", + "product_type_ref": 0, + "primary_key": [0], + "indexes": indexes, + "constraints": [{ "data": { "Unique": { "columns": [0] } } }], + "sequences": [{ "column": 0, "increment": 1 }], + "schedule": { "none": [] }, + "table_type": { "User": [] }, + "table_access": { "Public": [] } + }], + "reducers": [] + }) + } + + #[test] + fn current_schema_extracts_columns_and_table_properties() { + let schema = extract_schema(¤t_schema(true)); + let child_item = &schema.tables["child_item"]; + + assert_eq!(child_item["id"], r#"{"U64":[]}"#); + assert_eq!(child_item["owner_id"], r#"{"U64":[]}"#); + assert_eq!(child_item["@primary_key"], "id"); + assert_eq!(child_item["@indexes"], "BTree(id);BTree(owner_id)"); + assert_eq!(child_item["@constraints"], "Unique(id)"); + assert_eq!(child_item["@sequences"], "id:increment=1:start=:min=:max="); + assert_eq!(child_item["@table_access"], r#"{"Public":[]}"#); + assert!(schema.reducers.is_empty()); + assert!(schema.row_level_security.is_empty()); + } + + #[test] + fn missing_index_produces_a_schema_diff() { + let golden = extract_schema(¤t_schema(true)); + let candidate = extract_schema(¤t_schema(false)); + + assert!(!diff_maps(&golden.tables, &candidate.tables).is_null()); + } + + #[test] + fn missing_default_value_produces_a_schema_diff() { + let mut golden = current_schema(true); + golden["tables"][0]["default_values"] = json!([{ + "col_id": 1, + "value": [1] + }]); + + let golden = extract_schema(&golden); + let candidate = extract_schema(¤t_schema(true)); + + assert!(!diff_maps(&golden.tables, &candidate.tables).is_null()); + } + + #[test] + fn v10_schema_compares_explicit_canonical_table_names() { + let schema = |source_name: &str| { + json!({ + "sections": [ + { "Typespace": { "types": [{ "Product": { "elements": [] } }] } }, + { "Tables": [{ + "source_name": source_name, + "product_type_ref": 0, + "primary_key": [], + "indexes": [], + "constraints": [], + "sequences": [], + "table_type": { "User": [] }, + "table_access": { "Public": [] }, + "default_values": [], + "is_event": false + }] }, + { "ExplicitNames": { "entries": [{ + "Table": { + "source_name": source_name, + "canonical_name": "child_item" + } + }] } } + ] + }) + }; + + assert_eq!( + extract_schema(&schema("childItem")), + extract_schema(&schema("child_item")) + ); + } + + #[test] + fn row_level_security_produces_a_schema_diff() { + let mut golden = current_schema(true); + golden["row_level_security"] = json!([{ "sql": "SELECT * FROM users WHERE identity = :sender" }]); + let candidate = current_schema(true); + let golden = extract_schema(&golden); + let candidate = extract_schema(&candidate); + + assert!(!diff_sets(&golden.row_level_security, &candidate.row_level_security).is_null()); + } + + #[test] + fn current_schema_extracts_reducer_parameter_types() { + let mut schema = current_schema(true); + schema["reducers"] = json!([{ + "name": "set_owner", + "params": { + "elements": [ + { "name": { "some": "id" }, "algebraic_type": { "U64": [] } }, + { "name": { "some": "owner" }, "algebraic_type": { "String": [] } } + ] + } + }]); + + let reducers = extract_schema(&schema).reducers; + + assert_eq!( + reducers, + BTreeSet::from([r#"set_owner({"U64":[]},{"String":[]})"#.to_owned()]) + ); + } + + #[test] + fn reducer_parameter_type_produces_a_schema_diff() { + let mut golden = current_schema(true); + golden["reducers"] = json!([{ + "name": "set_owner", + "params": { "elements": [ + { "name": { "some": "owner" }, "algebraic_type": { "String": [] } } + ] } + }]); + let mut candidate = current_schema(true); + candidate["reducers"] = json!([{ + "name": "set_owner", + "params": { "elements": [ + { "name": { "some": "owner" }, "algebraic_type": { "U64": [] } } + ] } + }]); + let golden_reducers = extract_schema(&golden).reducers; + let candidate_reducers = extract_schema(&candidate).reducers; + + assert!(!diff_sets(&golden_reducers, &candidate_reducers).is_null()); + } + + #[test] + fn reducer_parameter_names_do_not_produce_a_schema_diff() { + let mut golden = current_schema(true); + golden["reducers"] = json!([{ + "name": "send_reminder", + "params": { "elements": [ + { "name": { "some": "timer" }, "algebraic_type": { "U64": [] } } + ] } + }]); + let mut candidate = current_schema(true); + candidate["reducers"] = json!([{ + "name": "send_reminder", + "params": { "elements": [ + { "name": { "some": "reminder" }, "algebraic_type": { "U64": [] } } + ] } + }]); + let golden_reducers = extract_schema(&golden).reducers; + let candidate_reducers = extract_schema(&candidate).reducers; + + assert!(diff_sets(&golden_reducers, &candidate_reducers).is_null()); + } + + #[test] + fn legacy_reducer_args_remain_supported() { + let mut schema = current_schema(true); + schema["reducers"] = json!([{ + "name": "set_owner", + "args": [{ "name": "owner", "type": "String" }] + }]); + + let reducers = extract_schema(&schema).reducers; + + assert_eq!(reducers, BTreeSet::from(["set_owner(String)".to_owned()])); + } + + #[test] + fn http_route_parity_ignores_unspecified_content_type() { + let golden = vec![(201, String::new(), "created".to_string())]; + let candidate = vec![(201, "text/plain".to_string(), "created".to_string())]; + + assert!(http_route_results_equal(&golden, &candidate, false)); + assert!(!http_route_results_equal(&golden, &candidate, true)); + } + + #[test] + fn http_route_schema_ignores_handler_names() { + let schema = |handler: &str| { + json!({ + "sections": [ + { "HttpHandlers": [{ "source_name": handler }] }, + { "HttpRoutes": [{ + "handler_function": handler, + "method": { "Method": { "Get": [] } }, + "path": "/items" + }] } + ] + }) + }; + + assert_eq!( + extract_schema(&schema("listItems")), + extract_schema(&schema("get_items")) + ); + } + + #[test] + fn v10_schema_extracts_lifecycle_procedures_views_and_http_exports() { + let schema = json!({ + "sections": [ + { "Typespace": { "types": [] } }, + { "Tables": [] }, + { "Reducers": [{ + "source_name": "connected", + "params": { "elements": [] }, + "visibility": { "Private": [] }, + "ok_return_type": { "Unit": [] }, + "err_return_type": { "String": [] } + }] }, + { "Procedures": [{ + "source_name": "fetch", + "params": { "elements": [] }, + "return_type": { "String": [] }, + "visibility": { "ClientCallable": [] } + }] }, + { "Views": [{ + "source_name": "profile", + "params": { "elements": [] }, + "return_type": { "String": [] }, + "is_public": true, + "is_anonymous": false + }] }, + { "LifeCycleReducers": [{ + "lifecycle_spec": { "OnConnect": [] }, + "function_name": "connected" + }] }, + { "HttpHandlers": [{ "source_name": "webhook" }] }, + { "HttpRoutes": [{ + "handler_function": "webhook", + "method": { "Any": [] }, + "path": "/hook" + }] }, + { "ViewPrimaryKeys": [{ "view_source_name": "profile", "columns": ["id"] }] } + ] + }); + + let extracted = extract_schema(&schema); + + assert_eq!(extracted.reducers.len(), 1); + assert!(extracted + .exports + .iter() + .any(|value| value.starts_with("procedure:fetch"))); + assert!(extracted.exports.iter().any(|value| value.starts_with("view:profile"))); + assert!(extracted.exports.iter().any(|value| value.starts_with("lifecycle:"))); + assert!(extracted.exports.iter().any(|value| value.starts_with("http_route:"))); + assert!(extracted + .exports + .iter() + .any(|value| value.starts_with("view_primary_key:"))); + } + + #[test] + fn extracts_exports_from_actual_v10_serialization() { + let mut module = RawModuleDefV10Builder::new(); + module.add_lifecycle_reducer(Lifecycle::OnConnect, "connected", ProductType::unit()); + module.add_procedure("fetch", ProductType::unit(), AlgebraicType::unit()); + module.add_view("profile", 0, true, false, ProductType::unit(), AlgebraicType::unit()); + module.add_view_primary_key("profile", ["id"]); + module.add_http_handler("webhook"); + module.add_http_route("webhook", MethodOrAny::Any, "/hook"); + let module = module.finish(); + let json = serde_json::to_value(SerdeWrapper::from_ref(&module)).unwrap(); + + let extracted = extract_schema(&json); + + assert_eq!(extracted.reducers.len(), 1, "serialized schema was {json}"); + assert_eq!(extracted.exports.len(), 5, "serialized schema was {json}"); + assert!( + extracted + .exports + .iter() + .any(|value| value.starts_with("procedure:fetch")), + "serialized schema was {json}" + ); + assert!( + extracted.exports.iter().any(|value| value.starts_with("view:profile")), + "serialized schema was {json}" + ); + + let mut missing_assignments = json.clone(); + missing_assignments["sections"] + .as_array_mut() + .unwrap() + .retain(|section| section.get("LifeCycleReducers").is_none() && section.get("ViewPrimaryKeys").is_none()); + let missing_assignments = extract_schema(&missing_assignments); + assert!(!diff_sets(&extracted.exports, &missing_assignments.exports).is_null()); + } +} diff --git a/tools/xtask-llm-benchmark/src/eval/utils.rs b/tools/xtask-llm-benchmark/src/eval/utils.rs index 1fd05c786e4..c35b1fb2e99 100644 --- a/tools/xtask-llm-benchmark/src/eval/utils.rs +++ b/tools/xtask-llm-benchmark/src/eval/utils.rs @@ -1,4 +1,9 @@ -use std::process::Command; +use std::env; +use std::io::{self, Read}; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::thread; +use std::time::{Duration, Instant}; pub fn derive_cat_task_from_file(src: &str) -> (String, String) { let p = std::path::Path::new(src); @@ -18,22 +23,135 @@ pub fn derive_cat_task_from_file(src: &str) -> (String, String) { (cat, task) } +pub(crate) fn spacetime_command() -> Command { + if let Some(executable) = env::var_os("LLM_BENCH_SPACETIME_BIN") { + return Command::new(executable); + } + + let workspace = Path::new(env!("CARGO_MANIFEST_DIR")) + .ancestors() + .nth(2) + .expect("xtask-llm-benchmark is under tools/xtask-llm-benchmark"); + let target = env::var_os("CARGO_TARGET_DIR") + .map(PathBuf::from) + .unwrap_or_else(|| workspace.join("target")); + let target = if target.is_absolute() { + target + } else { + workspace.join(target) + }; + let executable = if cfg!(windows) { + "spacetimedb-cli.exe" + } else { + "spacetimedb-cli" + }; + for profile in ["release", "debug"] { + let candidate = target.join(profile).join(executable); + if candidate.is_file() { + return Command::new(candidate); + } + } + + Command::new("spacetime") +} + pub fn sql_exec(db: &str, query: &str, host: Option<&str>) -> Result<(), String> { - let mut cmd = Command::new("spacetime"); + sql_exec_with_timeout(db, query, host, Duration::from_secs(30)) +} + +pub(crate) fn sql_exec_with_timeout( + db: &str, + query: &str, + host: Option<&str>, + timeout: Duration, +) -> Result<(), String> { + let mut cmd = spacetime_command(); cmd.arg("sql").arg(db).arg(query); if let Some(h) = host { cmd.arg("--server").arg(h); } - let out = cmd.output().map_err(|e| format!("spawn spacetime sql failed: {e}"))?; - if !out.status.success() { - return Err(format!( - "spacetime sql failed:\n{}", - String::from_utf8_lossy(&out.stderr) - )); + let (code, _, stderr) = run_with_timeout(cmd, Path::new("."), timeout) + .map_err(|e| format!("spacetime sql failed or timed out: {e}"))?; + if code != 0 { + return Err(format!("spacetime sql failed:\n{}", String::from_utf8_lossy(&stderr))); } Ok(()) } +pub(crate) fn run_with_timeout(mut cmd: Command, cwd: &Path, timeout: Duration) -> io::Result<(i32, Vec, Vec)> { + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + cmd.process_group(0); + } + + let mut child = cmd + .current_dir(cwd) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + let mut stdout = child.stdout.take().expect("stdout was configured as piped"); + let mut stderr = child.stderr.take().expect("stderr was configured as piped"); + let stdout_reader = thread::spawn(move || { + let mut output = Vec::new(); + stdout.read_to_end(&mut output)?; + Ok::<_, io::Error>(output) + }); + let stderr_reader = thread::spawn(move || { + let mut output = Vec::new(); + stderr.read_to_end(&mut output)?; + Ok::<_, io::Error>(output) + }); + let start = Instant::now(); + let status = loop { + if let Some(status) = child.try_wait()? { + break status; + } + if start.elapsed() >= timeout { + let termination_error = kill_process_tree(&mut child).err(); + let _ = child.wait(); + let _ = stdout_reader.join(); + let _ = stderr_reader.join(); + let message = termination_error + .map(|error| format!("process timeout; failed to terminate process tree: {error}")) + .unwrap_or_else(|| "process timeout".to_string()); + return Err(io::Error::new(io::ErrorKind::TimedOut, message)); + } + thread::sleep(Duration::from_millis(30)); + }; + let stdout = stdout_reader + .join() + .map_err(|_| io::Error::other("stdout reader thread panicked"))??; + let stderr = stderr_reader + .join() + .map_err(|_| io::Error::other("stderr reader thread panicked"))??; + Ok((status.code().unwrap_or(-1), stdout, stderr)) +} + +fn kill_process_tree(child: &mut Child) -> io::Result<()> { + #[cfg(windows)] + let killed = Command::new("taskkill") + .args(["/F", "/T", "/PID", &child.id().to_string()]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .is_ok_and(|status| status.success()); + + #[cfg(unix)] + let killed = Command::new("kill") + .args(["-KILL", &format!("-{}", child.id())]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .is_ok_and(|status| status.success()); + + if killed { + Ok(()) + } else { + child.kill() + } +} + pub fn normalize(s: &str, collapse_ws: bool) -> String { let t = s.trim(); if collapse_ws { @@ -42,3 +160,76 @@ pub fn normalize(s: &str, collapse_ws: bool) -> String { t.to_string() } } + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Mutex; + + static PROCESS_TEST_LOCK: Mutex<()> = Mutex::new(()); + + #[test] + fn process_timeout_kills_a_long_running_command() { + let _guard = PROCESS_TEST_LOCK + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + + #[cfg(windows)] + let command = { + let mut command = Command::new("powershell"); + command.args(["-NoProfile", "-Command", "Start-Sleep -Seconds 5"]); + command + }; + #[cfg(not(windows))] + let command = { + let mut command = Command::new("sh"); + command.args(["-c", "sleep 5"]); + command + }; + + let started = Instant::now(); + let error = run_with_timeout(command, Path::new("."), Duration::from_millis(100)).unwrap_err(); + + assert_eq!(error.kind(), io::ErrorKind::TimedOut); + assert!(started.elapsed() < Duration::from_secs(3)); + } + + #[test] + fn process_output_is_drained_while_the_command_runs() { + let _guard = PROCESS_TEST_LOCK + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + + #[cfg(windows)] + let command = { + let mut command = Command::new("powershell"); + command.args([ + "-NoProfile", + "-Command", + "$chunk = 'x' * 4096; 1..16 | ForEach-Object { [Console]::Out.Write($chunk); [Console]::Error.Write($chunk) }", + ]); + command + }; + #[cfg(not(windows))] + let command = { + let mut command = Command::new("sh"); + command.args([ + "-c", + "dd if=/dev/zero bs=4096 count=256 2>/dev/null; dd if=/dev/zero bs=4096 count=256 1>&2 2>/dev/null", + ]); + command + }; + + #[cfg(windows)] + let expected_len = 4096 * 16; + #[cfg(not(windows))] + let expected_len = 4096 * 256; + + let (code, stdout, stderr) = run_with_timeout(command, Path::new("."), Duration::from_secs(60)) + .expect("large output should not deadlock"); + + assert_eq!(code, 0); + assert_eq!(stdout.len(), expected_len); + assert_eq!(stderr.len(), expected_len); + } +} diff --git a/tools/xtask-llm-benchmark/src/generated/registry.rs b/tools/xtask-llm-benchmark/src/generated/registry.rs index ef74e3aac20..b1c15e5e32b 100644 --- a/tools/xtask-llm-benchmark/src/generated/registry.rs +++ b/tools/xtask-llm-benchmark/src/generated/registry.rs @@ -50,6 +50,18 @@ mod auth_t_046_shared_document { include!("../benchmarks/auth/t_046_shared_document/spec.rs"); } +#[allow(dead_code)] +#[allow(clippy::all)] +mod auth_t_068_secure_projection { + include!("../benchmarks/auth/t_068_secure_projection/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod auth_t_083_row_level_security { + include!("../benchmarks/auth/t_083_row_level_security/spec.rs"); +} + #[allow(dead_code)] #[allow(clippy::all)] mod basics_t_000_empty_reducers { @@ -176,6 +188,90 @@ mod data_modeling_t_031_unique_constraint { include!("../benchmarks/data_modeling/t_031_unique_constraint/spec.rs"); } +#[allow(dead_code)] +#[allow(clippy::all)] +mod lifecycle_t_069_scheduled_materialization { + include!("../benchmarks/lifecycle/t_069_scheduled_materialization/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod lifecycle_t_070_connection_scoped_presence { + include!("../benchmarks/lifecycle/t_070_connection_scoped_presence/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod lifecycle_t_071_scheduled_private { + include!("../benchmarks/lifecycle/t_071_scheduled_private/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod migrations_t_080_automatic_migration { + include!("../benchmarks/migrations/t_080_automatic_migration/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod migrations_t_081_incremental_migration { + include!("../benchmarks/migrations/t_081_incremental_migration/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod migrations_t_082_hot_swap_compatibility { + include!("../benchmarks/migrations/t_082_hot_swap_compatibility/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod procedures_t_072_procedure_return { + include!("../benchmarks/procedures/t_072_procedure_return/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod procedures_t_073_http_fetch { + include!("../benchmarks/procedures/t_073_http_fetch/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod procedures_t_074_fetch_and_store { + include!("../benchmarks/procedures/t_074_fetch_and_store/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod procedures_t_075_scheduled_procedure { + include!("../benchmarks/procedures/t_075_scheduled_procedure/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod procedures_t_076_http_handler { + include!("../benchmarks/procedures/t_076_http_handler/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod procedures_t_077_http_router { + include!("../benchmarks/procedures/t_077_http_router/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod procedures_t_078_idempotent_webhook { + include!("../benchmarks/procedures/t_078_idempotent_webhook/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod procedures_t_079_external_upload_flow { + include!("../benchmarks/procedures/t_079_external_upload_flow/spec.rs"); +} + #[allow(dead_code)] #[allow(clippy::all)] mod queries_t_022_view_basic { @@ -224,6 +320,42 @@ mod queries_t_037_multi_column_filter { include!("../benchmarks/queries/t_037_multi_column_filter/spec.rs"); } +#[allow(dead_code)] +#[allow(clippy::all)] +mod reducers_t_055_atomic_idempotent_transfer { + include!("../benchmarks/reducers/t_055_atomic_idempotent_transfer/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod reducers_t_056_nested_update { + include!("../benchmarks/reducers/t_056_nested_update/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod reducers_t_057_nested_cascade_delete { + include!("../benchmarks/reducers/t_057_nested_cascade_delete/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod reducers_t_058_batched_delete { + include!("../benchmarks/reducers/t_058_batched_delete/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod reducers_t_059_deterministic_context { + include!("../benchmarks/reducers/t_059_deterministic_context/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod reducers_t_060_reducer_result_table { + include!("../benchmarks/reducers/t_060_reducer_result_table/spec.rs"); +} + #[allow(dead_code)] #[allow(clippy::all)] mod schema_t_012_spacetime_product_type { @@ -284,6 +416,96 @@ mod schema_t_021_multi_column_index { include!("../benchmarks/schema/t_021_multi_column_index/spec.rs"); } +#[allow(dead_code)] +#[allow(clippy::all)] +mod tables_t_047_normalized_collection { + include!("../benchmarks/tables/t_047_normalized_collection/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod tables_t_048_heartbeat_isolation { + include!("../benchmarks/tables/t_048_heartbeat_isolation/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod tables_t_049_binary_storage { + include!("../benchmarks/tables/t_049_binary_storage/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod tables_t_050_normalized_schema { + include!("../benchmarks/tables/t_050_normalized_schema/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod tables_t_051_denormalized_index { + include!("../benchmarks/tables/t_051_denormalized_index/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod tables_t_052_autoinc_reference { + include!("../benchmarks/tables/t_052_autoinc_reference/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod tables_t_053_default_values { + include!("../benchmarks/tables/t_053_default_values/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod tables_t_054_special_types { + include!("../benchmarks/tables/t_054_special_types/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod views_t_061_three_table_join { + include!("../benchmarks/views/t_061_three_table_join/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod views_t_062_semijoin_intersection { + include!("../benchmarks/views/t_062_semijoin_intersection/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod views_t_063_timestamp_window { + include!("../benchmarks/views/t_063_timestamp_window/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod views_t_064_index_and_filter { + include!("../benchmarks/views/t_064_index_and_filter/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod views_t_065_materialized_aggregate { + include!("../benchmarks/views/t_065_materialized_aggregate/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod views_t_066_query_builder_view { + include!("../benchmarks/views/t_066_query_builder_view/spec.rs"); +} + +#[allow(dead_code)] +#[allow(clippy::all)] +mod views_t_067_view_primary_key { + include!("../benchmarks/views/t_067_view_primary_key/spec.rs"); +} + pub fn resolve_by_path(task_root: &Path) -> Result BenchmarkSpec> { let task = task_root .file_name() @@ -303,6 +525,8 @@ pub fn resolve_by_path(task_root: &Path) -> Result BenchmarkSpec> { ("auth", "t_044_ban_list") => auth_t_044_ban_list::spec, ("auth", "t_045_rate_limit") => auth_t_045_rate_limit::spec, ("auth", "t_046_shared_document") => auth_t_046_shared_document::spec, + ("auth", "t_068_secure_projection") => auth_t_068_secure_projection::spec, + ("auth", "t_083_row_level_security") => auth_t_083_row_level_security::spec, ("basics", "t_000_empty_reducers") => basics_t_000_empty_reducers::spec, ("basics", "t_001_basic_tables") => basics_t_001_basic_tables::spec, ("basics", "t_002_scheduled_table") => basics_t_002_scheduled_table::spec, @@ -324,6 +548,20 @@ pub fn resolve_by_path(task_root: &Path) -> Result BenchmarkSpec> { ("data_modeling", "t_029_filter_and_aggregate") => data_modeling_t_029_filter_and_aggregate::spec, ("data_modeling", "t_030_two_table_join") => data_modeling_t_030_two_table_join::spec, ("data_modeling", "t_031_unique_constraint") => data_modeling_t_031_unique_constraint::spec, + ("lifecycle", "t_069_scheduled_materialization") => lifecycle_t_069_scheduled_materialization::spec, + ("lifecycle", "t_070_connection_scoped_presence") => lifecycle_t_070_connection_scoped_presence::spec, + ("lifecycle", "t_071_scheduled_private") => lifecycle_t_071_scheduled_private::spec, + ("migrations", "t_080_automatic_migration") => migrations_t_080_automatic_migration::spec, + ("migrations", "t_081_incremental_migration") => migrations_t_081_incremental_migration::spec, + ("migrations", "t_082_hot_swap_compatibility") => migrations_t_082_hot_swap_compatibility::spec, + ("procedures", "t_072_procedure_return") => procedures_t_072_procedure_return::spec, + ("procedures", "t_073_http_fetch") => procedures_t_073_http_fetch::spec, + ("procedures", "t_074_fetch_and_store") => procedures_t_074_fetch_and_store::spec, + ("procedures", "t_075_scheduled_procedure") => procedures_t_075_scheduled_procedure::spec, + ("procedures", "t_076_http_handler") => procedures_t_076_http_handler::spec, + ("procedures", "t_077_http_router") => procedures_t_077_http_router::spec, + ("procedures", "t_078_idempotent_webhook") => procedures_t_078_idempotent_webhook::spec, + ("procedures", "t_079_external_upload_flow") => procedures_t_079_external_upload_flow::spec, ("queries", "t_022_view_basic") => queries_t_022_view_basic::spec, ("queries", "t_023_view_per_user") => queries_t_023_view_per_user::spec, ("queries", "t_032_range_query") => queries_t_032_range_query::spec, @@ -332,6 +570,12 @@ pub fn resolve_by_path(task_root: &Path) -> Result BenchmarkSpec> { ("queries", "t_035_select_distinct") => queries_t_035_select_distinct::spec, ("queries", "t_036_count_without_collect") => queries_t_036_count_without_collect::spec, ("queries", "t_037_multi_column_filter") => queries_t_037_multi_column_filter::spec, + ("reducers", "t_055_atomic_idempotent_transfer") => reducers_t_055_atomic_idempotent_transfer::spec, + ("reducers", "t_056_nested_update") => reducers_t_056_nested_update::spec, + ("reducers", "t_057_nested_cascade_delete") => reducers_t_057_nested_cascade_delete::spec, + ("reducers", "t_058_batched_delete") => reducers_t_058_batched_delete::spec, + ("reducers", "t_059_deterministic_context") => reducers_t_059_deterministic_context::spec, + ("reducers", "t_060_reducer_result_table") => reducers_t_060_reducer_result_table::spec, ("schema", "t_012_spacetime_product_type") => schema_t_012_spacetime_product_type::spec, ("schema", "t_013_spacetime_sum_type") => schema_t_013_spacetime_sum_type::spec, ("schema", "t_014_elementary_columns") => schema_t_014_elementary_columns::spec, @@ -342,6 +586,21 @@ pub fn resolve_by_path(task_root: &Path) -> Result BenchmarkSpec> { ("schema", "t_019_many_to_many") => schema_t_019_many_to_many::spec, ("schema", "t_020_ecs") => schema_t_020_ecs::spec, ("schema", "t_021_multi_column_index") => schema_t_021_multi_column_index::spec, + ("tables", "t_047_normalized_collection") => tables_t_047_normalized_collection::spec, + ("tables", "t_048_heartbeat_isolation") => tables_t_048_heartbeat_isolation::spec, + ("tables", "t_049_binary_storage") => tables_t_049_binary_storage::spec, + ("tables", "t_050_normalized_schema") => tables_t_050_normalized_schema::spec, + ("tables", "t_051_denormalized_index") => tables_t_051_denormalized_index::spec, + ("tables", "t_052_autoinc_reference") => tables_t_052_autoinc_reference::spec, + ("tables", "t_053_default_values") => tables_t_053_default_values::spec, + ("tables", "t_054_special_types") => tables_t_054_special_types::spec, + ("views", "t_061_three_table_join") => views_t_061_three_table_join::spec, + ("views", "t_062_semijoin_intersection") => views_t_062_semijoin_intersection::spec, + ("views", "t_063_timestamp_window") => views_t_063_timestamp_window::spec, + ("views", "t_064_index_and_filter") => views_t_064_index_and_filter::spec, + ("views", "t_065_materialized_aggregate") => views_t_065_materialized_aggregate::spec, + ("views", "t_066_query_builder_view") => views_t_066_query_builder_view::spec, + ("views", "t_067_view_primary_key") => views_t_067_view_primary_key::spec, _ => return Err(anyhow!("no spec registered for {}/{} (need spec.rs)", category, task)), }; diff --git a/tools/xtask-llm-benchmark/src/llm/clients/meta.rs b/tools/xtask-llm-benchmark/src/llm/clients/meta.rs index 4744361df73..05ea663019e 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/meta.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/meta.rs @@ -5,7 +5,7 @@ use super::http::HttpClient; use super::oa_compat::OACompatResp; use crate::llm::prompt::BuiltPrompt; use crate::llm::segmentation::{ - desired_output_tokens, deterministic_trim_prefix, meta_ctx_limit_tokens, non_context_reserve_tokens_env, Segment, + deterministic_trim_prefix, meta_ctx_limit_tokens, non_context_reserve_tokens_env, output_token_limit_env, Segment, }; use crate::llm::types::{LlmOutput, Vendor}; @@ -85,7 +85,7 @@ impl MetaLlamaClient { messages, temperature: 0.0, top_p: None, - max_tokens: Some(desired_output_tokens().max(1) as u32), + max_tokens: output_token_limit_env().map(|limit| limit.max(1) as u32), }; // Auth only; optional OpenRouter headers can live in HttpClient if desired diff --git a/tools/xtask-llm-benchmark/src/llm/clients/openrouter.rs b/tools/xtask-llm-benchmark/src/llm/clients/openrouter.rs index 8e8642ada0b..df4c062c1bb 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/openrouter.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/openrouter.rs @@ -6,7 +6,7 @@ use super::http::HttpClient; use super::oa_compat::OACompatResp; use crate::llm::prompt::BuiltPrompt; use crate::llm::segmentation::{ - desired_output_tokens, deterministic_trim_prefix, non_context_reserve_tokens_env, Segment, + deterministic_trim_prefix, non_context_reserve_tokens_env, output_token_limit_env, Segment, }; use crate::llm::types::{LlmOutput, Vendor}; @@ -218,13 +218,12 @@ impl OpenRouterClient { }); } - let max_tokens = desired_output_tokens().max(1) as u32; let req = Req { model, messages, temperature: 0.0, top_p: None, - max_tokens: Some(max_tokens), + max_tokens: output_token_limit_env().map(|limit| limit.max(1) as u32), }; let auth = HttpClient::bearer(&self.api_key); diff --git a/tools/xtask-llm-benchmark/src/llm/prompt.rs b/tools/xtask-llm-benchmark/src/llm/prompt.rs index 7a46d6826cd..6c47bef31c4 100644 --- a/tools/xtask-llm-benchmark/src/llm/prompt.rs +++ b/tools/xtask-llm-benchmark/src/llm/prompt.rs @@ -104,8 +104,14 @@ pub fn make_prompt_from_task(spec_file: &str, task_id: &str, lang: Lang) -> Resu let tasks_file = find_tasks_file(task_root, lang) .with_context(|| format!("missing tasks file for {} in {}", lang.as_str(), task_root.display()))?; - let instructions = + let mut instructions = std::fs::read_to_string(&tasks_file).with_context(|| format!("read {}", tasks_file.display()))?; + if let Some(setup_file) = find_setup_file(task_root, lang) { + let setup_source = + std::fs::read_to_string(&setup_file).with_context(|| format!("read {}", setup_file.display()))?; + instructions.push_str("\n\nEXISTING MODULE SOURCE TO UPDATE:\n"); + instructions.push_str(&setup_source); + } Ok(PromptBuilder { lang: lang.display_name().to_string(), @@ -132,8 +138,20 @@ fn find_tasks_file(task_root: &Path, lang: Lang) -> Option { } } +fn find_setup_file(task_root: &Path, lang: Lang) -> Option { + let file = match lang { + Lang::CSharp => "csharp.cs", + Lang::Rust => "rust.rs", + Lang::TypeScript => "typescript.ts", + }; + let path = task_root.join("setup").join(file); + path.exists().then_some(path) +} + #[cfg(test)] mod tests { + use crate::eval::Lang; + #[test] fn prompt_uses_workspace_version() { let pb = super::PromptBuilder { @@ -155,4 +173,17 @@ mod tests { assert!(parts.next().unwrap().parse::().is_ok(), "minor not numeric: {v}"); assert_eq!(parts.next(), None, "expected major.minor only: {v}"); } + + #[test] + fn update_task_prompt_includes_existing_module_source() { + let spec = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("src/benchmarks/tables/t_053_default_values/spec.rs"); + let prompt = super::make_prompt_from_task(spec.to_str().unwrap(), "t_053_default_values", Lang::Rust) + .expect("build migration prompt"); + + assert!(prompt.instructions.contains("EXISTING MODULE SOURCE TO UPDATE:")); + assert!(prompt.instructions.contains("pub struct Widget")); + assert!(prompt.instructions.contains("name: String")); + assert!(prompt.instructions.contains("pub fn touch")); + } } diff --git a/tools/xtask-llm-benchmark/src/llm/segmentation.rs b/tools/xtask-llm-benchmark/src/llm/segmentation.rs index 26bc481e52f..8d8a07ed060 100644 --- a/tools/xtask-llm-benchmark/src/llm/segmentation.rs +++ b/tools/xtask-llm-benchmark/src/llm/segmentation.rs @@ -193,12 +193,17 @@ pub fn xai_ctx_limit_tokens(model: &str) -> usize { 128_000 } -/// Desired output tokens (planning only). Env override: `LLM_DESIRED_OUTPUT_TOKENS` (usize). -pub fn desired_output_tokens() -> usize { +/// Explicit output-token limit for providers that accept an optional cap. +pub fn output_token_limit_env() -> Option { std::env::var("LLM_DESIRED_OUTPUT_TOKENS") .ok() .and_then(|s| s.parse::().ok()) - .unwrap_or(1500) +} + +/// Desired output tokens for context planning and providers that require a cap. +/// Env override: `LLM_DESIRED_OUTPUT_TOKENS` (usize). +pub fn desired_output_tokens() -> usize { + output_token_limit_env().unwrap_or(4096) } /// Static headroom from env, with sensible defaults and 500-token rounding. diff --git a/tools/xtask-llm-benchmark/src/templates/csharp/server/StdbModule.csproj b/tools/xtask-llm-benchmark/src/templates/csharp/server/StdbModule.csproj index f286932badd..6a14cc3a269 100644 --- a/tools/xtask-llm-benchmark/src/templates/csharp/server/StdbModule.csproj +++ b/tools/xtask-llm-benchmark/src/templates/csharp/server/StdbModule.csproj @@ -1,10 +1,11 @@ - net8.0 + net8.0;net10.0 wasi-wasm enable enable + $(NoWarn);STDB_UNSTABLE diff --git a/tools/xtask-llm-benchmark/src/templates/rust/server/Cargo.toml b/tools/xtask-llm-benchmark/src/templates/rust/server/Cargo.toml index d925f98168b..a0b4af5cf50 100644 --- a/tools/xtask-llm-benchmark/src/templates/rust/server/Cargo.toml +++ b/tools/xtask-llm-benchmark/src/templates/rust/server/Cargo.toml @@ -9,5 +9,5 @@ edition = "2021" crate-type = ["cdylib"] [dependencies] -spacetimedb = { path = "../../../../../../sdks/rust/" } +spacetimedb = { path = "../../../../../../sdks/rust/", features = ["unstable"] } log = "0.4"