LLM benchmark: new evals + improved tooling

.github/workflows/llm-benchmark-periodic.yml

@@ -48,9 +48,29 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  prepare_matrix:
+    name: Prepare benchmark matrix
+    runs-on: ubuntu-latest
+    outputs:
+      languages: ${{ steps.matrix.outputs.languages }}
+    steps:
+      - id: matrix
+        env:
+          INPUT_LANGUAGES: ${{ inputs.languages || 'rust,csharp,typescript' }}
+        run: |
+          languages="$(jq -cn --arg value "$INPUT_LANGUAGES" '$value | split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))')"
+          echo "languages=$languages" >> "$GITHUB_OUTPUT"
+
   run-benchmarks:
+    name: Run benchmarks (${{ matrix.lang }})
+    needs: prepare_matrix
     runs-on: spacetimedb-new-runner-2
 
+    strategy:
+      fail-fast: false
+      matrix:
+        lang: ${{ fromJSON(needs.prepare_matrix.outputs.languages) }}
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -61,11 +81,13 @@ jobs:
       - uses: Swatinem/rust-cache@v2
 
       - name: Setup .NET SDK
+        if: matrix.lang == 'csharp'
         uses: actions/setup-dotnet@v4
         with:
           global-json-file: global.json
 
       - name: Install WASI workload
+        if: matrix.lang == 'csharp'
         env:
           DOTNET_MULTILEVEL_LOOKUP: "0"
           DOTNET_CLI_HOME: ${{ runner.temp }}/dotnet-home
@@ -74,25 +96,25 @@ jobs:
           dotnet workload install wasi-experimental --skip-manifest-update --disable-parallel
 
       - name: Pack C# runtime packages
-        if: ${{ github.event_name != 'workflow_dispatch' || contains(inputs.languages || 'rust,csharp,typescript', 'csharp') }}
+        if: matrix.lang == 'csharp'
         run: |
           dotnet pack -c Release crates/bindings-csharp/BSATN.Runtime
           dotnet pack -c Release crates/bindings-csharp/Runtime
 
       - name: Set up Node.js
-        if: ${{ github.event_name != 'workflow_dispatch' || contains(inputs.languages || 'rust,csharp,typescript', 'typescript') }}
+        if: matrix.lang == 'typescript'
         uses: actions/setup-node@v4
         with:
           node-version: 22
 
       - name: Install pnpm
-        if: ${{ github.event_name != 'workflow_dispatch' || contains(inputs.languages || 'rust,csharp,typescript', 'typescript') }}
+        if: matrix.lang == 'typescript'
         uses: ./.github/actions/setup-pnpm
         with:
           run_install: true
 
       - name: Build TypeScript SDK
-        if: ${{ github.event_name != 'workflow_dispatch' || contains(inputs.languages || 'rust,csharp,typescript', 'typescript') }}
+        if: matrix.lang == 'typescript'
         run: pnpm build
         working-directory: crates/bindings-typescript
 
@@ -119,16 +141,19 @@ jobs:
           DOTNET_SKIP_FIRST_TIME_EXPERIENCE: "1"
           MSBUILDDISABLENODEREUSE: "1"
           DOTNET_CLI_USE_MSBUILD_SERVER: "0"
-          LLM_BENCH_CSHARP_CONCURRENCY: "1"
-          INPUT_LANGUAGES: ${{ inputs.languages || 'rust,csharp,typescript' }}
+          LLM_BENCH_CONCURRENCY: "8"
+          LLM_BENCH_RUST_CONCURRENCY: "4"
+          LLM_BENCH_CSHARP_CONCURRENCY: "2"
+          LLM_BENCH_ROUTE_CONCURRENCY: ${{ matrix.lang == 'typescript' && '4' || '2' }}
+          INPUT_LANGUAGE: ${{ matrix.lang }}
           INPUT_MODEL_SET: ${{ inputs.model_set || 'website_active' }}
           INPUT_MODELS: ${{ inputs.models || '' }}
           INPUT_MODES: ${{ inputs.modes || 'guidelines,no_context' }}
           INPUT_CATEGORIES: ${{ inputs.categories || '' }}
           INPUT_TASKS: ${{ inputs.tasks || '' }}
           INPUT_DRY_RUN: ${{ inputs.dry_run || 'false' }}
         run: |
-          LANGS="$INPUT_LANGUAGES"
+          LANG="$INPUT_LANGUAGE"
           MODEL_SET="$INPUT_MODEL_SET"
           MODELS="$INPUT_MODELS"
           MODES="$INPUT_MODES"
@@ -162,45 +187,21 @@ jobs:
               ;;
           esac
 
-          SUCCEEDED=0
-          FAILED=0
-          for LANG in $(echo "$LANGS" | tr ',' ' '); do
-            EXTRA_ARGS=()
-            if [ -n "$CATEGORIES" ]; then
-              EXTRA_ARGS+=(--categories "$CATEGORIES")
-            fi
-            if [ -n "$TASKS" ]; then
-              EXTRA_ARGS+=(--tasks "$TASKS")
-            fi
-            if [ "$DRY_RUN" = "true" ]; then
-              EXTRA_ARGS+=(--dry-run)
-            fi
-
-            if [ "$MODEL_SET" = "website_active" ]; then
-              if llm_benchmark run --lang "$LANG" --modes "$MODES" --model-source remote "${EXTRA_ARGS[@]}"; then
-                SUCCEEDED=$((SUCCEEDED + 1))
-              else
-                echo "::warning::Benchmark run failed for lang=$LANG"
-                FAILED=$((FAILED + 1))
-              fi
-            elif [ "$MODEL_SET" = "local_defaults" ]; then
-              if llm_benchmark run --lang "$LANG" --modes "$MODES" "${EXTRA_ARGS[@]}"; then
-                SUCCEEDED=$((SUCCEEDED + 1))
-              else
-                echo "::warning::Benchmark run failed for lang=$LANG"
-                FAILED=$((FAILED + 1))
-              fi
-            else
-              if llm_benchmark run --lang "$LANG" --modes "$MODES" --models "${MODEL_ARGS[@]}" "${EXTRA_ARGS[@]}"; then
-                SUCCEEDED=$((SUCCEEDED + 1))
-              else
-                echo "::warning::Benchmark run failed for lang=$LANG models=$MODELS"
-                FAILED=$((FAILED + 1))
-              fi
-            fi
-          done
-          echo "Benchmark runs: $SUCCEEDED succeeded, $FAILED failed"
-          if [ "$FAILED" -gt 0 ]; then
-            echo "::error::$FAILED benchmark run(s) failed"
-            exit 1
+          EXTRA_ARGS=()
+          if [ -n "$CATEGORIES" ]; then
+            EXTRA_ARGS+=(--categories "$CATEGORIES")
+          fi
+          if [ -n "$TASKS" ]; then
+            EXTRA_ARGS+=(--tasks "$TASKS")
+          fi
+          if [ "$DRY_RUN" = "true" ]; then
+            EXTRA_ARGS+=(--dry-run)
+          fi
+
+          if [ "$MODEL_SET" = "website_active" ]; then
+            llm_benchmark run --lang "$LANG" --modes "$MODES" --model-source remote "${EXTRA_ARGS[@]}"
+          elif [ "$MODEL_SET" = "local_defaults" ]; then
+            llm_benchmark run --lang "$LANG" --modes "$MODES" "${EXTRA_ARGS[@]}"
+          else
+            llm_benchmark run --lang "$LANG" --modes "$MODES" --models "${MODEL_ARGS[@]}" "${EXTRA_ARGS[@]}"
           fi

skills/csharp-server/SKILL.md

@@ -17,6 +17,8 @@ metadata:
 
 Reducers are static methods in a `static partial class`; tables are `public partial struct`s. This reference keeps everything in one `public static partial class Module`, which needs only `using SpacetimeDB;`:
 
+Methods exported through SpacetimeDB attributes, including reducers, procedures, views, HTTP handlers, and routers, must be `public static`; generated bindings invoke them from another class.
+
 ```csharp
 using SpacetimeDB;
 
@@ -51,8 +53,10 @@ public partial struct Entity
     [PrimaryKey]
     [AutoInc]
     public ulong Id;
+    [SpacetimeDB.Index.BTree]
     public Identity Owner;
     public string Name;
+    [SpacetimeDB.Index.BTree]
     public bool Active;
 }
 ```
@@ -90,8 +94,11 @@ The complete set of column attributes:
 [AutoInc]             // auto-increment (use 0 as placeholder on insert)
 [Unique]              // unique constraint; indexes the column, enables .Find()
 [SpacetimeDB.Index.BTree]  // btree index (enables .Filter() on this column)
+[Default(true)]       // migration-safe default for a newly appended field
 ```
 
+Defaults support compatible addition of a newly appended field. Do not apply `[Default(...)]` to primary-key, unique, or auto-increment fields.
+
 ## Indexes
 
 Write the index attribute fully qualified: `[SpacetimeDB.Index.BTree]`. Prefer inline for single-column; multi-column uses struct-level:
@@ -155,24 +162,53 @@ public static void OnConnect(ReducerContext ctx) { ... }
 public static void OnDisconnect(ReducerContext ctx) { ... }
 ```
 
+`ctx.ConnectionId` is `ConnectionId?`, including in connection lifecycle reducers. Check or unwrap it before storing it in a non-nullable column or passing it to an index accessor.
+
 ## Views
 
 ```csharp
 // Anonymous view (same result for all clients):
 [SpacetimeDB.View(Accessor = "ActiveUsers", Public = true)]
 public static List<Entity> ActiveUsers(AnonymousViewContext ctx)
 {
-    return ctx.Db.Entity.Iter().Where(e => e.Active).ToList();
+    return ctx.Db.Entity.Active.Filter(true).ToList();
 }
 
 // Per-user view:
-[SpacetimeDB.View(Accessor = "MyProfile", Public = true)]
-public static Entity? MyProfile(ViewContext ctx)
+[SpacetimeDB.View(Accessor = "MyEntities", Public = true)]
+public static List<Entity> MyEntities(ViewContext ctx)
 {
-    return ctx.Db.Entity.Identity.Find(ctx.Sender) as Entity?;
+    return ctx.Db.Entity.Owner.Filter(ctx.Sender).ToList();
 }
 ```
 
+View contexts expose read-only table handles. These support `Count` plus `Find`/`Filter` on declared indexes, but not full-table `Iter()`. Start procedural view traversal from an indexed lookup or filter.
+
+Query-builder views use `ViewContext`, `ctx.From`, and return `IQuery<T>` directly. Use `Where` for predicates and `RightSemijoin` when the result should contain right-side rows that have a matching left-side row:
+
+```csharp
+ctx.From.Article().Where(article => article.Published.Eq(true));
+ctx.From.Subscription().RightSemijoin(
+    ctx.From.Account(),
+    (subscription, account) => subscription.AccountId.Eq(account.Id)
+);
+```
+
+Declare a procedural view primary key in its attribute: `[SpacetimeDB.View(Accessor = "CatalogEntry", Public = true, PrimaryKey = nameof(CatalogRow.Sku))]`.
+
+Procedural-view primary keys are explicit schema metadata. Add one only when the view itself is required to expose a primary key; a source table's primary key is not inherited by the view.
+
+Inclusive btree ranges use tuples, for example `ctx.Db.Shipment.DeliverBy.Filter((new Timestamp(1_000), new Timestamp(2_000)))`.
+
+## Client Visibility Filters
+
+```csharp
+[ClientVisibilityFilter]
+public static readonly Filter PrivateNoteFilter = new Filter.Sql(
+    "SELECT * FROM OwnedRow WHERE Owner = :sender"
+);
+```
+
 ## Reducer Context API
 
 `ReducerContext` (`ctx`) is the only source of sender identity, time, and randomness; stdlib clocks and RNG are unavailable in modules.
@@ -229,6 +265,71 @@ var at = new ScheduleAt.Interval(TimeSpan.FromSeconds(5));
 ctx.Db.TickTimer.Insert(new TickTimer { ScheduledId = 0, ScheduledAt = at });
 ```
 
+Scheduled reducer callbacks use the ordinary `[SpacetimeDB.Reducer]` attribute. There is no `ReducerKind.Scheduled`; the table's `Scheduled` option associates the callback.
+
+To construct a `ConnectionId` from a 128-bit numeric representation, encode it as exactly 16 little-endian bytes and call `ConnectionId.From(bytes)`. The result is nullable and must be checked before use.
+
+## Procedures and HTTP
+
+Procedures are unstable APIs, so modules using them should include `#pragma warning disable STDB_UNSTABLE`. They receive `ProcedureContext` and may return `[SpacetimeDB.Type]` values:
+
+```csharp
+[SpacetimeDB.Type]
+public partial struct ResultValue { public string Value; }
+
+[SpacetimeDB.Procedure]
+public static ResultValue Inspect(ProcedureContext ctx, string input) =>
+    new() { Value = input };
+```
+
+Outbound HTTP is available through `ctx.Http`; handle its success/error result before using the response. Open short database transactions with `ctx.WithTx`. Perform network I/O before opening the transaction, and keep only database work inside its callback.
+
+```csharp
+var result = ctx.Http.Get(uri);
+var text = result.Match(
+    response => response.Body.ToStringUtf8Lossy(),
+    error => throw new Exception(error.Message)
+);
+
+ctx.WithTx(tx =>
+{
+    tx.Db.ScoreRecord.Insert(new ScoreRecord { Id = 0, Owner = ctx.Sender, Value = 1 });
+    return 0;
+});
+```
+
+`WithTx` is generic: its callback must return a value (use `return 0;` when no result is needed). The callback's generated `tx.Db` exposes module table accessors; `ProcedureContext` and `HandlerContext` do not expose tables directly.
+
+For non-GET requests, construct and send an `HttpRequest` directly:
+
+```csharp
+var result = ctx.Http.Send(new HttpRequest
+{
+    Uri = uri,
+    Method = SpacetimeDB.HttpMethod.Post,
+    Headers = new() { new HttpHeader("content-type", "text/plain") },
+    Body = HttpBody.FromString(payload),
+});
+```
+
+`HttpResponse.StatusCode` is `ushort`. HTTP headers are `HttpHeader` values, not tuples, and each header's `Value` is `byte[]`; decode text values with `System.Text.Encoding.UTF8.GetString(header.Value)`. Treat bodies as bytes: use `HttpBody.FromString(...)` to create text, `new HttpBody(bytes)` to supply raw bytes, `ToBytes()` to read raw bytes, and `ToStringUtf8Lossy()` to read text. There is no `HttpBody.FromBytes`, and `ToString()` does not return body contents. Qualify `SpacetimeDB.HttpMethod` when .NET's implicit `System.Net.Http` imports could make the name ambiguous.
+
+Scheduled procedures use the ordinary scheduled-table shape. Its `Scheduled` name refers to a `[SpacetimeDB.Procedure]` method taking `ProcedureContext` plus the scheduled row, and database access inside that procedure goes through `ctx.WithTx`.
+
+Inbound HTTP uses handler attributes and one router. Handler database access also goes through `ctx.WithTx`:
+
+```csharp
+[SpacetimeDB.HttpHandler]
+public static HttpResponse Health(HandlerContext ctx, HttpRequest request) => new(
+    200, HttpVersion.Http11, new(), HttpBody.FromString("ok")
+);
+
+[SpacetimeDB.HttpRouter]
+public static Router Routes() => SpacetimeDB.Router.New().Get("/health", Handlers.Health);
+```
+
+`Handlers` is generated from methods marked `[SpacetimeDB.HttpHandler]`; do not declare it yourself. Reference an attributed method in a router as `Handlers.MethodName`.
+
 ## Custom Types
 
 ```csharp