feat: add DrexClassifier, a Drex decision-model scorer (Python and TypeScript)

.env.example

@@ -1,2 +1,4 @@
 OPENAI_API_KEY=your_openai_api_key
 BRAINTRUST_API_KEY=your_braintrust_api_key
+# Optional: only for the Drex scorer (DrexClassifier) and its live tests
+NACE_API_KEY=your_nace_api_key

.github/scripts/check_version_sync.py

@@ -7,6 +7,7 @@
 ROOT = Path(__file__).resolve().parents[2]
 PACKAGE_JSON = ROOT / "package.json"
 PY_VERSION = ROOT / "py" / "autoevals" / "version.py"
+JS_VERSION = ROOT / "js" / "version.ts"
 
 package_version = json.loads(PACKAGE_JSON.read_text(encoding="utf-8"))["version"]
 match = re.search(
@@ -21,6 +22,21 @@
 
 python_version = match.group(1)
 
+js_match = re.search(
+    r'^export const VERSION = ["\']([^"\']+)["\'];?\s*$', JS_VERSION.read_text(encoding="utf-8"), re.MULTILINE
+)
+
+if not js_match:
+    print(f"Could not parse VERSION from {JS_VERSION}", file=sys.stderr)
+    sys.exit(1)
+
+if js_match.group(1) != package_version:
+    print(
+        "Version mismatch detected:\n" f"- package.json: {package_version}\n" f"- js/version.ts: {js_match.group(1)}",
+        file=sys.stderr,
+    )
+    sys.exit(1)
+
 if package_version != python_version:
     print(
         "Version mismatch detected:\n"

.github/workflows/publish.yaml

@@ -41,7 +41,7 @@ on:
   workflow_dispatch:
     inputs:
       _instructions:
-        description: "⚠️ Before starting: merge a version-bump PR (package.json + py/autoevals/version.py, matching). The release publishes the version at the SHA — it can't be overridden here."
+        description: "⚠️ Before starting: merge a version-bump PR (package.json + py/autoevals/version.py + js/version.ts, matching). The release publishes the version at the SHA — it can't be overridden here."
         type: string
         default: "I have merged a version bump PR"
         required: false

AGENTS.md

@@ -90,14 +90,15 @@ type Scorer<Output, Extra> = (
 ## CI and Releases
 
 - Publishing is handled by trusted publishing workflows documented in `docs/PUBLISHING.md`.
-- JavaScript and Python package versions must stay in sync between `package.json` and `py/autoevals/version.py`; CI enforces this via `.github/workflows/version-sync.yaml` and `.github/scripts/check_version_sync.py`.
+- JavaScript and Python package versions must stay in sync between `package.json`, `py/autoevals/version.py` and `js/version.ts`; CI enforces this via `.github/workflows/version-sync.yaml` and `.github/scripts/check_version_sync.py`.
 
 ## Environment Variables
 
 Tests require:
 
 - `OPENAI_API_KEY` or `BRAINTRUST_API_KEY` - For LLM-based evaluations
 - `OPENAI_BASE_URL` (optional) - Custom API endpoint
+- `NACE_API_KEY` - Only for the Drex scorer (`DrexClassifier`) and its live tests, which skip without it; `NACE_BASE_URL` (optional) points it elsewhere than Nace direct
 
 ## Testing Notes
 

README.md

@@ -333,6 +333,7 @@ Eval(
 - SQL
 - Translation
 - Fine-tuned binary classifiers
+- Drex decision-model judge (`DrexClassifier`; probabilities, not verdicts)
 
 ### RAG evaluations
 

SCORERS.md

@@ -5,6 +5,7 @@ Complete reference for all scorers available in Autoevals, including parameters,
 ## Table of contents
 
 - [LLM-as-a-judge scorers](#llm-as-a-judge-scorers)
+- [Drex decision-model scorer](#drex-decision-model-scorer)
 - [RAG (Retrieval-Augmented Generation) scorers](#rag-retrieval-augmented-generation-scorers)
 - [Heuristic scorers](#heuristic-scorers)
 - [JSON scorers](#json-scorers)
@@ -194,6 +195,64 @@ Evaluates translation quality.
 
 ---
 
+## Drex decision-model scorer
+
+### DrexClassifier
+
+Asks a [Drex](https://console.nace.ai/docs) decision model one typed question (`noul` yes/no, `choice` or `score`) about a case and returns the probability mass on the outcomes you call a pass. Drex answers with probabilities, not a verdict, so the full distribution stays in `metadata`.
+
+No extra package is needed: the scorer calls `POST {base_url}/v1/systemone` itself, with `httpx` when it is installed (it ships with `openai`) and the standard library otherwise, and with the global `fetch` in TypeScript. It works on Python 3.10 and later. Set `NACE_API_KEY` (and `NACE_BASE_URL` if you do not use Nace direct).
+
+**Parameters:**
+
+- `question` (required): `DrexQuestion.noul(instructions=None, *, true=None, false=None, yes_is="pass"|"fail")`, `DrexQuestion.choice(choices, *, scores={label: 0..1}, instructions=None)` or `DrexQuestion.score(levels, *, instructions=None, reverse=False)`. TypeScript: `drexQuestion.noul(...)`, `.choice(...)`, `.score(...)`. A criteria-only `noul` (`true=`/`false=` without instructions) is supported. Python also accepts a dict `{name: DrexQuestion}`: with more than one question the scorer must be called with `eval_many(output, expected, **case)` / `eval_many_async(...)`, which returns one `Score` per question in order, and `.eval()` raises (a one-entry dict behaves like a single question, named after its key). TypeScript: `DrexClassifierMany({ questions, output, ... })` returns `Score[]` and is not a `.partial` scorer
+- `output` (required), `expected`, `input`, `context`: the case. Drex reads the fields in `state_fields` (default all four; `output` alone is sent as plain text)
+- `model` (string, optional): any Drex model id, passed through. Default `drex-max`, which also reads `images`. Use `drex-v2.0` for large text-only rubrics if Drex rejects the question count
+- `images` (optional, `drex-max` only): bytes (PNG, JPEG or WebP), a base64 string, a data URL, or a `{"type": "workspace_file", "workspace_id": ..., "file_id": ...}` reference. Images sent to a text-only model are refused, never dropped. Through OpenRouter only inline images work; uploaded files are refused
+- `threshold` (float, optional): turn the 0-1 score into 1 or 0
+- `api_key` (optional): defaults to `NACE_API_KEY` (`OPENROUTER_API_KEY` for OpenRouter)
+- `base_url` (string, optional): defaults to `NACE_BASE_URL`, then Nace direct. Drex can also run on your own infrastructure under your own domain: set `base_url` (or `NACE_BASE_URL`) to that host, for example `https://drex.example.com`, and the scorer works as it does against Nace, with your Nace key from `api_key` or `NACE_API_KEY`. A 402 then points to `<base_url>/dashboard/billing`. `https://openrouter.ai/api/v1` runs the same scorer through OpenRouter: pass the OpenRouter key as `api_key` or in `OPENROUTER_API_KEY` (it is never taken from `NACE_API_KEY`, and a Nace host never uses `OPENROUTER_API_KEY`) and use namespaced model ids (`nace-ai/drex-max`, `nace-ai/drex-v2.0`). The default model there is `nace-ai/drex-max`; a bare id such as `drex-max` does not exist on OpenRouter, so it is refused with a clear error. `nace-ai/drex-latest` stays text-only like `drex-latest`. OpenRouter is recognized by its host (`openrouter.ai` or a subdomain)
+- `timeout` (seconds per attempt, default 60; keep it at 60 or more) and `max_retries` (default 2; retries 408, 429 and 5xx including 529, and connection errors, honouring `retry-after-ms`, then `retry-after`; a requested wait over 60 s is not slept: `DrexError` is raised at once). A request over Drex's 4 MiB body limit is refused before sending; the scorer does not upload files. TypeScript: `apiKey`, `timeoutMs`, `maxRetries`
+
+TypeScript takes the same options in camelCase (`yesIs`, `stateFields`, `baseUrl`, `timeoutMs`, `maxRetries`) and `DrexError` carries `requestId`.
+
+**Score Range:** 0-1, higher is better.
+
+- `noul`: probability of yes (or of no with `yes_is="fail"`)
+- `choice`: probability-weighted sum of the `scores` you give each label
+- `score`: expected level divided by the top level (`reverse` flips it); the raw level and legend stay in `metadata`
+
+`metadata` holds `probabilities`, `request_id`, `model`, `usage` (`input_tokens`, `image_tokens`, `output_tokens`), `evaluation_time_ms` and the unthresholded `raw_value`. `usage` is attached only to the first score of a request; the others carry `usage: null`, and `usage_counted_on` names the score that holds it. API failures (401, 402, 422, 429/529 after retries, timeouts) raise `DrexError` (`status`, `request_id`) and are never a low score. Invalid arguments are caught before any request and raise `ValueError` (TypeScript `Error`). Answer several questions about one case in a single request with `eval_many` / `DrexClassifierMany`.
+
+**Example:**
+
+```python
+from autoevals.drex import DrexClassifier, DrexQuestion
+
+grounded = DrexClassifier(DrexQuestion.noul("Is every claim in the output supported by the context?"), name="Grounded")
+result = grounded.eval("Paris is the capital of France.", context="France's capital is Paris.")
+print(result.score, result.metadata["probabilities"])
+```
+
+```typescript
+import { DrexClassifier, drexQuestion } from "autoevals";
+
+const grounded = DrexClassifier.partial({
+  name: "Grounded",
+  question: drexQuestion.noul(
+    "Is every claim in the output supported by the context?",
+  ),
+});
+const result = await grounded({
+  output: "Paris is the capital of France.",
+  context: "France's capital is Paris.",
+});
+```
+
+See [`examples/drex`](examples/drex) for a Braintrust experiment.
+
+---
+
 ## RAG (Retrieval-Augmented Generation) scorers
 
 These scorers evaluate RAG systems by assessing both context retrieval and answer generation quality.

docs/PUBLISHING.md

@@ -6,8 +6,8 @@ actions from [`braintrustdata/sdk-actions`](https://github.com/braintrustdata/sd
 
 ## Versioning
 
-`package.json` and `py/autoevals/version.py` must hold the same version; `version-sync.yaml` (CI)
-enforces it on PRs. The release publishes the **committed** version — bump both and merge first.
+`package.json`, `py/autoevals/version.py` and `js/version.ts` must hold the same version; `version-sync.yaml` (CI)
+enforces it on PRs. The release publishes the **committed** version — bump all three and merge first.
 
 ## Releasing
 
@@ -26,7 +26,7 @@ One approval releases both packages; if either lane fails before the gate, neith
 
 ### Steps
 
-1. Bump `package.json` + `py/autoevals/version.py` together and merge to `main`.
+1. Bump `package.json` + `py/autoevals/version.py` + `js/version.ts` together and merge to `main`.
 2. Run `publish` with `release_type=stable`, `sha=<merged commit>`.
 3. Approve the `publish` environment when the run parks at the gate.
 

examples/drex/drex_eval.eval.ts

@@ -0,0 +1,41 @@
+// Braintrust experiment that judges answers with Drex.
+//
+//   npm install autoevals braintrust
+//   export NACE_API_KEY=nace_sk_...      # https://console.nace.ai
+//   export BRAINTRUST_API_KEY=...
+//   npx braintrust eval examples/drex/drex_eval.eval.ts
+//
+// Drex returns probabilities. The scorer keeps them in each score's metadata, so you can pick a
+// threshold in Braintrust instead of guessing one up front.
+import { Eval } from "braintrust";
+import { DrexClassifier, drexQuestion } from "autoevals";
+
+const cases = [
+  {
+    question: "What is the capital of France?",
+    context: "Paris is the capital of France.",
+    output: "Paris.",
+  },
+  {
+    question: "What is the capital of France?",
+    context: "Paris is the capital of France.",
+    output: "Lyon, since 1960.",
+  },
+];
+
+// The default model is drex-max. For large text-only rubrics Drex may require model: "drex-v2.0".
+const grounded = DrexClassifier.partial({
+  name: "Grounded",
+  question: drexQuestion.noul(
+    "Is every claim in the output supported by the context?",
+  ),
+});
+
+Eval("Drex judge example", {
+  data: () => cases.map((c) => ({ input: c })),
+  task: (input) => input.output,
+  scores: [
+    ({ input, output }) =>
+      grounded({ output, input: input.question, context: input.context }),
+  ],
+});

examples/drex/drex_eval.py

@@ -0,0 +1,58 @@
+"""Braintrust experiment that judges answers with Drex.
+
+    pip install autoevals braintrust
+    export NACE_API_KEY=nace_sk_...      # https://console.nace.ai
+    export BRAINTRUST_API_KEY=...
+    braintrust eval examples/drex/drex_eval.py
+
+Drex returns probabilities. The scorer keeps them in each score's metadata, so you can pick a threshold
+in Braintrust instead of guessing one up front.
+"""
+
+from autoevals.drex import DrexClassifier, DrexQuestion
+from braintrust import Eval
+
+DATA = [
+    {
+        "input": {"question": "What is the capital of France?", "context": "Paris is the capital of France."},
+        "output": "Paris.",
+    },
+    {
+        "input": {"question": "What is the capital of France?", "context": "Paris is the capital of France."},
+        "output": "Lyon, since 1960.",
+    },
+]
+
+grounded = DrexClassifier(
+    DrexQuestion.noul("Is every claim in the output supported by the context?"),
+    name="Grounded",
+    # Default model is drex-max. For large text-only rubrics Drex may require model="drex-v2.0".
+)
+
+helpful = DrexClassifier(
+    DrexQuestion.score(
+        ["useless", "partly useful", "useful", "very useful"], instructions="How useful is the output?"
+    ),
+    name="Helpful",
+)
+
+
+def task(case):
+    # Replace with your own application. Here the "application" returns the prepared output.
+    return case["output"]
+
+
+def grounded_scorer(input, output, expected=None):
+    return grounded.eval(output, expected, input=input["question"], context=input["context"])
+
+
+def helpful_scorer(input, output, expected=None):
+    return helpful.eval(output, expected, input=input["question"])
+
+
+Eval(
+    "Drex judge example",
+    data=lambda: [{"input": {**row["input"], "output": row["output"]}} for row in DATA],
+    task=task,
+    scores=[grounded_scorer, helpful_scorer],
+)