diff --git a/.env.example b/.env.example index fca388a1..f632a9b1 100644 --- a/.env.example +++ b/.env.example @@ -1,2 +1,4 @@ OPENAI_API_KEY=your_openai_api_key BRAINTRUST_API_KEY=your_braintrust_api_key +# Optional: only for the Drex scorer (DrexClassifier) and its live tests +NACE_API_KEY=your_nace_api_key diff --git a/.github/scripts/check_version_sync.py b/.github/scripts/check_version_sync.py index 7a57151e..01f58086 100644 --- a/.github/scripts/check_version_sync.py +++ b/.github/scripts/check_version_sync.py @@ -7,6 +7,7 @@ ROOT = Path(__file__).resolve().parents[2] PACKAGE_JSON = ROOT / "package.json" PY_VERSION = ROOT / "py" / "autoevals" / "version.py" +JS_VERSION = ROOT / "js" / "version.ts" package_version = json.loads(PACKAGE_JSON.read_text(encoding="utf-8"))["version"] match = re.search( @@ -21,6 +22,21 @@ python_version = match.group(1) +js_match = re.search( + r'^export const VERSION = ["\']([^"\']+)["\'];?\s*$', JS_VERSION.read_text(encoding="utf-8"), re.MULTILINE +) + +if not js_match: + print(f"Could not parse VERSION from {JS_VERSION}", file=sys.stderr) + sys.exit(1) + +if js_match.group(1) != package_version: + print( + "Version mismatch detected:\n" f"- package.json: {package_version}\n" f"- js/version.ts: {js_match.group(1)}", + file=sys.stderr, + ) + sys.exit(1) + if package_version != python_version: print( "Version mismatch detected:\n" diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 6411a179..c199f7b1 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -41,7 +41,7 @@ on: workflow_dispatch: inputs: _instructions: - description: "⚠️ Before starting: merge a version-bump PR (package.json + py/autoevals/version.py, matching). The release publishes the version at the SHA — it can't be overridden here." + description: "⚠️ Before starting: merge a version-bump PR (package.json + py/autoevals/version.py + js/version.ts, matching). The release publishes the version at the SHA — it can't be overridden here." type: string default: "I have merged a version bump PR" required: false diff --git a/AGENTS.md b/AGENTS.md index 85089885..2d60d434 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -90,7 +90,7 @@ type Scorer = ( ## CI and Releases - Publishing is handled by trusted publishing workflows documented in `docs/PUBLISHING.md`. -- JavaScript and Python package versions must stay in sync between `package.json` and `py/autoevals/version.py`; CI enforces this via `.github/workflows/version-sync.yaml` and `.github/scripts/check_version_sync.py`. +- JavaScript and Python package versions must stay in sync between `package.json`, `py/autoevals/version.py` and `js/version.ts`; CI enforces this via `.github/workflows/version-sync.yaml` and `.github/scripts/check_version_sync.py`. ## Environment Variables @@ -98,6 +98,7 @@ Tests require: - `OPENAI_API_KEY` or `BRAINTRUST_API_KEY` - For LLM-based evaluations - `OPENAI_BASE_URL` (optional) - Custom API endpoint +- `NACE_API_KEY` - Only for the Drex scorer (`DrexClassifier`) and its live tests, which skip without it; `NACE_BASE_URL` (optional) points it elsewhere than Nace direct ## Testing Notes diff --git a/README.md b/README.md index 27a9f063..0a2d8f23 100644 --- a/README.md +++ b/README.md @@ -333,6 +333,7 @@ Eval( - SQL - Translation - Fine-tuned binary classifiers +- Drex decision-model judge (`DrexClassifier`; probabilities, not verdicts) ### RAG evaluations diff --git a/SCORERS.md b/SCORERS.md index 56893245..d5022cb5 100644 --- a/SCORERS.md +++ b/SCORERS.md @@ -5,6 +5,7 @@ Complete reference for all scorers available in Autoevals, including parameters, ## Table of contents - [LLM-as-a-judge scorers](#llm-as-a-judge-scorers) +- [Drex decision-model scorer](#drex-decision-model-scorer) - [RAG (Retrieval-Augmented Generation) scorers](#rag-retrieval-augmented-generation-scorers) - [Heuristic scorers](#heuristic-scorers) - [JSON scorers](#json-scorers) @@ -194,6 +195,64 @@ Evaluates translation quality. --- +## Drex decision-model scorer + +### DrexClassifier + +Asks a [Drex](https://console.nace.ai/docs) decision model one typed question (`noul` yes/no, `choice` or `score`) about a case and returns the probability mass on the outcomes you call a pass. Drex answers with probabilities, not a verdict, so the full distribution stays in `metadata`. + +No extra package is needed: the scorer calls `POST {base_url}/v1/systemone` itself, with `httpx` when it is installed (it ships with `openai`) and the standard library otherwise, and with the global `fetch` in TypeScript. It works on Python 3.10 and later. Set `NACE_API_KEY` (and `NACE_BASE_URL` if you do not use Nace direct). + +**Parameters:** + +- `question` (required): `DrexQuestion.noul(instructions=None, *, true=None, false=None, yes_is="pass"|"fail")`, `DrexQuestion.choice(choices, *, scores={label: 0..1}, instructions=None)` or `DrexQuestion.score(levels, *, instructions=None, reverse=False)`. TypeScript: `drexQuestion.noul(...)`, `.choice(...)`, `.score(...)`. A criteria-only `noul` (`true=`/`false=` without instructions) is supported. Python also accepts a dict `{name: DrexQuestion}`: with more than one question the scorer must be called with `eval_many(output, expected, **case)` / `eval_many_async(...)`, which returns one `Score` per question in order, and `.eval()` raises (a one-entry dict behaves like a single question, named after its key). TypeScript: `DrexClassifierMany({ questions, output, ... })` returns `Score[]` and is not a `.partial` scorer +- `output` (required), `expected`, `input`, `context`: the case. Drex reads the fields in `state_fields` (default all four; `output` alone is sent as plain text) +- `model` (string, optional): any Drex model id, passed through. Default `drex-max`, which also reads `images`. Use `drex-v2.0` for large text-only rubrics if Drex rejects the question count +- `images` (optional, `drex-max` only): bytes (PNG, JPEG or WebP), a base64 string, a data URL, or a `{"type": "workspace_file", "workspace_id": ..., "file_id": ...}` reference. Images sent to a text-only model are refused, never dropped. Through OpenRouter only inline images work; uploaded files are refused +- `threshold` (float, optional): turn the 0-1 score into 1 or 0 +- `api_key` (optional): defaults to `NACE_API_KEY` (`OPENROUTER_API_KEY` for OpenRouter) +- `base_url` (string, optional): defaults to `NACE_BASE_URL`, then Nace direct. Drex can also run on your own infrastructure under your own domain: set `base_url` (or `NACE_BASE_URL`) to that host, for example `https://drex.example.com`, and the scorer works as it does against Nace, with your Nace key from `api_key` or `NACE_API_KEY`. A 402 then points to `/dashboard/billing`. `https://openrouter.ai/api/v1` runs the same scorer through OpenRouter: pass the OpenRouter key as `api_key` or in `OPENROUTER_API_KEY` (it is never taken from `NACE_API_KEY`, and a Nace host never uses `OPENROUTER_API_KEY`) and use namespaced model ids (`nace-ai/drex-max`, `nace-ai/drex-v2.0`). The default model there is `nace-ai/drex-max`; a bare id such as `drex-max` does not exist on OpenRouter, so it is refused with a clear error. `nace-ai/drex-latest` stays text-only like `drex-latest`. OpenRouter is recognized by its host (`openrouter.ai` or a subdomain) +- `timeout` (seconds per attempt, default 60; keep it at 60 or more) and `max_retries` (default 2; retries 408, 429 and 5xx including 529, and connection errors, honouring `retry-after-ms`, then `retry-after`; a requested wait over 60 s is not slept: `DrexError` is raised at once). A request over Drex's 4 MiB body limit is refused before sending; the scorer does not upload files. TypeScript: `apiKey`, `timeoutMs`, `maxRetries` + +TypeScript takes the same options in camelCase (`yesIs`, `stateFields`, `baseUrl`, `timeoutMs`, `maxRetries`) and `DrexError` carries `requestId`. + +**Score Range:** 0-1, higher is better. + +- `noul`: probability of yes (or of no with `yes_is="fail"`) +- `choice`: probability-weighted sum of the `scores` you give each label +- `score`: expected level divided by the top level (`reverse` flips it); the raw level and legend stay in `metadata` + +`metadata` holds `probabilities`, `request_id`, `model`, `usage` (`input_tokens`, `image_tokens`, `output_tokens`), `evaluation_time_ms` and the unthresholded `raw_value`. `usage` is attached only to the first score of a request; the others carry `usage: null`, and `usage_counted_on` names the score that holds it. API failures (401, 402, 422, 429/529 after retries, timeouts) raise `DrexError` (`status`, `request_id`) and are never a low score. Invalid arguments are caught before any request and raise `ValueError` (TypeScript `Error`). Answer several questions about one case in a single request with `eval_many` / `DrexClassifierMany`. + +**Example:** + +```python +from autoevals.drex import DrexClassifier, DrexQuestion + +grounded = DrexClassifier(DrexQuestion.noul("Is every claim in the output supported by the context?"), name="Grounded") +result = grounded.eval("Paris is the capital of France.", context="France's capital is Paris.") +print(result.score, result.metadata["probabilities"]) +``` + +```typescript +import { DrexClassifier, drexQuestion } from "autoevals"; + +const grounded = DrexClassifier.partial({ + name: "Grounded", + question: drexQuestion.noul( + "Is every claim in the output supported by the context?", + ), +}); +const result = await grounded({ + output: "Paris is the capital of France.", + context: "France's capital is Paris.", +}); +``` + +See [`examples/drex`](examples/drex) for a Braintrust experiment. + +--- + ## RAG (Retrieval-Augmented Generation) scorers These scorers evaluate RAG systems by assessing both context retrieval and answer generation quality. diff --git a/docs/PUBLISHING.md b/docs/PUBLISHING.md index 731f3cc9..411f4fc7 100644 --- a/docs/PUBLISHING.md +++ b/docs/PUBLISHING.md @@ -6,8 +6,8 @@ actions from [`braintrustdata/sdk-actions`](https://github.com/braintrustdata/sd ## Versioning -`package.json` and `py/autoevals/version.py` must hold the same version; `version-sync.yaml` (CI) -enforces it on PRs. The release publishes the **committed** version — bump both and merge first. +`package.json`, `py/autoevals/version.py` and `js/version.ts` must hold the same version; `version-sync.yaml` (CI) +enforces it on PRs. The release publishes the **committed** version — bump all three and merge first. ## Releasing @@ -26,7 +26,7 @@ One approval releases both packages; if either lane fails before the gate, neith ### Steps -1. Bump `package.json` + `py/autoevals/version.py` together and merge to `main`. +1. Bump `package.json` + `py/autoevals/version.py` + `js/version.ts` together and merge to `main`. 2. Run `publish` with `release_type=stable`, `sha=`. 3. Approve the `publish` environment when the run parks at the gate. diff --git a/examples/drex/drex_eval.eval.ts b/examples/drex/drex_eval.eval.ts new file mode 100644 index 00000000..106d2963 --- /dev/null +++ b/examples/drex/drex_eval.eval.ts @@ -0,0 +1,41 @@ +// Braintrust experiment that judges answers with Drex. +// +// npm install autoevals braintrust +// export NACE_API_KEY=nace_sk_... # https://console.nace.ai +// export BRAINTRUST_API_KEY=... +// npx braintrust eval examples/drex/drex_eval.eval.ts +// +// Drex returns probabilities. The scorer keeps them in each score's metadata, so you can pick a +// threshold in Braintrust instead of guessing one up front. +import { Eval } from "braintrust"; +import { DrexClassifier, drexQuestion } from "autoevals"; + +const cases = [ + { + question: "What is the capital of France?", + context: "Paris is the capital of France.", + output: "Paris.", + }, + { + question: "What is the capital of France?", + context: "Paris is the capital of France.", + output: "Lyon, since 1960.", + }, +]; + +// The default model is drex-max. For large text-only rubrics Drex may require model: "drex-v2.0". +const grounded = DrexClassifier.partial({ + name: "Grounded", + question: drexQuestion.noul( + "Is every claim in the output supported by the context?", + ), +}); + +Eval("Drex judge example", { + data: () => cases.map((c) => ({ input: c })), + task: (input) => input.output, + scores: [ + ({ input, output }) => + grounded({ output, input: input.question, context: input.context }), + ], +}); diff --git a/examples/drex/drex_eval.py b/examples/drex/drex_eval.py new file mode 100644 index 00000000..d78336f6 --- /dev/null +++ b/examples/drex/drex_eval.py @@ -0,0 +1,58 @@ +"""Braintrust experiment that judges answers with Drex. + + pip install autoevals braintrust + export NACE_API_KEY=nace_sk_... # https://console.nace.ai + export BRAINTRUST_API_KEY=... + braintrust eval examples/drex/drex_eval.py + +Drex returns probabilities. The scorer keeps them in each score's metadata, so you can pick a threshold +in Braintrust instead of guessing one up front. +""" + +from autoevals.drex import DrexClassifier, DrexQuestion +from braintrust import Eval + +DATA = [ + { + "input": {"question": "What is the capital of France?", "context": "Paris is the capital of France."}, + "output": "Paris.", + }, + { + "input": {"question": "What is the capital of France?", "context": "Paris is the capital of France."}, + "output": "Lyon, since 1960.", + }, +] + +grounded = DrexClassifier( + DrexQuestion.noul("Is every claim in the output supported by the context?"), + name="Grounded", + # Default model is drex-max. For large text-only rubrics Drex may require model="drex-v2.0". +) + +helpful = DrexClassifier( + DrexQuestion.score( + ["useless", "partly useful", "useful", "very useful"], instructions="How useful is the output?" + ), + name="Helpful", +) + + +def task(case): + # Replace with your own application. Here the "application" returns the prepared output. + return case["output"] + + +def grounded_scorer(input, output, expected=None): + return grounded.eval(output, expected, input=input["question"], context=input["context"]) + + +def helpful_scorer(input, output, expected=None): + return helpful.eval(output, expected, input=input["question"]) + + +Eval( + "Drex judge example", + data=lambda: [{"input": {**row["input"], "output": row["output"]}} for row in DATA], + task=task, + scores=[grounded_scorer, helpful_scorer], +) diff --git a/js/drex.test.ts b/js/drex.test.ts new file mode 100644 index 00000000..a81025d3 --- /dev/null +++ b/js/drex.test.ts @@ -0,0 +1,1293 @@ +// Offline tests run against a fake Drex (msw). Live tests call the real API and run only when +// NACE_API_KEY is set: `NACE_API_KEY=nace_sk_... pnpm run test -- js/drex.test.ts -t live`. +import { deflateSync } from "node:zlib"; +import { delay, http, HttpResponse } from "msw"; +import { setupServer } from "msw/node"; +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + test, + vi, +} from "vitest"; +import { + buildState, + DREX_CLIENT_NAME, + DrexClassifier, + DrexClassifierMany, + DrexError, + drexQuestion, + isTextOnly, +} from "./drex"; + +const BASE = "https://drex.test"; +const URL_ = `${BASE}/v1/systemone`; + +function crc32(buf: Uint8Array): number { + let c = ~0; + for (const byte of buf) { + c ^= byte; + for (let k = 0; k < 8; k++) c = c & 1 ? (c >>> 1) ^ 0xedb88320 : c >>> 1; + } + return ~c >>> 0; +} + +function png(rgb: [number, number, number]): Uint8Array { + const chunk = (kind: string, data: Uint8Array) => { + const body = Buffer.concat([Buffer.from(kind, "latin1"), data]); + const out = Buffer.alloc(12 + data.length); + out.writeUInt32BE(data.length, 0); + body.copy(out, 4); + out.writeUInt32BE(crc32(body), 8 + data.length); + return out; + }; + const ihdr = Buffer.alloc(13); + ihdr.writeUInt32BE(2, 0); + ihdr.writeUInt32BE(2, 4); + ihdr[8] = 8; + ihdr[9] = 2; + const row = Buffer.concat([Buffer.from([0]), Buffer.from([...rgb, ...rgb])]); + return new Uint8Array( + Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + chunk("IHDR", ihdr), + chunk("IDAT", deflateSync(Buffer.concat([row, row]))), + chunk("IEND", Buffer.alloc(0)), + ]), + ); +} + +const RED = png([255, 0, 0]); +const BLUE = png([0, 0, 255]); + +function body( + answers: Record, + extra: { model?: string; input_tokens?: number } = {}, +) { + return { + model: extra.model ?? "drex-max", + answers, + usage: { + input_tokens: extra.input_tokens ?? 120, + image_tokens: 0, + output_tokens: 0, + }, + evaluation_time_ms: 41.5, + request_id: "req_test123", + }; +} + +const NOUL = { type: "noul", noul: 0.9 }; +const CHOICE = { + type: "choice", + choice: "good", + confidence: 0.8, + probabilities: { good: 0.7, ok: 0.2, bad: 0.1 }, +}; +const SCORE = { + type: "score", + score: 3, + confidence: 0.7, + legend: ["awful", "poor", "fine", "great", "perfect"], + probabilities: { "2": 0.2, "3": 0.6, "4": 0.2 }, +}; + +const YES_NO = drexQuestion.noul("Is the output correct?"); +const CHOICES = drexQuestion.choice( + { good: "Fully right", ok: null, bad: "Wrong" }, + { scores: { good: 1, ok: 0.5, bad: 0 } }, +); +const LEVELS: [string, ...string[]] = [ + "awful", + "poor", + "fine", + "great", + "perfect", +]; +const RUBRIC = drexQuestion.score(LEVELS, { instructions: "Rate the answer." }); + +const server = setupServer(); +let requests: { body: any; headers: Headers }[] = []; + +beforeAll(() => + server.listen({ + onUnhandledRequest: (req) => { + throw new Error(`Unhandled request ${req.method}, ${req.url}`); + }, + }), +); +afterEach(() => { + server.resetHandlers(); + requests = []; +}); +afterAll(() => server.close()); + +function respond( + json: unknown, + status = 200, + headers: Record = {}, + url = URL_, +) { + server.use( + http.post(url, async ({ request }) => { + requests.push({ body: await request.json(), headers: request.headers }); + return HttpResponse.json(json as any, { status, headers }); + }), + ); +} + +function respondSequence( + responses: { + status: number; + json: unknown; + headers?: Record; + }[], +) { + let calls = 0; + server.use( + http.post(URL_, async ({ request }) => { + requests.push({ body: await request.json(), headers: request.headers }); + const next = responses[Math.min(calls++, responses.length - 1)]; + return HttpResponse.json(next.json as any, { + status: next.status, + headers: next.headers, + }); + }), + ); +} + +const auth = { apiKey: "nace_sk_test", baseUrl: BASE, maxRetries: 2 }; +const score = async (args: Record) => + DrexClassifier({ ...auth, ...args } as any); + +describe("DrexClassifier", () => { + for (const model of ["drex-max", "drex-v2.0"]) { + test(`noul, choice and score on ${model}`, async () => { + respond(body({ DrexClassifier: NOUL }, { model })); + let result = await score({ question: YES_NO, output: "Paris", model }); + expect(result.score).toBeCloseTo(0.9); + expect(requests[0].body.model).toBe(model); + expect((result.metadata as any).probabilities.yes).toBe(0.9); + expect((result.metadata as any).request_id).toBe("req_test123"); + expect((result.metadata as any).model).toBe(model); + + respond(body({ DrexClassifier: CHOICE }, { model })); + result = await score({ question: CHOICES, output: "Paris", model }); + expect(result.score).toBeCloseTo(0.8); // 0.7 * 1 + 0.2 * 0.5 + 0.1 * 0 + expect((result.metadata as any).choice).toBe("good"); + expect((result.metadata as any).probabilities).toEqual({ + good: 0.7, + ok: 0.2, + bad: 0.1, + }); + + respond(body({ DrexClassifier: SCORE }, { model })); + result = await score({ question: RUBRIC, output: "Paris", model }); + expect(result.score).toBeCloseTo(0.75); // 3 / 4 + expect((result.metadata as any).raw_score).toBe(3); + expect((result.metadata as any).scale).toEqual([0, 4]); + expect((result.metadata as any).legend[3]).toBe("great"); + expect((result.metadata as any).probabilities["3"]).toBe(0.6); + }); + } + + test("defaults to drex-max and passes unknown model ids through", async () => { + respond(body({ DrexClassifier: NOUL })); + await score({ question: YES_NO, output: "x" }); + expect(requests[0].body.model).toBe("drex-max"); + await score({ question: YES_NO, output: "x", model: "drex-v9" }); + expect(requests[1].body.model).toBe("drex-v9"); + }); + + test("a mixed batch is one request and counts usage once", async () => { + respond( + body( + { correct: NOUL, quality: CHOICE, depth: SCORE }, + { input_tokens: 300 }, + ), + ); + const scores = await DrexClassifierMany({ + ...auth, + output: "Paris", + input: "Capital of France?", + questions: { correct: YES_NO, quality: CHOICES, depth: RUBRIC }, + }); + expect(requests).toHaveLength(1); + expect(scores.map((s) => s.name)).toEqual(["correct", "quality", "depth"]); + expect(Object.keys(requests[0].body.questions)).toEqual([ + "correct", + "quality", + "depth", + ]); + expect((scores[0].metadata as any).usage.input_tokens).toBe(300); + expect((scores[1].metadata as any).usage).toBeNull(); + expect((scores[2].metadata as any).usage).toBeNull(); + expect((scores[1].metadata as any).usage_counted_on).toBe("correct"); + }); + + test("a criteria-only noul sends empty instructions", async () => { + respond(body({ DrexClassifier: NOUL })); + await score({ + question: drexQuestion.noul(undefined, { + true: "grounded", + false: "invented", + }), + output: "x", + }); + const question = requests[0].body.questions.DrexClassifier; + expect(question.instructions).toBe(""); + expect(question.criteria).toEqual({ true: "grounded", false: "invented" }); + }); + + test("a choice option with a null description is kept", async () => { + respond(body({ DrexClassifier: CHOICE })); + await score({ question: CHOICES, output: "x" }); + expect(requests[0].body.questions.DrexClassifier.criteria.ok).toBeNull(); + }); + + test("state is built from the case fields", async () => { + respond(body({ DrexClassifier: NOUL })); + await score({ + question: YES_NO, + output: "answer", + expected: "reference", + input: "question", + context: "docs", + }); + expect(requests[0].body.state).toEqual({ + input: "question", + output: "answer", + expected: "reference", + context: "docs", + }); + await score({ + question: YES_NO, + output: "answer", + expected: "reference", + context: "docs", + stateFields: ["output", "context"], + }); + expect(requests[1].body.state).toEqual({ + output: "answer", + context: "docs", + }); + await score({ question: YES_NO, output: "answer" }); + expect(requests[2].body.state).toBe("answer"); + expect(buildState({ output: { a: 1 } })).toEqual({ output: { a: 1 } }); + }); + + test("direction, reverse and threshold", async () => { + respond(body({ DrexClassifier: NOUL })); + const defect = drexQuestion.noul("Does the output contain a defect?", { + yesIs: "fail", + }); + let result = await score({ question: defect, output: "x" }); + expect(result.score).toBeCloseTo(0.1); + expect((result.metadata as any).yes_is).toBe("fail"); + expect((result.metadata as any).probabilities.yes).toBe(0.9); // the answer itself is not flipped + + respond(body({ DrexClassifier: SCORE })); + result = await score({ + question: drexQuestion.score(LEVELS, { reverse: true }), + output: "x", + }); + expect(result.score).toBeCloseTo(0.25); + + respond(body({ DrexClassifier: NOUL })); + result = await score({ question: YES_NO, output: "x", threshold: 0.95 }); + expect(result.score).toBe(0); + expect((result.metadata as any).raw_value).toBeCloseTo(0.9); + expect( + (await score({ question: YES_NO, output: "x", threshold: 0.5 })).score, + ).toBe(1); + }); + + test("attribution headers", async () => { + respond(body({ DrexClassifier: NOUL })); + await score({ question: YES_NO, output: "x" }); + const { headers } = requests[0]; + expect(headers.get("authorization")).toBe("Bearer nace_sk_test"); + expect(headers.get("x-nace-client")).toBe(DREX_CLIENT_NAME); + expect(headers.get("user-agent")).toBe(DREX_CLIENT_NAME); + }); + + test("one image, two images and reversed order", async () => { + respond(body({ DrexClassifier: NOUL })); + const q = drexQuestion.noul("Is Image 1 red?"); + await score({ question: q, output: "Look.", images: [RED] }); + expect(requests[0].body.images).toHaveLength(1); + await score({ question: q, output: "Look.", images: [RED, BLUE] }); + const forward = requests[1].body.images; + await score({ question: q, output: "Look.", images: [BLUE, RED] }); + const reverse = requests[2].body.images; + expect(forward).toHaveLength(2); + expect(forward).toEqual([...reverse].reverse()); + expect(forward[0].startsWith("data:image/png;base64,")).toBe(true); + expect( + Buffer.from(forward[0].split(",")[1], "base64").equals(Buffer.from(RED)), + ).toBe(true); + }); + + test("images on a text-only model are refused before sending", async () => { + respond(body({ DrexClassifier: NOUL })); + for (const model of [ + "drex-v2.0", + "drex-v1.5", + "drex-latest", + "nace-ai/drex-v2.0", + ]) { + await expect( + score({ question: YES_NO, output: "x", images: [RED], model }), + ).rejects.toThrow(/reads text only/); + } + expect(requests).toHaveLength(0); + expect(isTextOnly("drex-v2.0")).toBe(true); + expect(isTextOnly("drex-max")).toBe(false); + }); + + test("remote image URLs are refused", async () => { + await expect( + score({ + question: YES_NO, + output: "x", + images: ["https://example.com/a.png"], + }), + ).rejects.toThrow(/doesn't fetch image URLs/); + }); + + test("a workspace_file reference passes through", async () => { + respond(body({ DrexClassifier: NOUL })); + const ref = { + type: "workspace_file", + workspace_id: "ws_1", + file_id: "file_1", + }; + await score({ question: YES_NO, output: "x", images: [ref] }); + expect(requests[0].body.images).toEqual([ref]); + }); + + test("an uploaded file becomes a workspace_file reference", async () => { + respond(body({ DrexClassifier: NOUL })); + await score({ + question: YES_NO, + output: "x", + images: [{ workspace_id: "ws_1", file_id: "file_1" }], + }); + expect(requests[0].body.images).toEqual([ + expect.objectContaining({ + type: "workspace_file", + workspace_id: "ws_1", + file_id: "file_1", + }), + ]); + }); + + test("more than 16 questions on Max surfaces Drex's 422", async () => { + respond( + { + error: { + type: "invalid_request_error", + message: "invalid request", + issues: [ + { + path: "questions", + message: "at most 16 questions allowed for drex-max", + }, + ], + }, + }, + 422, + ); + const questions = Object.fromEntries( + Array.from({ length: 17 }, (_, i) => [ + `q${i}`, + drexQuestion.noul(`Question ${i}?`), + ]), + ); + const error = await DrexClassifierMany({ + ...auth, + output: "x", + questions, + }).catch((e) => e); + expect(error).toBeInstanceOf(DrexError); + expect(error.status).toBe(422); + expect(error.message).toContain( + "questions: at most 16 questions allowed for drex-max", + ); + }); + + test("401 and 402 messages", async () => { + respond( + { error: { type: "authentication_error", message: "Invalid API key" } }, + 401, + ); + let error = await score({ question: YES_NO, output: "x" }).catch((e) => e); + expect(error).toBeInstanceOf(DrexError); + expect(error.status).toBe(401); + + respond( + { error: { type: "insufficient_credit", message: "No available funds" } }, + 402, + ); + error = await score({ question: YES_NO, output: "x" }).catch((e) => e); + expect(error.message).toContain(`${BASE}/dashboard/billing`); + }); + + for (const status of [429, 529]) { + test(`${status} is retried after retry-after-ms`, async () => { + respondSequence([ + { + status, + headers: { "retry-after-ms": "5" }, + json: { error: { type: "rate_limit_error", message: "slow down" } }, + }, + { status: 200, json: body({ DrexClassifier: NOUL }) }, + ]); + const started = Date.now(); + const result = await score({ question: YES_NO, output: "x" }); + expect(requests).toHaveLength(2); + expect(result.score).toBeCloseTo(0.9); + // The header's 5 ms, not the SDK's own backoff (at least 375 ms). + expect(Date.now() - started).toBeLessThan(300); + }); + } + + test("a huge retry-after-ms is not slept: the error is raised at once", async () => { + respondSequence([ + { + status: 529, + headers: { "retry-after-ms": "86400000" }, + json: { error: { type: "overloaded_error", message: "busy" } }, + }, + ]); + const started = Date.now(); + const error = await score({ + question: YES_NO, + output: "x", + maxRetries: 1, + }).catch((e) => e); + expect(error).toBeInstanceOf(DrexError); + expect(error.status).toBe(529); + expect(requests).toHaveLength(1); + expect(Date.now() - started).toBeLessThan(1000); + }); + + test("exhausted retries are an error, not a score", async () => { + respondSequence([ + { + status: 529, + headers: { "retry-after-ms": "1" }, + json: { error: { type: "overloaded_error", message: "busy" } }, + }, + ]); + const error = await score({ + question: YES_NO, + output: "x", + maxRetries: 1, + }).catch((e) => e); + expect(error).toBeInstanceOf(DrexError); + expect(error.status).toBe(529); + expect(requests).toHaveLength(2); + }); + + test("OpenRouter payload and headers", async () => { + respond( + body({ DrexClassifier: NOUL }), + 200, + {}, + "https://openrouter.ai/api/v1/systemone", + ); + await score({ + question: drexQuestion.noul(undefined, { true: "red" }), + output: "Is Image 1 red?", + images: [RED], + model: "nace-ai/drex-max", + baseUrl: "https://openrouter.ai/api/v1", + }); + const { body: payload, headers } = requests[0]; + expect(payload.images).toBeUndefined(); + expect(payload.state[0]).toEqual({ type: "text", text: "Is Image 1 red?" }); + expect(payload.state[1].type).toBe("image_url"); + expect( + payload.state[1].image_url.url.startsWith("data:image/png;base64,"), + ).toBe(true); + expect(payload.questions.DrexClassifier.instructions).toBe(""); + expect(headers.get("http-referer")).toBe("https://nace.ai"); + expect(headers.get("x-openrouter-title")).toBe("Drex for Autoevals"); + expect(headers.get("x-nace-client")).toBe(DREX_CLIENT_NAME); + }); + + test("OpenRouter refuses workspace files and links its own credits", async () => { + const baseUrl = "https://openrouter.ai/api/v1"; + const model = "nace-ai/drex-max"; + await expect( + score({ + question: YES_NO, + output: "x", + baseUrl, + model, + images: [ + { type: "workspace_file", workspace_id: "ws_1", file_id: "f" }, + ], + }), + ).rejects.toThrow(/inline images only/); + respond( + { error: { type: "insufficient_credit", message: "No credits" } }, + 402, + {}, + "https://openrouter.ai/api/v1/systemone", + ); + const error = await score({ + question: YES_NO, + output: "x", + baseUrl, + model, + }).catch((e) => e); + expect(error.message).toContain("openrouter.ai/settings/credits"); + expect(error.message).not.toContain("/dashboard/billing"); + }); + + test("inline base64 images get their MIME type on OpenRouter", async () => { + respond( + body({ DrexClassifier: NOUL }), + 200, + {}, + "https://openrouter.ai/api/v1/systemone", + ); + const b64 = (bytes: number[]) => + Buffer.from([...bytes, ...new Array(20).fill(0)]).toString("base64"); + await score({ + question: YES_NO, + output: "x", + baseUrl: "https://openrouter.ai/api/v1", + model: "nace-ai/drex-max", + images: [ + b64([0xff, 0xd8, 0xff, 0xe0]), + b64([0x52, 0x49, 0x46, 0x46, 0, 0, 0, 0, 0x57, 0x45, 0x42, 0x50]), + b64([0, 0, 0, 0]), + ], + }); + const urls = requests[0].body.state + .slice(1) + .map((part: any) => part.image_url.url.split(";")[0]); + expect(urls).toEqual([ + "data:image/jpeg", + "data:image/webp", + "data:application/octet-stream", + ]); + }); + + test("a timeout is an error, not a score", async () => { + server.use( + http.post(URL_, async () => { + await delay(300); + return HttpResponse.json(body({ DrexClassifier: NOUL })); + }), + ); + const error = await score({ + question: YES_NO, + output: "x", + timeoutMs: 50, + maxRetries: 0, + }).catch((e) => e); + expect(error).toBeInstanceOf(DrexError); + expect(error.status).toBeUndefined(); + }); + + test("OpenRouter is detected from NACE_BASE_URL", async () => { + respond( + body({ DrexClassifier: NOUL }), + 200, + {}, + "https://openrouter.ai/api/v1/systemone", + ); + const saved = process.env.NACE_BASE_URL; + process.env.NACE_BASE_URL = "https://openrouter.ai/api/v1"; + try { + await score({ + question: YES_NO, + output: "x", + baseUrl: undefined, + apiKey: "nace_sk_test", + }); + } finally { + if (saved === undefined) delete process.env.NACE_BASE_URL; + else process.env.NACE_BASE_URL = saved; + } + expect(requests[0].headers.get("x-openrouter-title")).toBe( + "Drex for Autoevals", + ); + }); + + test("an empty output string is scored, not rejected", async () => { + respond(body({ DrexClassifier: NOUL })); + await score({ question: YES_NO, output: "" }); + expect(requests[0].body.state).toBe(""); + await expect( + score({ question: YES_NO, output: "x", stateFields: ["input"] }), + ).rejects.toThrow(/Nothing to evaluate/); + }); + + describe("API key and secrets follow the host", () => { + const OR = "https://openrouter.ai/api/v1"; + const OR_URL = `${OR}/systemone`; + const names = [ + "NACE_API_KEY", + "DREX_API_KEY", + "OPENROUTER_API_KEY", + "NACE_BASE_URL", + "DREX_BASE_URL", + "NACE_VERCEL_PROTECTION_BYPASS", + ]; + const saved: Record = {}; + beforeEach(() => { + for (const n of names) { + saved[n] = process.env[n]; + delete process.env[n]; + } + }); + afterEach(() => { + for (const n of names) { + if (saved[n] === undefined) delete process.env[n]; + else process.env[n] = saved[n]; + } + }); + const bare = async (args: Record) => + DrexClassifier({ question: YES_NO, output: "x", ...args } as any); + + test("OpenRouter uses apiKey or OPENROUTER_API_KEY, never NACE_API_KEY", async () => { + respond(body({ DrexClassifier: NOUL }), 200, {}, OR_URL); + process.env.NACE_API_KEY = "nace_sk_must_not_leave"; + await expect(bare({ baseUrl: OR })).rejects.toThrow(/OPENROUTER_API_KEY/); + expect(requests).toHaveLength(0); + process.env.OPENROUTER_API_KEY = "or_env_key"; + await bare({ baseUrl: OR }); + expect(requests[0].headers.get("authorization")).toBe( + "Bearer or_env_key", + ); + await bare({ baseUrl: OR, apiKey: "or_explicit" }); + expect(requests[1].headers.get("authorization")).toBe( + "Bearer or_explicit", + ); + }); + + test("a Nace host never uses OPENROUTER_API_KEY", async () => { + respond(body({ DrexClassifier: NOUL })); + process.env.OPENROUTER_API_KEY = "or_must_not_leave"; + await expect(bare({ baseUrl: BASE })).rejects.toThrow(/API key/); + expect(requests).toHaveLength(0); + process.env.NACE_API_KEY = "nace_sk_env"; + await bare({ baseUrl: BASE }); + expect(requests[0].headers.get("authorization")).toBe( + "Bearer nace_sk_env", + ); + }); + + test("the Vercel bypass secret is never forwarded", async () => { + respond(body({ DrexClassifier: NOUL }), 200, {}, OR_URL); + respond(body({ DrexClassifier: NOUL })); + process.env.NACE_VERCEL_PROTECTION_BYPASS = "bypass-secret"; + await bare({ baseUrl: OR, apiKey: "or_key" }); + await bare({ baseUrl: BASE, apiKey: "nace_sk_test" }); + expect(requests).toHaveLength(2); + for (const request of requests) { + expect(request.headers.has("x-vercel-protection-bypass")).toBe(false); + } + }); + + test("an on-prem host uses the Nace key and plain Nace shaping", async () => { + const onPrem = "https://drex.customer.example"; + process.env.NACE_API_KEY = "nace_sk_onprem"; + process.env.OPENROUTER_API_KEY = "or_must_not_leave"; + respond( + body({ DrexClassifier: NOUL }), + 200, + {}, + `${onPrem}/v1/systemone`, + ); + await bare({ baseUrl: `${onPrem}/`, images: [RED] }); + const { body: payload, headers } = requests[0]; + expect(headers.get("authorization")).toBe("Bearer nace_sk_onprem"); + expect(payload.model).toBe("drex-max"); + expect(payload.images).toHaveLength(1); + expect(headers.has("x-openrouter-title")).toBe(false); + expect(headers.has("http-referer")).toBe(false); + expect(headers.get("x-nace-client")).toBe(DREX_CLIENT_NAME); + // NACE_BASE_URL alone reaches the same host, and an explicit apiKey wins over the environment. + process.env.NACE_BASE_URL = onPrem; + await bare({ apiKey: "nace_sk_explicit" }); + expect(requests[1].headers.get("authorization")).toBe( + "Bearer nace_sk_explicit", + ); + }); + + test("the billing link follows the configured host", async () => { + const onPrem = "https://drex.customer.example"; + const insufficient = { + error: { type: "insufficient_credit", message: "No available funds" }, + }; + respond(insufficient, 402, {}, `${onPrem}/v1/systemone`); + let error = await bare({ + baseUrl: onPrem, + apiKey: "nace_sk_onprem", + }).catch((e) => e); + expect(error.message).toContain(`${onPrem}/dashboard/billing`); + // With no base URL the link is Nace's console. + respond(insufficient, 402, {}, "https://console.nace.ai/v1/systemone"); + error = await bare({ apiKey: "nace_sk_test" }).catch((e) => e); + expect(error.message).toContain( + "https://console.nace.ai/dashboard/billing", + ); + }); + + test("the OpenRouter host match is exact", async () => { + respond( + body({ DrexClassifier: NOUL }), + 200, + {}, + "https://notopenrouter.ai/v1/systemone", + ); + await bare({ baseUrl: "https://notopenrouter.ai", apiKey: "k" }); + expect(requests[0].headers.has("x-openrouter-title")).toBe(false); + respond( + body({ DrexClassifier: NOUL }), + 200, + {}, + "https://api.openrouter.ai/v1/systemone", + ); + await bare({ baseUrl: "https://api.openrouter.ai/v1", apiKey: "k" }); + respond( + body({ DrexClassifier: NOUL }), + 200, + {}, + "https://openrouter.ai./api/v1/systemone", + ); + await bare({ baseUrl: "https://openrouter.ai./api/v1", apiKey: "k" }); + expect(requests[2].headers.get("x-openrouter-title")).toBe( + "Drex for Autoevals", + ); + expect(requests[1].headers.get("x-openrouter-title")).toBe( + "Drex for Autoevals", + ); + }); + }); + + test("OpenRouter's default model is namespaced and bare ids are refused", async () => { + const baseUrl = "https://openrouter.ai/api/v1"; + respond( + body({ DrexClassifier: NOUL }), + 200, + {}, + "https://openrouter.ai/api/v1/systemone", + ); + await score({ question: YES_NO, output: "x", baseUrl, apiKey: "or_key" }); + expect(requests[0].body.model).toBe("nace-ai/drex-max"); + await score({ + question: YES_NO, + output: "x", + baseUrl, + apiKey: "or_key", + model: "nace-ai/drex-v2.0", + }); + expect(requests[1].body.model).toBe("nace-ai/drex-v2.0"); + for (const model of ["drex-max", "drex-v2.0", "drex-latest"]) { + await expect( + score({ + question: YES_NO, + output: "x", + baseUrl, + apiKey: "or_key", + model, + }), + ).rejects.toThrow(/namespaced/); + } + expect(requests).toHaveLength(2); + // Nace direct keeps the bare default. + respond(body({ DrexClassifier: NOUL })); + await score({ question: YES_NO, output: "x" }); + expect(requests[2].body.model).toBe("drex-max"); + }); + + test("drex-latest stays text-only on OpenRouter", async () => { + await expect( + score({ + question: YES_NO, + output: "x", + baseUrl: "https://openrouter.ai/api/v1", + apiKey: "or_key", + model: "nace-ai/drex-latest", + images: [RED], + }), + ).rejects.toThrow(/reads text only/); + }); + + test("a response without a request id is still scored", async () => { + const { request_id: _omitted, ...payload } = body({ + DrexClassifier: NOUL, + }); + respond(payload); + const result = await score({ question: YES_NO, output: "x" }); + expect(result.score).toBeCloseTo(0.9); + expect((result.metadata as any).request_id).toBeUndefined(); + }); + + test("the request body and headers follow the Drex contract", async () => { + respond(body({ DrexClassifier: CHOICE })); + await score({ question: CHOICES, output: "Paris", input: "Capital?" }); + const { body: payload, headers } = requests[0]; + expect(payload).toEqual({ + state: { input: "Capital?", output: "Paris" }, + questions: { + DrexClassifier: { + type: "choice", + instructions: "", + criteria: { good: "Fully right", ok: null, bad: "Wrong" }, + }, + }, + model: "drex-max", + }); + expect(headers.get("content-type")).toBe("application/json"); + expect(headers.get("accept")).toBe("application/json"); + expect(headers.get("authorization")).toBe("Bearer nace_sk_test"); + expect(headers.get("x-nace-client")).toBe(DREX_CLIENT_NAME); + expect(headers.get("user-agent")).toBe(DREX_CLIENT_NAME); + }); + + test("the request id comes from the header, else the body, and reaches errors", async () => { + respond(body({ DrexClassifier: NOUL }), 200, { "x-request-id": "req_h" }); + let result = await score({ question: YES_NO, output: "x" }); + expect((result.metadata as any).request_id).toBe("req_h"); + respond(body({ DrexClassifier: NOUL })); + result = await score({ question: YES_NO, output: "x" }); + expect((result.metadata as any).request_id).toBe("req_test123"); + respond({ error: { type: "authentication_error", message: "no" } }, 401, { + "x-request-id": "req_401", + }); + const error = await score({ question: YES_NO, output: "x" }).catch( + (e) => e, + ); + expect(error.requestId).toBe("req_401"); + expect(error.message).toContain("req_401"); + }); + + test("authentication errors and a server veto are not retried", async () => { + respondSequence([ + { + status: 401, + json: { error: { type: "authentication_error", message: "no" } }, + }, + ]); + await expect(score({ question: YES_NO, output: "x" })).rejects.toThrow( + /401/, + ); + expect(requests).toHaveLength(1); + respondSequence([ + { + status: 503, + json: { error: { message: "stop", retryable: false } }, + }, + ]); + await expect(score({ question: YES_NO, output: "x" })).rejects.toThrow( + /503/, + ); + expect(requests).toHaveLength(2); + }); + + test("retry-after seconds and dates are honoured, retry-after-ms wins, backoff is the fallback", async () => { + const busy = { error: { type: "overloaded_error", message: "busy" } }; + const ok = { status: 200, json: body({ DrexClassifier: NOUL }) }; + const elapsed = async (first: Record, status = 429) => { + requests = []; + respondSequence([{ status, headers: first, json: busy }, ok]); + const started = Date.now(); + await score({ question: YES_NO, output: "x" }); + expect(requests).toHaveLength(2); + return Date.now() - started; + }; + expect(await elapsed({ "retry-after": "0" })).toBeLessThan(300); + expect( + await elapsed({ + "retry-after": new Date(Date.now() - 5000).toUTCString(), + }), + ).toBeLessThan(300); + expect( + await elapsed({ "retry-after-ms": "5", "retry-after": "30" }), + ).toBeLessThan(300); + // No usable header: 500 ms backoff, less up to 25% jitter. + expect(await elapsed({}, 500)).toBeGreaterThanOrEqual(300); + }); + + test("connection errors are retried, then reported", async () => { + let calls = 0; + server.use( + http.post(URL_, async ({ request }) => { + requests.push({ body: await request.json(), headers: request.headers }); + if (calls++ === 0) return HttpResponse.error(); + return HttpResponse.json(body({ DrexClassifier: NOUL })); + }), + ); + const result = await score({ question: YES_NO, output: "x" }); + expect(result.score).toBeCloseTo(0.9); + expect(requests).toHaveLength(2); + server.use(http.post(URL_, () => HttpResponse.error())); + const error = await score({ + question: YES_NO, + output: "x", + maxRetries: 0, + }).catch((e) => e); + expect(error).toBeInstanceOf(DrexError); + expect(error.message).toMatch(/Connection error/); + expect(error.status).toBeUndefined(); + }); + + test("a timeout is retried, and the default waits at least 60 seconds", async () => { + let aborts = 0; + server.use( + http.post(URL_, async () => { + aborts++; + await delay(200); + return HttpResponse.json(body({ DrexClassifier: NOUL })); + }), + ); + const error = await score({ + question: YES_NO, + output: "x", + timeoutMs: 30, + maxRetries: 1, + }).catch((e) => e); + expect(error.message).toMatch(/timed out/); + expect(aborts).toBe(2); + const spy = vi.spyOn(globalThis, "setTimeout"); + respond(body({ DrexClassifier: NOUL })); + await score({ question: YES_NO, output: "x" }); + expect(spy.mock.calls.some(([, ms]) => ms === 60_000)).toBe(true); + spy.mockRestore(); + }); + + test("invalid timeout and retry settings are refused", async () => { + for (const timeoutMs of [0, -1, Infinity, NaN]) { + await expect( + score({ question: YES_NO, output: "x", timeoutMs }), + ).rejects.toThrow(/timeoutMs/); + } + await expect( + score({ question: YES_NO, output: "x", maxRetries: -1 }), + ).rejects.toThrow(/maxRetries/); + }); + + test("a request over 4 MiB is refused before sending", async () => { + respond(body({ DrexClassifier: NOUL })); + await expect( + score({ + question: YES_NO, + output: "x".repeat(4 * 1024 * 1024 + 1), + }), + ).rejects.toThrow(/4 MiB/); + expect(requests).toHaveLength(0); + await score({ question: YES_NO, output: "y".repeat(3 * 1024 * 1024) }); + expect(requests).toHaveLength(1); + }); + + test("422 issues are listed as path and message", async () => { + respond( + { + error: { + type: "invalid_request_error", + message: "invalid request", + issues: [ + { path: "questions.q1.criteria", message: "must not be empty" }, + { path: "images", message: "at most 8 images" }, + ], + }, + }, + 422, + ); + const error = await score({ question: YES_NO, output: "x" }).catch( + (e) => e, + ); + expect(error.message).toContain("422 invalid request"); + expect(error.message).toContain( + "\n questions.q1.criteria: must not be empty", + ); + expect(error.message).toContain("\n images: at most 8 images"); + }); + + test("error bodies that are not Drex JSON still give a message", async () => { + server.use( + http.post( + URL_, + () => new HttpResponse("bad gateway", { status: 502 }), + ), + ); + let error = await score({ + question: YES_NO, + output: "x", + maxRetries: 0, + }).catch((e) => e); + expect(error.message).toContain("502 bad gateway"); + server.use(http.post(URL_, () => new HttpResponse(null, { status: 500 }))); + error = await score({ question: YES_NO, output: "x", maxRetries: 0 }).catch( + (e) => e, + ); + expect(error.message).toContain("500 status code (no body)"); + }); + + test("malformed and missing answers are errors", async () => { + respond({ model: "drex-max", usage: {} }); + await expect(score({ question: YES_NO, output: "x" })).rejects.toThrow( + /unexpected response/, + ); + respond(body({ DrexClassifier: { type: "noul" } })); + await expect(score({ question: YES_NO, output: "x" })).rejects.toThrow( + /malformed answer/, + ); + respond(body({ DrexClassifier: { type: "noul", noul: "high" } })); + await expect(score({ question: YES_NO, output: "x" })).rejects.toThrow( + /malformed answer/, + ); + // An answer of another type, or one this client does not know, is no answer to the question asked. + respond(body({ DrexClassifier: CHOICE })); + await expect(score({ question: YES_NO, output: "x" })).rejects.toThrow( + /no answer/, + ); + respond(body({ DrexClassifier: { type: "future", x: 1 } })); + await expect(score({ question: YES_NO, output: "x" })).rejects.toThrow( + /no answer/, + ); + }); + + test("usage fields are optional", async () => { + const { evaluation_time_ms: _t, ...payload } = body({ + DrexClassifier: NOUL, + }); + respond({ ...payload, usage: { input_tokens: 5 } }); + const result = await score({ question: YES_NO, output: "x" }); + const metadata = result.metadata as any; + expect(metadata.usage).toEqual({ + input_tokens: 5, + image_tokens: null, + output_tokens: null, + }); + expect(metadata.evaluation_time_ms).toBeNull(); + }); + + test("images must be inline and are never dropped", async () => { + respond(body({ DrexClassifier: NOUL })); + const run = (images: unknown[]) => + score({ question: YES_NO, output: "x", images }); + await expect( + run([new TextEncoder().encode("%PDF-1.7 no")]), + ).rejects.toThrow(/PNG, JPEG and WebP/); + await expect(run(["/tmp/not-base64.png"])).rejects.toThrow( + /neither base64 nor a data URL/, + ); + await expect(run([42])).rejects.toThrow(/images\[0\]/); + await expect( + run([RED, { type: "url", url: "https://x.test/a.png" }]), + ).rejects.toThrow(/images\[1\]/); + expect(requests).toHaveLength(0); + const b64 = Buffer.from(RED).toString("base64"); + await run([ + b64, + `data:image/png;base64,${b64}`, + BLUE, + { workspace_id: "ws_1", file_id: "file_1" }, + ]); + const images = requests[0].body.images; + expect(images[0]).toBe(b64); + expect(images[1]).toBe(`data:image/png;base64,${b64}`); + expect(Buffer.from(images[2].split(",")[1], "base64")).toEqual( + Buffer.from(BLUE), + ); + expect(images[3]).toEqual({ + type: "workspace_file", + workspace_id: "ws_1", + file_id: "file_1", + }); + }); + + test("a missing answer is an error", async () => { + respond(body({})); + await expect(score({ question: YES_NO, output: "x" })).rejects.toThrow( + /no answer/, + ); + }); + + test("noul criteria must be non-empty strings or null", async () => { + await expect( + score({ + question: drexQuestion.noul(undefined, { true: "" }), + output: "x", + }), + ).rejects.toThrow(/instructions or criteria/); + await expect( + score({ + question: { + type: "noul", + instructions: "Right?", + criteria: { true: 1 }, + }, + output: "x", + }), + ).rejects.toThrow(/strings or null/); + respond(body({ DrexClassifier: NOUL })); + await score({ + question: { + type: "noul", + instructions: "Right?", + criteria: { true: "yes", false: null }, + }, + output: "x", + }); + expect(requests).toHaveLength(1); + }); + + test("invalid questions fail before any request", async () => { + await expect( + score({ + question: drexQuestion.choice( + { a: null, b: null }, + { scores: { a: 1 } }, + ), + output: "x", + }), + ).rejects.toThrow(/cover exactly/); + await expect( + score({ question: drexQuestion.score(["only"]), output: "x" }), + ).rejects.toThrow(/at least two levels/); + await expect( + score({ question: drexQuestion.noul(), output: "x" }), + ).rejects.toThrow(/instructions or criteria/); + await expect( + score({ question: YES_NO, output: "x", threshold: 2 }), + ).rejects.toThrow(/threshold/); + expect(requests).toHaveLength(0); + }); + + test("works through .partial", async () => { + respond(body({ Grounded: NOUL })); + const grounded = DrexClassifier.partial({ + ...auth, + question: YES_NO, + name: "Grounded", + } as any); + const result = await grounded({ output: "x" } as any); + expect(result.name).toBe("Grounded"); + }); +}); + +// Live checks: need a real key. +describe.skipIf(!process.env.NACE_API_KEY)("live", () => { + // The offline tests have run by now. Stop msw so these calls reach the real Drex instead of failing + // as unhandled requests (closing restores the global fetch). + beforeAll(() => server.close()); + const live = { baseUrl: process.env.NACE_BASE_URL, apiKey: undefined }; + for (const model of ["drex-max", "drex-v2.0"]) { + test(`live text on ${model}`, async () => { + const question = drexQuestion.noul( + "Does the output state that Paris is the capital of France?", + ); + const good = await DrexClassifier({ + ...live, + question, + model, + output: "The capital of France is Paris.", + }); + const bad = await DrexClassifier({ + ...live, + question, + model, + output: "The capital of France is Berlin.", + }); + expect(good.score!).toBeGreaterThan(0.8); + expect(bad.score!).toBeLessThan(0.2); + }); + } + + test("live images on drex-max", async () => { + const question = drexQuestion.noul("Is Image 1 solid red?"); + const forward = await DrexClassifier({ + ...live, + question, + model: "drex-max", + output: "Look at the images.", + images: [RED, BLUE], + }); + const reverse = await DrexClassifier({ + ...live, + question, + model: "drex-max", + output: "Look at the images.", + images: [BLUE, RED], + }); + expect(forward.score!).toBeGreaterThan(0.8); + expect(reverse.score!).toBeLessThan(0.2); + }); +}); + +// Live OpenRouter checks: need OPENROUTER_API_KEY and never NACE_API_KEY. +describe.skipIf(!process.env.OPENROUTER_API_KEY)("live OpenRouter", () => { + const saved: Record = {}; + const names = [ + "NACE_API_KEY", + "DREX_API_KEY", + "NACE_BASE_URL", + "DREX_BASE_URL", + "NACE_VERCEL_PROTECTION_BYPASS", + ]; + beforeAll(() => { + server.close(); // reach the real OpenRouter, not msw + for (const n of names) { + saved[n] = process.env[n]; + delete process.env[n]; // prove the Nace key is neither needed nor sent + } + }); + afterAll(() => { + for (const n of names) { + if (saved[n] !== undefined) process.env[n] = saved[n]; + } + }); + const base = { baseUrl: "https://openrouter.ai/api/v1" }; + + for (const model of [undefined, "nace-ai/drex-v2.0", "nace-ai/drex-max"]) { + test(`live OpenRouter text on ${model ?? "the default model"}`, async () => { + const question = drexQuestion.noul( + "Does the output state that Paris is the capital of France?", + ); + const good = await DrexClassifier({ + ...base, + question, + model, + output: "The capital of France is Paris.", + }); + const bad = await DrexClassifier({ + ...base, + question, + model, + output: "The capital of France is Berlin.", + }); + expect(good.score!).toBeGreaterThan(0.8); + expect(bad.score!).toBeLessThan(0.2); + }); + } + + test("live OpenRouter inline image on nace-ai/drex-max", async () => { + const question = drexQuestion.noul("Is Image 1 solid red?"); + const args = { ...base, question, model: "nace-ai/drex-max" }; + const forward = await DrexClassifier({ + ...args, + output: "Look at the images.", + images: [RED, BLUE], + }); + const reverse = await DrexClassifier({ + ...args, + output: "Look at the images.", + images: [BLUE, RED], + }); + expect(forward.score!).toBeGreaterThan(0.8); + expect(reverse.score!).toBeLessThan(0.2); + }); +}); diff --git a/js/drex.ts b/js/drex.ts new file mode 100644 index 00000000..66963913 --- /dev/null +++ b/js/drex.ts @@ -0,0 +1,852 @@ +/** + * Drex decision-model scorer. + * + * `DrexClassifier` asks a Drex decision model (https://console.nace.ai/docs) one typed question about an + * evaluation case and turns the answer's probabilities into a `Score`. Drex answers with probabilities, + * not a verdict, so the score is the probability mass on the outcomes you call a pass, and the original + * probabilities stay in `metadata`. + * + * It calls `POST {baseUrl}/v1/systemone` with the global `fetch` and needs no extra package. Credentials + * come from `NACE_API_KEY` (and `NACE_BASE_URL`, which may name your own Drex deployment) unless passed + * explicitly. + * + * @module drex + */ + +import { makePartial, ScorerWithPartial } from "./partial"; +import { Score } from "./score"; +import { VERSION } from "./version"; + +export const DREX_CLIENT_NAME = `autoevals/${VERSION}`; +export const DREX_DEFAULT_MODEL = "drex-max"; +// OpenRouter namespaces model ids: `drex-max` there is `nace-ai/drex-max`, and a bare id does not exist. +const OPENROUTER_MODEL_PREFIX = "nace-ai/"; +// Released versions that read text only; Drex enforces every other limit itself. +const TEXT_ONLY_MODELS = new Set(["drex-v1.5", "drex-v2.0", "drex-latest"]); +// Drex holds a decision for up to 55 s, so the client must wait longer. +const DEFAULT_TIMEOUT_MS = 60_000; +const DEFAULT_MAX_RETRIES = 2; +const DEFAULT_BASE_URL = "https://console.nace.ai"; +const SYSTEM_ONE_PATH = "/v1/systemone"; +// A server-requested wait longer than this is not slept: the error is raised instead. +const MAX_RETRY_WAIT_MS = 60_000; +const BACKOFF_INITIAL_MS = 500; +const BACKOFF_MAX_MS = 5_000; +const BACKOFF_JITTER = 0.25; +const RETRY_STATUSES = (status: number) => + status === 408 || status === 429 || (status >= 500 && status <= 599); +// Drex refuses a larger request body; images go in inline, so the client says so before sending. +const MAX_BODY_BYTES = 4 * 1024 * 1024; +const MAX_IMAGE_BYTES = 16 * 1024 * 1024; +const STATE_FIELDS = ["input", "output", "expected", "context"] as const; +type StateField = (typeof STATE_FIELDS)[number]; + +/** A request to Drex failed. This is an error in the evaluation, never a low score. */ +export class DrexError extends Error { + readonly status?: number; + readonly requestId?: string; + constructor(message: string, status?: number, requestId?: string) { + super(message); + this.name = "DrexError"; + this.status = status; + this.requestId = requestId; + } +} + +/** Questions for Drex, with how each answer becomes a 0-1 score (higher is better). */ +export type DrexQuestion = + | { + type: "noul"; + instructions?: string | null; + /** Optional descriptions of the yes and no outcomes. */ + criteria?: { true?: string | null; false?: string | null } | null; + /** Whether a yes answer is good. With `"fail"` the score is the probability of no. Default `"pass"`. */ + yesIs?: "pass" | "fail"; + } + | { + type: "choice"; + instructions?: string | null; + /** Labels mapped to descriptions, or null for undescribed labels. */ + criteria: Record; + /** The score of each label, from 0 to 1. The score is the probability-weighted sum. */ + scores: Record; + } + | { + type: "score"; + instructions?: string | null; + /** An ordered rubric, lowest level first. */ + criteria: [string, ...string[]]; + /** Treat the lowest level as best: the score is one minus the normalized level. */ + reverse?: boolean; + }; + +/** Builders for `DrexQuestion`. */ +export const drexQuestion = { + noul( + instructions?: string, + options: { true?: string; false?: string; yesIs?: "pass" | "fail" } = {}, + ): DrexQuestion { + const criteria: { true?: string; false?: string } = {}; + if (options.true !== undefined) criteria.true = options.true; + if (options.false !== undefined) criteria.false = options.false; + return { + type: "noul", + instructions, + criteria: Object.keys(criteria).length ? criteria : undefined, + yesIs: options.yesIs ?? "pass", + }; + }, + choice( + choices: Record, + options: { scores: Record; instructions?: string }, + ): DrexQuestion { + return { + type: "choice", + instructions: options.instructions, + criteria: choices, + scores: options.scores, + }; + }, + score( + levels: [string, ...string[]], + options: { instructions?: string; reverse?: boolean } = {}, + ): DrexQuestion { + return { + type: "score", + instructions: options.instructions, + criteria: levels, + reverse: options.reverse ?? false, + }; + }, +}; + +/** An image for Drex Max: bytes, a base64 string or data URL, or an uploaded `workspace_file` reference. */ +export type DrexImage = Uint8Array | string | Record; + +export interface DrexAuth { + /** + * Defaults to `NACE_API_KEY`. For OpenRouter it defaults to `OPENROUTER_API_KEY` instead and never to + * `NACE_API_KEY`. + */ + apiKey?: string; + /** + * Defaults to `NACE_BASE_URL`, then Nace direct. Use `https://openrouter.ai/api/v1` for OpenRouter + * (model ids `nace-ai/drex-max`). + */ + baseUrl?: string; + /** Milliseconds per attempt. At least 60000 is advised. */ + timeoutMs?: number; + /** + * Retries on 408, 429 and 5xx (529 included) and on connection errors, honouring `retry-after-ms`, then + * `retry-after`. A requested wait over 60 s is not slept: the error is raised. + */ + maxRetries?: number; +} + +export type DrexArgs = DrexAuth & { + /** The question and how to score its answer. */ + question: DrexQuestion; + /** The score's name. Defaults to `DrexClassifier`. */ + name?: string; + /** A Drex model id, passed through unchanged. Defaults to `drex-max` (`nace-ai/drex-max` on OpenRouter, where a bare id is refused). */ + model?: string; + /** Images every question sees (`drex-max` only). */ + images?: DrexImage[]; + /** Which of `input`, `output`, `expected` and `context` Drex reads. */ + stateFields?: StateField[]; + /** When set, the score is 1 if the 0-1 score reaches it and 0 otherwise; the raw value stays in metadata. */ + threshold?: number; + input?: unknown; + context?: unknown; +}; + +export type DrexManyArgs = Omit & { + /** Questions to answer in one request, keyed by name. */ + questions: Record; + output: unknown; + expected?: unknown; +}; + +export function isTextOnly(model: string): boolean { + const id = model.split("/").pop()!.replace(/^~+/, ""); + return TEXT_ONLY_MODELS.has(id); +} + +function isOpenRouterHost(host: string): boolean { + const name = host.toLowerCase().replace(/\.+$/, ""); + return name === "openrouter.ai" || name.endsWith(".openrouter.ai"); +} + +function isOpenRouter(baseUrl?: string): boolean { + if (!baseUrl) return false; + try { + return isOpenRouterHost(new URL(baseUrl).hostname); + } catch { + return false; + } +} + +/** An environment value: blank counts as unset, then the legacy `DREX_*` name is tried. */ +function env(name: string, legacy: string): string | undefined { + return process.env[name]?.trim() || process.env[legacy]?.trim() || undefined; +} + +/** What Drex reads: `output` alone as text, otherwise an object of the case fields that are set. */ +export function buildState( + values: Partial>, + fields: readonly StateField[] = STATE_FIELDS, +): unknown { + const chosen: Record = {}; + for (const field of fields) { + const value = values[field]; + if (value !== undefined && value !== null) chosen[field] = value; + } + const keys = Object.keys(chosen); + if ( + keys.length === 1 && + keys[0] === "output" && + typeof chosen.output === "string" + ) { + return chosen.output; + } + return chosen; +} + +function validate(name: string, q: DrexQuestion): void { + if (q.type === "noul") { + if (q.yesIs !== undefined && q.yesIs !== "pass" && q.yesIs !== "fail") { + throw new Error(`${name}: yesIs must be 'pass' or 'fail'`); + } + const outcomes = Object.values(q.criteria ?? {}); + if ( + outcomes.some( + (v) => v !== null && v !== undefined && typeof v !== "string", + ) + ) { + throw new Error(`${name}: noul criteria must be strings or null`); + } + // Drex refuses a yes/no question with neither instructions nor a described outcome. + if (!q.instructions && !outcomes.some((v) => v)) { + throw new Error( + `${name}: a noul question needs instructions or criteria`, + ); + } + } else if (q.type === "choice") { + const labels = Object.keys(q.criteria ?? {}); + if (!labels.length) + throw new Error(`${name}: a choice question needs at least one label`); + const scored = Object.keys(q.scores ?? {}); + if ( + scored.length !== labels.length || + !labels.every((l) => scored.includes(l)) + ) { + throw new Error( + `${name}: choice scores must cover exactly the labels ${JSON.stringify([...labels].sort())}`, + ); + } + if (scored.some((l) => !(q.scores[l] >= 0 && q.scores[l] <= 1))) { + throw new Error(`${name}: choice scores must be between 0 and 1`); + } + } else if (q.type === "score") { + if (!q.criteria || q.criteria.length < 2) { + throw new Error(`${name}: a score question needs at least two levels`); + } + } else { + throw new Error( + `${name}: unknown question type ${JSON.stringify((q as any).type)}`, + ); + } +} + +/** The request form. `instructions` is always a string: OpenRouter requires it, Drex rejects null. */ +function wire(q: DrexQuestion): Record { + const question: Record = { + type: q.type, + instructions: q.instructions ?? "", + }; + if (q.criteria !== undefined && q.criteria !== null) + question.criteria = q.criteria; + return question; +} + +function dataUrl(image: string): string { + if (image.startsWith("data:")) return image; + const head = Buffer.from(image.slice(0, 24), "base64"); + let mime = "application/octet-stream"; + if ( + head + .subarray(0, 8) + .equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])) + ) { + mime = "image/png"; + } else if (head[0] === 0xff && head[1] === 0xd8 && head[2] === 0xff) { + mime = "image/jpeg"; + } else if ( + head.subarray(0, 4).toString("latin1") === "RIFF" && + head.subarray(8, 12).toString("latin1") === "WEBP" + ) { + mime = "image/webp"; + } + return `data:${mime};base64,${image}`; +} + +function mediaType(bytes: Uint8Array): string | undefined { + const starts = (...head: number[]) => head.every((b, i) => bytes[i] === b); + if (starts(0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a)) + return "image/png"; + if (starts(0xff, 0xd8, 0xff)) return "image/jpeg"; + if ( + starts(0x52, 0x49, 0x46, 0x46) && + bytes[8] === 0x57 && + bytes[9] === 0x45 && + bytes[10] === 0x42 && + bytes[11] === 0x50 + ) { + return "image/webp"; + } + return undefined; +} + +const BASE64 = /^[A-Za-z0-9+/]+={0,2}$/; + +/** + * One `images` entry as the request sends it: base64, a data URL, or a `workspace_file` reference. Drex + * does not fetch URLs or read paths, and images are never dropped: anything else is refused. + */ +function prepareImage(image: DrexImage, index: number): unknown { + if (typeof image === "string") { + if (/^https?:\/\//.test(image)) { + throw new Error( + "Drex doesn't fetch image URLs. Pass the image bytes, a base64 string or data URL, or a workspace_file reference.", + ); + } + if ( + !image.startsWith("data:") && + (image.length % 4 || !BASE64.test(image)) + ) { + throw new Error( + `images[${index}] is neither base64 nor a data URL; pass the image bytes instead.`, + ); + } + return image; + } + if (image instanceof Uint8Array) { + const type = mediaType(image); + if (!type) { + throw new Error( + "Drex Max reads PNG, JPEG and WebP images only; these bytes are none of them.", + ); + } + if (image.length > MAX_IMAGE_BYTES) { + throw new Error( + `The image is ${image.length} bytes, over Drex's 16 MiB per image.`, + ); + } + return `data:${type};base64,${Buffer.from(image).toString("base64")}`; + } + if (image && typeof image === "object" && !Array.isArray(image)) { + const { type, ...rest } = image as Record; + if ( + type === "workspace_file" || + (type === undefined && "file_id" in rest && "workspace_id" in rest) + ) { + return { type: "workspace_file", ...rest }; + } + } + throw new Error( + `images[${index}] must be bytes, a base64 string, a data URL or a workspace_file reference.`, + ); +} + +/** OpenRouter takes images as `image_url` parts of a `state` content array, not as `images`. */ +function openrouterState(state: unknown, images: unknown[]): unknown[] { + const parts: unknown[] = [ + { + type: "text", + text: typeof state === "string" ? state : JSON.stringify(state), + }, + ]; + for (const image of images) { + if (typeof image !== "string") { + throw new Error( + "OpenRouter accepts inline images only, not uploaded workspace files.", + ); + } + parts.push({ type: "image_url", image_url: { url: dataUrl(image) } }); + } + return parts; +} + +type Answer = any; + +function num(value: unknown): number { + if (typeof value !== "number") throw new TypeError("not a number"); + return value; +} + +/** Return the 0-1 score (higher is better) and the answer's own figures. */ +function normalize( + q: DrexQuestion, + answer: Answer, +): [number, Record] { + if (q.type === "noul") { + const pYes = num(answer.noul); + const yesIs = q.yesIs ?? "pass"; + return [ + yesIs === "pass" ? pYes : 1 - pYes, + { + probabilities: { yes: pYes, no: 1 - pYes }, + yes_is: yesIs, + // Computed here: older Drex deployments return no confidence on noul answers. + confidence: Math.max(pYes, 1 - pYes), + }, + ]; + } + if (q.type === "choice") { + const probabilities: Record = {}; + for (const [label, p] of Object.entries( + answer.probabilities as Record, + )) { + probabilities[label] = num(p); + } + let score = 0; + for (const [label, p] of Object.entries(probabilities)) + score += p * (q.scores[label] ?? 0); + return [ + score, + { + choice: answer.choice, + confidence: num(answer.confidence), + probabilities, + choice_scores: { ...q.scores }, + }, + ]; + } + const levels = q.criteria.length; + const probabilities: Record = {}; + for (const [level, p] of Object.entries( + answer.probabilities as Record, + )) { + probabilities[String(level)] = num(p); + } + const normalized = num(answer.score) / (levels - 1); + return [ + q.reverse ? 1 - normalized : normalized, + { + raw_score: num(answer.score), + scale: [0, levels - 1], + legend: [...answer.legend], + confidence: num(answer.confidence), + probabilities, + reverse: q.reverse ?? false, + }, + ]; +} + +const clamp = (value: number) => Math.min(1, Math.max(0, value)); + +class TransportError extends Error { + constructor( + message: string, + readonly timeout = false, + ) { + super(message); + } +} + +interface Reply { + status: number; + headers: Headers; + text: string; +} + +async function post( + url: string, + headers: Record, + body: string, + timeoutMs: number, +): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + const response = await fetch(url, { + method: "POST", + headers, + body, + signal: controller.signal, + }); + return { + status: response.status, + headers: response.headers, + text: await response.text(), + }; + } catch (error) { + if (controller.signal.aborted) { + throw new TransportError( + `Request timed out after ${timeoutMs / 1000}s`, + true, + ); + } + const cause = (error as { cause?: { code?: string } }).cause?.code; + throw new TransportError( + `Connection error: ${(error as Error).message}${cause ? ` (${cause})` : ""}`, + ); + } finally { + clearTimeout(timer); + } +} + +/** The server's requested wait in ms: `retry-after-ms`, else `retry-after` (seconds or an HTTP date). */ +function retryAfterMs(headers: Headers): number | undefined { + for (const [name, scale] of [ + ["retry-after-ms", 1], + ["retry-after", 1000], + ] as const) { + const raw = headers.get(name); + if (raw === null) continue; + const value = Number(raw.trim() || "0"); + if (Number.isFinite(value) && value >= 0) return value * scale; + if (name === "retry-after") { + const date = Date.parse(raw); + if (!Number.isNaN(date)) return Math.max(0, date - Date.now()); + } + } + return undefined; +} + +function parseJson(text: string): unknown { + if (!text) return undefined; + try { + return JSON.parse(text); + } catch { + return text; + } +} + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); + +/** Milliseconds to wait before retry number `attempt`, or undefined to stop and raise. */ +function retryDelayMs( + attempt: number, + maxRetries: number, + reply?: Reply, +): number | undefined { + if (attempt > maxRetries) return undefined; + if (reply) { + if (!RETRY_STATUSES(reply.status)) return undefined; + const body = parseJson(reply.text); + if ( + isRecord(body) && + isRecord(body.error) && + body.error.retryable === false + ) { + return undefined; + } + const wait = retryAfterMs(reply.headers); + if (wait !== undefined) return wait <= MAX_RETRY_WAIT_MS ? wait : undefined; + } + const backoff = Math.min( + BACKOFF_INITIAL_MS * 2 ** (attempt - 1), + BACKOFF_MAX_MS, + ); + return Math.round(backoff * (1 - Math.random() * BACKOFF_JITTER)); +} + +function errorMessage(body: unknown): string | undefined { + if (typeof body === "string") return body.slice(0, 200) || undefined; + if (!isRecord(body)) return undefined; + const { error, message, detail } = body; + if (typeof error === "string") return error; + if (isRecord(error) && typeof error.message === "string") + return error.message; + if (typeof message === "string") return message; + if (typeof detail === "string") return detail; + if (isRecord(detail) && typeof detail.message === "string") + return detail.message; + return undefined; +} + +function apiError( + reply: Reply, + baseUrl: string | undefined, + openrouter: boolean, +): DrexError { + const body = parseJson(reply.text); + let message = `${reply.status} ${errorMessage(body) ?? "status code (no body)"}`; + const issues = + isRecord(body) && isRecord(body.error) ? body.error.issues : undefined; + if (Array.isArray(issues)) { + for (const issue of issues.filter(isRecord)) + message += `\n ${issue.path ?? ""}: ${issue.message ?? ""}`; + } + if (reply.status === 402) { + if (openrouter) { + message += "\nAdd credits at https://openrouter.ai/settings/credits."; + } else { + const host = (baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, ""); + message += `\nAdd funds or a card at ${host}/dashboard/billing.`; + } + } + const bodyId = + isRecord(body) && typeof body.request_id === "string" + ? body.request_id + : undefined; + const requestId = reply.headers.get("x-request-id") ?? bodyId; + if (requestId) message += ` (request_id=${requestId})`; + return new DrexError(message, reply.status, requestId); +} + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +async function run( + args: DrexAuth & { + model?: string; + images?: DrexImage[]; + stateFields?: StateField[]; + threshold?: number; + input?: unknown; + context?: unknown; + output: unknown; + expected?: unknown; + }, + questions: Record, + singleName?: string, +): Promise { + const names = Object.keys(questions); + if (!names.length) throw new Error("At least one question is required"); + for (const name of names) validate(name, questions[name]); + const { threshold } = args; + if (threshold !== undefined && !(threshold >= 0 && threshold <= 1)) { + throw new Error("threshold must be between 0 and 1"); + } + const timeoutMs = args.timeoutMs ?? DEFAULT_TIMEOUT_MS; + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { + throw new Error( + "timeoutMs must be a positive, finite number of milliseconds", + ); + } + const maxRetries = args.maxRetries ?? DEFAULT_MAX_RETRIES; + if (!(maxRetries >= 0)) throw new Error("maxRetries must not be negative"); + const fields = args.stateFields ?? STATE_FIELDS; + const unknownFields = fields.filter((f) => !STATE_FIELDS.includes(f)); + if (unknownFields.length) { + throw new Error( + `Unknown state fields ${JSON.stringify(unknownFields)}; choose from ${JSON.stringify(STATE_FIELDS)}`, + ); + } + const configuredBaseUrl = + args.baseUrl || env("NACE_BASE_URL", "DREX_BASE_URL"); + const openrouter = isOpenRouter(configuredBaseUrl); + if (args.model !== undefined && !args.model.trim()) { + throw new Error("model must not be empty"); + } + if (openrouter && args.model && !args.model.includes("/")) { + throw new Error( + `OpenRouter model ids are namespaced: use '${OPENROUTER_MODEL_PREFIX}${args.model}' (for example ${OPENROUTER_MODEL_PREFIX}${DREX_DEFAULT_MODEL}), not '${args.model}'.`, + ); + } + const model = + args.model || + (openrouter + ? OPENROUTER_MODEL_PREFIX + DREX_DEFAULT_MODEL + : DREX_DEFAULT_MODEL); + let apiKey: string | undefined; + if (openrouter) { + // The key follows the host: never fall back to NACE_API_KEY for OpenRouter. + apiKey = args.apiKey?.trim() || process.env.OPENROUTER_API_KEY?.trim(); + if (!apiKey) { + throw new DrexError( + "No OpenRouter API key was provided. Pass apiKey or set OPENROUTER_API_KEY.", + ); + } + } else { + apiKey = args.apiKey?.trim() || env("NACE_API_KEY", "DREX_API_KEY"); + if (!apiKey) { + throw new DrexError( + "No API key was provided. Pass apiKey or set the NACE_API_KEY environment variable.", + ); + } + } + if (!/^[\x21-\x7e]+$/.test(apiKey)) { + throw new DrexError( + "The API key must contain only printable ASCII characters without whitespace.", + ); + } + + let state = buildState( + { + input: args.input, + output: args.output, + expected: args.expected, + context: args.context, + }, + fields, + ); + if ( + state === undefined || + (typeof state === "object" && state !== null && !Object.keys(state).length) + ) { + throw new Error( + "Nothing to evaluate: pass at least one of output, expected, input or context", + ); + } + + let images = (args.images ?? []).map((image, index) => + prepareImage(image, index), + ); + if (images.length && isTextOnly(model)) { + throw new Error( + `${model} reads text only, so it can't take images. Use an image model such as model='drex-max'.`, + ); + } + if (images.length && openrouter) { + state = openrouterState(state, images); + images = []; + } + + let base = (configuredBaseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, ""); + // OpenRouter documents `.../api/v1`; the route is `/v1/systemone` under `.../api`. + if (openrouter) base = base.replace(/\/v1$/, ""); + const url = base + SYSTEM_ONE_PATH; + const headers: Record = { + Authorization: `Bearer ${apiKey}`, + "Content-Type": "application/json", + Accept: "application/json", + "X-Nace-Client": DREX_CLIENT_NAME, + "User-Agent": DREX_CLIENT_NAME, + ...(openrouter + ? { + "HTTP-Referer": "https://nace.ai", + "X-OpenRouter-Title": "Drex for Autoevals", + } + : {}), + }; + let content: string; + try { + content = JSON.stringify({ + state, + questions: Object.fromEntries(names.map((n) => [n, wire(questions[n])])), + model, + ...(images.length ? { images } : {}), + }); + } catch (error) { + throw new Error( + `The case can't be sent as JSON: ${(error as Error).message}`, + ); + } + // OpenRouter's own limit is not Drex's, so only Drex deployments are checked here. + const bytes = Buffer.byteLength(content); + if (!openrouter && bytes > MAX_BODY_BYTES) { + throw new Error( + `The request is ${(bytes / 1048576).toFixed(1)} MiB, over Drex's 4 MiB limit. Send fewer or smaller images or less text; the scorer does not upload files.`, + ); + } + + let reply: Reply; + for (let attempt = 1; ; attempt++) { + let delay: number | undefined; + try { + reply = await post(url, headers, content, timeoutMs); + if (reply.status >= 200 && reply.status < 300) break; + delay = retryDelayMs(attempt, maxRetries, reply); + if (delay === undefined) { + throw apiError(reply, configuredBaseUrl, openrouter); + } + } catch (error) { + if (!(error instanceof TransportError)) throw error; + delay = retryDelayMs(attempt, maxRetries); + if (delay === undefined) throw new DrexError(error.message); + } + await sleep(delay); + } + + const parsed = parseJson(reply.text); + const bodyId = + isRecord(parsed) && typeof parsed.request_id === "string" + ? parsed.request_id + : undefined; + const requestId = reply.headers.get("x-request-id") ?? bodyId; + if (!isRecord(parsed) || !isRecord(parsed.answers)) { + throw new DrexError( + "Drex returned an unexpected response (no answers).", + reply.status, + requestId, + ); + } + const answers = parsed.answers; + const rawUsage = isRecord(parsed.usage) ? parsed.usage : {}; + const usage = { + input_tokens: rawUsage.input_tokens ?? null, + image_tokens: rawUsage.image_tokens ?? null, + output_tokens: rawUsage.output_tokens ?? null, + }; + return names.map((questionName, index) => { + const question = questions[questionName]; + const answer = answers[questionName]; + if (!isRecord(answer) || answer.type !== question.type) { + throw new DrexError( + `Drex returned no answer for question '${questionName}'`, + undefined, + requestId, + ); + } + let raw: number; + let figures: Record; + try { + [raw, figures] = normalize(question, answer); + } catch { + throw new DrexError( + `Drex returned a malformed answer for question '${questionName}'`, + undefined, + requestId, + ); + } + const value = clamp(raw); + return { + name: singleName ?? questionName, + score: threshold === undefined ? value : value >= threshold ? 1 : 0, + metadata: { + question: questionName, + type: question.type, + direction: "higher is better (probability of a pass)", + ...figures, + raw_value: value, + threshold: threshold ?? null, + model: parsed.model ?? null, + requested_model: model, + request_id: requestId, + evaluation_time_ms: parsed.evaluation_time_ms ?? null, + // One request serves every question; count its tokens once, on the first score. + usage: index === 0 ? usage : null, + usage_counted_on: names[0], + }, + }; + }); +} + +/** + * Score an evaluation case with a Drex decision model. + * + * @example + * ```ts + * const grounded = DrexClassifier.partial({ + * question: drexQuestion.noul("Is every claim in the output supported by the context?"), + * name: "Grounded", + * }); + * const result = await grounded({ output: "Paris is the capital of France.", context: "..." }); + * ``` + */ +export const DrexClassifier: ScorerWithPartial = makePartial( + async (args) => { + const name = args.name ?? "DrexClassifier"; + const [score] = await run(args, { [name]: args.question }, name); + return score; + }, + "DrexClassifier", +); + +/** Answer several questions about one case in a single request; one `Score` per question, in order. */ +export async function DrexClassifierMany(args: DrexManyArgs): Promise { + return run(args, args.questions); +} diff --git a/js/index.ts b/js/index.ts index 40a2a541..bdf41254 100644 --- a/js/index.ts +++ b/js/index.ts @@ -34,6 +34,7 @@ export type { InitOptions } from "./oai"; export * from "./string"; export * from "./list"; export * from "./moderation"; +export * from "./drex"; export * from "./number"; export * from "./json"; export * from "./templates"; diff --git a/js/version.ts b/js/version.ts new file mode 100644 index 00000000..de04e980 --- /dev/null +++ b/js/version.ts @@ -0,0 +1 @@ +export const VERSION = "0.4.1"; diff --git a/py/autoevals/__init__.py b/py/autoevals/__init__.py index 0a5841ad..a92a4a3a 100644 --- a/py/autoevals/__init__.py +++ b/py/autoevals/__init__.py @@ -125,6 +125,7 @@ async def evaluate_qa(): See individual module documentation for detailed usage and options. """ +from .drex import DrexClassifier, DrexError, DrexQuestion from .json import * from .list import * from .litellm import AsyncLiteLLMClient, LiteLLMClient diff --git a/py/autoevals/drex.py b/py/autoevals/drex.py new file mode 100644 index 00000000..b3ac8f00 --- /dev/null +++ b/py/autoevals/drex.py @@ -0,0 +1,748 @@ +"""Drex decision-model scorer. + +`DrexClassifier` asks a Drex decision model (https://console.nace.ai/docs) one typed question about an +evaluation case and turns the answer's probabilities into an Autoevals `Score`. Drex answers with +probabilities, not a verdict, so the score is the probability mass on the outcomes you call a pass, and +the original probabilities stay in `Score.metadata`. + +It calls `POST {base_url}/v1/systemone` over plain HTTP (`httpx` when it is installed, which `openai` +brings, else the standard library) and needs no extra package. Credentials come from ``NACE_API_KEY`` +(and ``NACE_BASE_URL``, which may name your own Drex deployment) unless passed explicitly. + +Example:: + + from autoevals.drex import DrexClassifier, DrexQuestion + + grounded = DrexClassifier( + DrexQuestion.noul("Is every claim in the output supported by the context?"), + name="Grounded", + ) + result = grounded.eval(output="Paris is the capital of France.", context="France's capital is Paris.") + print(result.score) # probability the answer is "yes" + print(result.metadata["probabilities"]) +""" + +from __future__ import annotations + +import asyncio +import base64 +import dataclasses +import email.utils +import json +import math +import os +import random +import re +import time +from collections.abc import Mapping, Sequence +from typing import Any, Literal, NamedTuple +from urllib.parse import urlsplit + +from .score import Score, Scorer +from .version import VERSION + +CLIENT_NAME = f"autoevals/{VERSION}" +DEFAULT_BASE_URL = "https://console.nace.ai" +SYSTEM_ONE_PATH = "/v1/systemone" +DEFAULT_MODEL = "drex-max" +# OpenRouter namespaces model ids: `drex-max` there is `nace-ai/drex-max`, and a bare id does not exist. +OPENROUTER_MODEL_PREFIX = "nace-ai/" +# Released versions that read text only; Drex enforces every other limit itself. +TEXT_ONLY_MODELS = frozenset({"drex-v1.5", "drex-v2.0", "drex-latest"}) +# Drex holds a decision for up to 55 s, so the client must wait longer. +DEFAULT_TIMEOUT = 60.0 +DEFAULT_MAX_RETRIES = 2 +# A server-requested wait longer than this is not slept: the error is raised instead. +MAX_RETRY_WAIT = 60.0 +BACKOFF_INITIAL = 0.5 +BACKOFF_MAX = 5.0 +BACKOFF_JITTER = 0.25 +RETRY_STATUSES = frozenset({408, 429, *range(500, 600)}) +# Drex refuses a larger request body; images go in inline, so the client says so before sending. +MAX_BODY_BYTES = 4 * 1024 * 1024 +MAX_IMAGE_BYTES = 16 * 1024 * 1024 +STATE_FIELDS = ("input", "output", "expected", "context") + +_BASE64 = re.compile(r"[A-Za-z0-9+/]+={0,2}") + + +class DrexError(RuntimeError): + """A request to Drex failed. This is an error in the evaluation, never a low score.""" + + def __init__(self, message: str, *, status: int | None = None, request_id: str | None = None): + super().__init__(message) + self.status = status + self.request_id = request_id + + +@dataclasses.dataclass +class DrexQuestion: + """One question for Drex and how to turn its answer into a 0-1 score (higher is better). + + Use the `noul`, `choice` and `score` constructors. + """ + + type: Literal["noul", "choice", "score"] + instructions: str | None = None + criteria: Any = None + yes_is: Literal["pass", "fail"] = "pass" + """`noul` only: whether a yes answer is good. With `"fail"` the score is the probability of no.""" + choice_scores: Mapping[str, float] | None = None + """`choice` only: the score of each label, from 0 to 1. The score is the probability-weighted sum.""" + reverse: bool = False + """`score` only: treat the lowest level as best, so the score is one minus the normalized level.""" + + @classmethod + def noul( + cls, + instructions: str | None = None, + *, + true: str | None = None, + false: str | None = None, + yes_is: Literal["pass", "fail"] = "pass", + ) -> DrexQuestion: + """A yes/no question. Describe the outcomes with `true` and `false` instead of, or as well as, instructions.""" + criteria = {k: v for k, v in (("true", true), ("false", false)) if v is not None} or None + return cls("noul", instructions, criteria, yes_is=yes_is) + + @classmethod + def choice( + cls, + choices: Mapping[str, str | None], + *, + scores: Mapping[str, float], + instructions: str | None = None, + ) -> DrexQuestion: + """Pick one label. `choices` maps each label to a description or None; `scores` maps each label to 0-1.""" + return cls("choice", instructions, dict(choices), choice_scores=dict(scores)) + + @classmethod + def score( + cls, + levels: Sequence[str], + *, + instructions: str | None = None, + reverse: bool = False, + ) -> DrexQuestion: + """Rate on an ordered rubric, lowest level first. The score is the expected level divided by the top level.""" + return cls("score", instructions, list(levels), reverse=reverse) + + def validate(self, name: str) -> None: + if self.type == "noul": + if self.yes_is not in ("pass", "fail"): + raise ValueError(f"{name}: yes_is must be 'pass' or 'fail'") + outcomes = (self.criteria or {}).values() + if any(v is not None and not isinstance(v, str) for v in outcomes): + raise ValueError(f"{name}: noul criteria must be strings or None") + # Drex refuses a yes/no question with neither instructions nor a described outcome. + if not self.instructions and not any(outcomes): + raise ValueError(f"{name}: a noul question needs instructions or criteria") + elif self.type == "choice": + if not self.criteria: + raise ValueError(f"{name}: a choice question needs at least one label") + scores = self.choice_scores or {} + if set(scores) != set(self.criteria): + raise ValueError(f"{name}: choice scores must cover exactly the labels {sorted(self.criteria)}") + if any(not 0 <= v <= 1 for v in scores.values()): + raise ValueError(f"{name}: choice scores must be between 0 and 1") + elif self.type == "score": + if not self.criteria or len(self.criteria) < 2: + raise ValueError(f"{name}: a score question needs at least two levels") + else: + raise ValueError(f"{name}: unknown question type {self.type!r}") + + def wire(self) -> dict[str, Any]: + """The request form. `instructions` is always a string: OpenRouter requires it, Drex rejects null.""" + question: dict[str, Any] = {"type": self.type, "instructions": self.instructions or ""} + if self.criteria is not None: + question["criteria"] = self.criteria + return question + + +def is_text_only(model: str) -> bool: + """Whether `model` is a version known to read text only. Unknown ids are left to Drex.""" + return model.rsplit("/", 1)[-1].lstrip("~") in TEXT_ONLY_MODELS + + +def _is_openrouter_host(host: str | None) -> bool: + host = (host or "").lower().rstrip(".") + return host == "openrouter.ai" or host.endswith(".openrouter.ai") + + +def _is_openrouter(base_url: str | None) -> bool: + return bool(base_url) and _is_openrouter_host(urlsplit(str(base_url)).hostname) + + +def _env(name: str, legacy: str) -> str | None: + """A stripped environment value: blank counts as unset, then the legacy `DREX_*` name is tried.""" + for key in (name, legacy): + value = os.environ.get(key, "").strip() + if value: + return value + return None + + +def build_state(output: Any, expected: Any = None, fields: Sequence[str] = STATE_FIELDS, **kwargs: Any) -> Any: + """What Drex reads: `output` alone as text, otherwise an object of the case fields that are set.""" + values = {"input": kwargs.get("input"), "output": output, "expected": expected, "context": kwargs.get("context")} + chosen = {name: values[name] for name in fields if values.get(name) is not None} + if list(chosen) == ["output"] and isinstance(chosen["output"], str): + return chosen["output"] + return chosen + + +# -- images --------------------------------------------------------------------------------------- + + +def _media_type(data: bytes) -> str | None: + if data.startswith(b"\x89PNG\r\n\x1a\n"): + return "image/png" + if data.startswith(b"\xff\xd8\xff"): + return "image/jpeg" + if data[:4] == b"RIFF" and data[8:12] == b"WEBP": + return "image/webp" + return None + + +def _data_url(image: str) -> str: + """A base64 string as a data URL, typed from its first bytes (OpenRouter needs the type).""" + if image.startswith("data:"): + return image + head = base64.b64decode(image[:24] + "=" * (-len(image[:24]) % 4), validate=False) + return f"data:{_media_type(head) or 'application/octet-stream'};base64,{image}" + + +def _prepare_image(image: Any, index: int = 0) -> Any: + """One `images` entry as the request sends it: base64, a data URL, or a `workspace_file` reference. + + Drex does not fetch URLs or read paths, and images are never dropped: anything else is refused. + """ + if isinstance(image, str): + if image.startswith(("http://", "https://")): + raise ValueError( + "Drex doesn't fetch image URLs. Pass the image bytes, a base64 string or data URL, " + "or a workspace_file reference." + ) + if not image.startswith("data:") and (len(image) % 4 or not _BASE64.fullmatch(image)): + raise ValueError(f"images[{index}] is neither base64 nor a data URL; pass the image bytes instead.") + return image + if isinstance(image, (bytes, bytearray)): + data = bytes(image) + media_type = _media_type(data) + if media_type is None: + raise ValueError("Drex Max reads PNG, JPEG and WebP images only; these bytes are none of them.") + if len(data) > MAX_IMAGE_BYTES: + raise ValueError(f"The image is {len(data)} bytes, over Drex's 16 MiB per image.") + return f"data:{media_type};base64,{base64.b64encode(data).decode('ascii')}" + if isinstance(image, Mapping): + if image.get("type") == "workspace_file" or ( + "type" not in image and "file_id" in image and "workspace_id" in image + ): + return {"type": "workspace_file", **{k: v for k, v in image.items() if k != "type"}} + raise ValueError( + f"images[{index}] must be bytes, a base64 string, a data URL or a workspace_file reference, " + f"not {type(image).__name__}." + ) + + +def _openrouter_state(state: Any, images: Sequence[Any]) -> list[dict[str, Any]]: + """OpenRouter takes images as `image_url` parts of a `state` content array, not as `images`.""" + text = state if isinstance(state, str) else json.dumps(state) + parts: list[dict[str, Any]] = [{"type": "text", "text": text}] + for image in images: + if not isinstance(image, str): + raise ValueError("OpenRouter accepts inline images only, not uploaded workspace files.") + parts.append({"type": "image_url", "image_url": {"url": _data_url(image)}}) + return parts + + +# -- responses ------------------------------------------------------------------------------------ + + +def _malformed(name: str) -> DrexError: + return DrexError(f"Drex returned a malformed answer for question {name!r}") + + +def _number(value: Any) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise TypeError("not a number") + return float(value) + + +def _normalize(question: DrexQuestion, answer: Mapping[str, Any]) -> tuple[float, dict[str, Any]]: + """Return the 0-1 score (higher is better) and the answer's own figures.""" + if question.type == "noul": + p_yes = _number(answer["noul"]) + probabilities = {"yes": p_yes, "no": 1 - p_yes} + score = p_yes if question.yes_is == "pass" else 1 - p_yes + return score, { + "probabilities": probabilities, + "yes_is": question.yes_is, + # Computed here: a deployment may return no confidence on noul answers. + "confidence": max(p_yes, 1 - p_yes), + } + if question.type == "choice": + probabilities = {str(label): _number(p) for label, p in answer["probabilities"].items()} + weights = question.choice_scores or {} + score = sum(p * weights.get(label, 0.0) for label, p in probabilities.items()) + return score, { + "choice": str(answer["choice"]), + "confidence": _number(answer["confidence"]), + "probabilities": probabilities, + "choice_scores": dict(weights), + } + levels = len(question.criteria) + probabilities = {str(level): _number(p) for level, p in answer["probabilities"].items()} + raw = _number(answer["score"]) + normalized = raw / (levels - 1) + score = 1 - normalized if question.reverse else normalized + return score, { + "raw_score": raw, + "scale": [0, levels - 1], + "legend": [str(item) for item in answer["legend"]], + "confidence": _number(answer["confidence"]), + "probabilities": probabilities, + "reverse": question.reverse, + } + + +def _clamp(value: float) -> float: + return min(1.0, max(0.0, value)) + + +class _Response(NamedTuple): + status: int + headers: dict[str, str] # lower-cased names + body: bytes + + +class _TransportError(Exception): + def __init__(self, message: str, timeout: bool = False): + super().__init__(message) + self.timeout = timeout + + +def _send_urllib(url: str, headers: Mapping[str, str], content: bytes, timeout: float) -> _Response: + """The standard-library transport, used when `httpx` is not installed.""" + import socket + import urllib.error + import urllib.request + + request = urllib.request.Request(url, data=content, headers=dict(headers), method="POST") + message = f"Request timed out after {timeout:g}s" + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + return _Response(response.status, {k.lower(): v for k, v in response.headers.items()}, response.read()) + except urllib.error.HTTPError as error: + try: + body = error.read() + finally: + error.close() + return _Response(error.code, {k.lower(): v for k, v in error.headers.items()}, body) + except (socket.timeout, TimeoutError): + raise _TransportError(message, timeout=True) from None + except urllib.error.URLError as error: + if isinstance(error.reason, (socket.timeout, TimeoutError)): + raise _TransportError(message, timeout=True) from None + raise _TransportError(f"Connection error: {error.reason}") from None + except OSError as error: + raise _TransportError(f"Connection error: {error}") from None + + +def _httpx() -> Any: + try: + import httpx + except ImportError: + return None + return httpx + + +def _from_httpx(response: Any) -> _Response: + return _Response(response.status_code, {k.lower(): v for k, v in response.headers.items()}, response.content) + + +def _httpx_failure(httpx: Any, error: Exception, timeout: float) -> _TransportError: + if isinstance(error, httpx.TimeoutException): + return _TransportError(f"Request timed out after {timeout:g}s", timeout=True) + return _TransportError(f"Connection error: {type(error).__name__}: {error}") + + +def _retry_after(headers: Mapping[str, str]) -> float | None: + """The server's requested wait in seconds: `retry-after-ms`, else `retry-after` (seconds or an HTTP date).""" + for name, scale in (("retry-after-ms", 0.001), ("retry-after", 1.0)): + raw = headers.get(name) + if raw is None: + continue + try: + value = float(raw.strip() or "0") + except ValueError: + if name == "retry-after": + try: + parsed = email.utils.parsedate_to_datetime(raw) + return max(0.0, parsed.timestamp() - time.time()) + except (TypeError, ValueError, OverflowError): + pass + continue + if math.isfinite(value) and value >= 0: + return value * scale + return None + + +def _json_or_text(body: bytes) -> Any: + if not body: + return None + try: + return json.loads(body) + except ValueError: + return body.decode("utf-8", "replace") + + +def _error_message(body: Any) -> str | None: + """The server's message, from `error` (string or object), `message` or `detail`.""" + if isinstance(body, str): + return body[:200] or None + if not isinstance(body, dict): + return None + error, message, detail = body.get("error"), body.get("message"), body.get("detail") + if isinstance(error, str): + return error + if isinstance(error, dict) and isinstance(error.get("message"), str): + return error["message"] + if isinstance(message, str): + return message + if isinstance(detail, str): + return detail + if isinstance(detail, dict) and isinstance(detail.get("message"), str): + return detail["message"] + return None + + +def _body_request_id(body: Any) -> str | None: + value = body.get("request_id") if isinstance(body, dict) else None + return value if isinstance(value, str) else None + + +class DrexClassifier(Scorer): + """Score an evaluation case with a Drex decision model. + + Args: + question: A `DrexQuestion`, or a mapping of names to questions to answer in one request + (see `eval_many`). + name: The score's name. Defaults to `DrexClassifier`. + model: A Drex model id, passed through unchanged. Defaults to `drex-max`, which also reads + images; use `drex-v2.0` for large text-only rubrics if Drex rejects the question count. On OpenRouter + the default is `nace-ai/drex-max` and a bare id is refused. + images: Images every question sees (`drex-max` only), or None. Can also be passed per call as `images=`. + state_fields: Which of `input`, `output`, `expected` and `context` Drex reads. + threshold: When set, the score is 1 if the 0-1 score reaches it and 0 otherwise; the + unthresholded value stays in `metadata["raw_value"]`. + api_key, base_url: Default to `NACE_API_KEY` and `NACE_BASE_URL` (Nace direct). Any base URL that is + not OpenRouter is a Drex deployment, Nace's or your own. Point `base_url` at + `https://openrouter.ai/api/v1` to use OpenRouter (model ids `nace-ai/drex-max`); + the key then comes from `api_key` or `OPENROUTER_API_KEY` and never from `NACE_API_KEY`. + timeout: Seconds per attempt. At least 60 is advised. + max_retries: Retries on 408, 429 and 5xx (529 included) and on connection errors, honouring + `retry-after-ms`, then `retry-after`. A requested wait over 60 s is not slept: the error is raised. + """ + + def __init__( + self, + question: DrexQuestion | Mapping[str, DrexQuestion], + *, + name: str | None = None, + model: str | None = None, + images: Sequence[Any] | None = None, + state_fields: Sequence[str] = STATE_FIELDS, + threshold: float | None = None, + api_key: str | None = None, + base_url: str | None = None, + timeout: float = DEFAULT_TIMEOUT, + max_retries: int = DEFAULT_MAX_RETRIES, + ): + self.questions: dict[str, DrexQuestion] = ( + {name or "DrexClassifier": question} if isinstance(question, DrexQuestion) else dict(question) + ) + if not self.questions: + raise ValueError("At least one question is required") + for key, item in self.questions.items(): + item.validate(key) + unknown = [f for f in state_fields if f not in STATE_FIELDS] + if unknown: + raise ValueError(f"Unknown state fields {unknown}; choose from {list(STATE_FIELDS)}") + if threshold is not None and not 0 <= threshold <= 1: + raise ValueError("threshold must be between 0 and 1") + if not math.isfinite(timeout) or timeout <= 0: + raise ValueError("timeout must be a positive, finite number of seconds") + if max_retries < 0: + raise ValueError("max_retries must not be negative") + self.name = name or (next(iter(self.questions)) if len(self.questions) == 1 else "DrexClassifier") + self.base_url = base_url or _env("NACE_BASE_URL", "DREX_BASE_URL") + self.openrouter = _is_openrouter(self.base_url) + if model is not None and not model.strip(): + raise ValueError("model must not be empty") + if self.openrouter and model and "/" not in model: + raise ValueError( + f"OpenRouter model ids are namespaced: use {OPENROUTER_MODEL_PREFIX}{model!r} " + f"(for example {OPENROUTER_MODEL_PREFIX}{DEFAULT_MODEL}), not {model!r}." + ) + self.model = model or (OPENROUTER_MODEL_PREFIX + DEFAULT_MODEL if self.openrouter else DEFAULT_MODEL) + self.images = list(images) if images else None + self.state_fields = tuple(state_fields) + self.threshold = threshold + self.api_key = api_key + self.timeout = timeout + self.max_retries = max_retries + self._client: Any = None + + def _name(self) -> str: + return self.name + + # -- request ---------------------------------------------------------------------------------- + + def _url(self) -> str: + base = (self.base_url or DEFAULT_BASE_URL).rstrip("/") + if self.openrouter: + # OpenRouter documents `.../api/v1`; the route is `/v1/systemone` under `.../api`. + base = base.removesuffix("/v1") + return base + SYSTEM_ONE_PATH + + def _headers(self) -> dict[str, str]: + if self.openrouter: + # The key follows the host: never fall back to NACE_API_KEY for OpenRouter. + key = self.api_key or os.environ.get("OPENROUTER_API_KEY", "").strip() + if not key: + raise DrexError("No OpenRouter API key was provided. Pass api_key or set OPENROUTER_API_KEY.") + else: + key = (self.api_key or _env("NACE_API_KEY", "DREX_API_KEY") or "").strip() + if not key: + raise DrexError("No API key was provided. Pass api_key or set the NACE_API_KEY environment variable.") + if not key.isascii() or not key.isprintable() or " " in key: + raise DrexError("The API key must contain only printable ASCII characters without whitespace.") + headers = { + "Authorization": f"Bearer {key}", + "Content-Type": "application/json", + "Accept": "application/json", + "X-Nace-Client": CLIENT_NAME, + "User-Agent": CLIENT_NAME, + } + if self.openrouter: + headers.update({"HTTP-Referer": "https://nace.ai", "X-OpenRouter-Title": "Drex for Autoevals"}) + return headers + + def _payload(self, output: Any, expected: Any, kwargs: dict[str, Any], names: Sequence[str]) -> dict[str, Any]: + images = kwargs.get("images", self.images) + state = build_state(output, expected, self.state_fields, **kwargs) + if state == {}: + raise ValueError("Nothing to evaluate: pass at least one of output, expected, input or context") + prepared = [_prepare_image(image, index) for index, image in enumerate(images or ())] + if prepared and is_text_only(self.model): + raise ValueError( + f"{self.model} reads text only, so it can't take images. Use an image model such as model='drex-max'." + ) + if prepared and self.openrouter: + state, prepared = _openrouter_state(state, prepared), [] + payload: dict[str, Any] = { + "state": state, + "questions": {n: self.questions[n].wire() for n in names}, + "model": self.model, + } + if prepared: + payload["images"] = prepared + return payload + + def _encode(self, payload: dict[str, Any]) -> bytes: + try: + content = json.dumps(payload, ensure_ascii=False, separators=(",", ":"), allow_nan=False).encode("utf-8") + except (TypeError, ValueError) as error: + raise ValueError(f"The case can't be sent as JSON: {error}") from error + # OpenRouter's own limit is not Drex's, so only Drex deployments are checked here. + if not self.openrouter and len(content) > MAX_BODY_BYTES: + raise ValueError( + f"The request is {len(content) / 1048576:.1f} MiB, over Drex's 4 MiB limit. " + "Send fewer or smaller images or less text; the scorer does not upload files." + ) + return content + + # -- retries and errors ----------------------------------------------------------------------- + + def _delay(self, attempt: int, response: _Response | None) -> float | None: + """Seconds to wait before retry number `attempt`, or None to stop and raise.""" + if attempt > self.max_retries: + return None + if response is not None: + if response.status not in RETRY_STATUSES: + return None + body = _json_or_text(response.body) + error = body.get("error") if isinstance(body, dict) else None + if isinstance(error, dict) and error.get("retryable") is False: + return None + wait = _retry_after(response.headers) + if wait is not None: + return wait if wait <= MAX_RETRY_WAIT else None + backoff = min(BACKOFF_INITIAL * 2 ** (attempt - 1), BACKOFF_MAX) + return backoff * (1 - random.random() * BACKOFF_JITTER) + + def _api_error(self, response: _Response) -> DrexError: + body = _json_or_text(response.body) + message = f"{response.status} {_error_message(body) or 'status code (no body)'}" + error = body.get("error") if isinstance(body, dict) else None + issues = error.get("issues") if isinstance(error, dict) else None + if isinstance(issues, list): + message += "".join( + f"\n {issue.get('path', '')}: {issue.get('message', '')}" + for issue in issues + if isinstance(issue, dict) + ) + if response.status == 402: + if self.openrouter: + message += "\nAdd credits at https://openrouter.ai/settings/credits." + else: + host = (self.base_url or DEFAULT_BASE_URL).rstrip("/") + message += f"\nAdd funds or a card at {host}/dashboard/billing." + request_id = response.headers.get("x-request-id") or _body_request_id(body) + if request_id: + message += f" (request_id={request_id})" + return DrexError(message, status=response.status, request_id=request_id) + + def _parse(self, response: _Response) -> dict[str, Any]: + body = _json_or_text(response.body) + request_id = response.headers.get("x-request-id") or _body_request_id(body) + if not isinstance(body, dict) or not isinstance(body.get("answers"), dict): + raise DrexError( + "Drex returned an unexpected response (no answers).", status=response.status, request_id=request_id + ) + return {**body, "request_id": request_id} + + def _post_sync(self, payload: dict[str, Any]) -> dict[str, Any]: + url, headers, content = self._url(), self._headers(), self._encode(payload) + httpx = _httpx() + attempt = 0 + while True: + attempt += 1 + try: + if httpx is None: + response = _send_urllib(url, headers, content, self.timeout) + else: + if self._client is None: + self._client = httpx.Client() + try: + response = _from_httpx( + self._client.post(url, content=content, headers=headers, timeout=self.timeout) + ) + except httpx.RequestError as error: + raise _httpx_failure(httpx, error, self.timeout) from None + except _TransportError as error: + delay = self._delay(attempt, None) + if delay is None: + raise DrexError(str(error)) from None + else: + if 200 <= response.status < 300: + return self._parse(response) + delay = self._delay(attempt, response) + if delay is None: + raise self._api_error(response) + time.sleep(delay) + + async def _post_async(self, payload: dict[str, Any]) -> dict[str, Any]: + url, headers, content = self._url(), self._headers(), self._encode(payload) + httpx = _httpx() + # An httpx connection pool is bound to the event loop that first used it, so a client kept across + # `asyncio.run()` calls fails with "Event loop is closed". Hence one per call. + client = httpx.AsyncClient() if httpx is not None else None + attempt = 0 + try: + while True: + attempt += 1 + try: + if client is None: + response = await asyncio.to_thread(_send_urllib, url, headers, content, self.timeout) + else: + try: + response = _from_httpx( + await client.post(url, content=content, headers=headers, timeout=self.timeout) + ) + except httpx.RequestError as error: + raise _httpx_failure(httpx, error, self.timeout) from None + except _TransportError as error: + delay = self._delay(attempt, None) + if delay is None: + raise DrexError(str(error)) from None + else: + if 200 <= response.status < 300: + return self._parse(response) + delay = self._delay(attempt, response) + if delay is None: + raise self._api_error(response) + await asyncio.sleep(delay) + finally: + if client is not None: + await client.aclose() + + def _scores(self, response: dict[str, Any], names: Sequence[str]) -> list[Score]: + usage = response.get("usage") if isinstance(response.get("usage"), dict) else {} + shared = { + "input_tokens": usage.get("input_tokens"), + "image_tokens": usage.get("image_tokens"), + "output_tokens": usage.get("output_tokens"), + } + request_id = response["request_id"] + scores = [] + for index, question_name in enumerate(names): + question = self.questions[question_name] + answer = response["answers"].get(question_name) + if not isinstance(answer, dict) or answer.get("type") != question.type: + raise DrexError(f"Drex returned no answer for question {question_name!r}", request_id=request_id) + try: + value, figures = _normalize(question, answer) + except (KeyError, TypeError, ValueError, AttributeError): + raise _malformed(question_name) from None + value = _clamp(value) + final = value if self.threshold is None else float(value >= self.threshold) + metadata = { + "question": question_name, + "type": question.type, + "direction": "higher is better (probability of a pass)", + **figures, + "raw_value": value, + "threshold": self.threshold, + "model": response.get("model"), + "requested_model": self.model, + "request_id": request_id, + "evaluation_time_ms": response.get("evaluation_time_ms"), + # One request serves every question; count its tokens once, on the first score. + "usage": shared if index == 0 else None, + "usage_counted_on": names[0], + } + scores.append( + Score(name=question_name if len(self.questions) > 1 else self.name, score=final, metadata=metadata) + ) + return scores + + def _single(self) -> str: + if len(self.questions) != 1: + raise ValueError("This scorer has several questions: use eval_many(), which answers them in one request") + return next(iter(self.questions)) + + # -- Scorer API ------------------------------------------------------------------------------- + + def _run_eval_sync(self, output: Any, expected: Any = None, **kwargs: Any) -> Score: + self._single() + return self.eval_many(output, expected, **kwargs)[0] + + async def _run_eval_async(self, output: Any, expected: Any = None, **kwargs: Any) -> Score: + self._single() + return (await self.eval_many_async(output, expected, **kwargs))[0] + + def eval_many(self, output: Any, expected: Any = None, **kwargs: Any) -> list[Score]: + """Answer every question about one case in a single request; one `Score` per question, in order.""" + names = list(self.questions) + payload = self._payload(output, expected, kwargs, names) + return self._scores(self._post_sync(payload), names) + + async def eval_many_async(self, output: Any, expected: Any = None, **kwargs: Any) -> list[Score]: + names = list(self.questions) + payload = self._payload(output, expected, kwargs, names) + return self._scores(await self._post_async(payload), names) + + +__all__ = ["DrexClassifier", "DrexError", "DrexQuestion", "build_state", "is_text_only"] diff --git a/py/autoevals/test_drex.py b/py/autoevals/test_drex.py new file mode 100644 index 00000000..57fd05b8 --- /dev/null +++ b/py/autoevals/test_drex.py @@ -0,0 +1,977 @@ +"""Tests for the Drex decision-model scorer. + +Offline tests run against a fake Drex (respx). Live tests call the real API and run only when +NACE_API_KEY is set: `NACE_API_KEY=nace_sk_... pytest py/autoevals/test_drex.py -k live`. +""" + +import base64 +import email.utils +import json +import os +import struct +import sys +import time +import zlib + +import httpx +import pytest +import respx + +from autoevals import DrexClassifier, DrexError, DrexQuestion +from autoevals.drex import CLIENT_NAME, build_state, is_text_only + +BASE = "https://drex.test" +URL = f"{BASE}/v1/systemone" + + +def png(rgb): + def chunk(kind, data): + body = kind + data + return struct.pack(">I", len(data)) + body + struct.pack(">I", zlib.crc32(body)) + + row = b"\x00" + bytes(rgb) * 2 + return ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", struct.pack(">IIBBBBB", 2, 2, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(row * 2)) + + chunk(b"IEND", b"") + ) + + +RED, BLUE = png((255, 0, 0)), png((0, 0, 255)) + + +def body(answers, model="drex-max", input_tokens=120, image_tokens=0): + return { + "model": model, + "answers": answers, + "usage": {"input_tokens": input_tokens, "image_tokens": image_tokens, "output_tokens": 0}, + "evaluation_time_ms": 41.5, + "request_id": "req_test123", + } + + +NOUL = {"type": "noul", "noul": 0.9} +CHOICE = { + "type": "choice", + "choice": "good", + "confidence": 0.8, + "probabilities": {"good": 0.7, "ok": 0.2, "bad": 0.1}, +} +SCORE = { + "type": "score", + "score": 3.0, + "confidence": 0.7, + "legend": ["awful", "poor", "fine", "great", "perfect"], + "probabilities": {"2": 0.2, "3": 0.6, "4": 0.2}, +} + + +def classifier(question, **kwargs): + kwargs.setdefault("api_key", "nace_sk_test") + kwargs.setdefault("base_url", BASE) + kwargs.setdefault("max_retries", 2) + return DrexClassifier(question, **kwargs) + + +def sent(route, index=-1): + return json.loads(route.calls[index].request.content) + + +@pytest.fixture +def drex(): + with respx.mock(assert_all_called=False) as mock: + yield mock + + +YES_NO = DrexQuestion.noul("Is the output correct?") +CHOICES = DrexQuestion.choice( + {"good": "Fully right", "ok": None, "bad": "Wrong"}, scores={"good": 1, "ok": 0.5, "bad": 0} +) +RUBRIC = DrexQuestion.score(["awful", "poor", "fine", "great", "perfect"], instructions="Rate the answer.") + + +@pytest.mark.parametrize("model", ["drex-max", "drex-v2.0"]) +def test_noul_choice_score_on_both_models(drex, model): + route = drex.post(URL).respond(json=body({"DrexClassifier": NOUL}, model=model)) + result = classifier(YES_NO, model=model).eval("Paris") + assert result.score == pytest.approx(0.9) + assert sent(route)["model"] == model + assert result.metadata["probabilities"] == {"yes": 0.9, "no": pytest.approx(0.1)} + assert result.metadata["request_id"] == "req_test123" + assert result.metadata["model"] == model + + drex.post(URL).respond(json=body({"DrexClassifier": CHOICE}, model=model)) + result = classifier(CHOICES, model=model).eval("Paris") + # 0.7 * 1 + 0.2 * 0.5 + 0.1 * 0 + assert result.score == pytest.approx(0.8) + assert result.metadata["choice"] == "good" + assert result.metadata["probabilities"] == {"good": 0.7, "ok": 0.2, "bad": 0.1} + + drex.post(URL).respond(json=body({"DrexClassifier": SCORE}, model=model)) + result = classifier(RUBRIC, model=model).eval("Paris") + assert result.score == pytest.approx(0.75) # 3 / 4 + assert result.metadata["raw_score"] == 3.0 + assert result.metadata["scale"] == [0, 4] + assert result.metadata["legend"][3] == "great" + assert result.metadata["probabilities"]["3"] == 0.6 + + +def test_default_model_is_drex_max(drex): + route = drex.post(URL).respond(json=body({"DrexClassifier": NOUL})) + classifier(YES_NO).eval("x") + assert sent(route)["model"] == "drex-max" + + +def test_unknown_model_ids_pass_through(drex): + route = drex.post(URL).respond(json=body({"DrexClassifier": NOUL}, model="drex-v9")) + classifier(YES_NO, model="drex-v9").eval("x") + assert sent(route)["model"] == "drex-v9" + + +def test_mixed_batch_is_one_request_and_counts_usage_once(drex): + route = drex.post(URL).respond(json=body({"correct": NOUL, "quality": CHOICE, "depth": SCORE}, input_tokens=300)) + scorer = classifier({"correct": YES_NO, "quality": CHOICES, "depth": RUBRIC}) + scores = scorer.eval_many("Paris", input="Capital of France?") + assert route.call_count == 1 + assert [s.name for s in scores] == ["correct", "quality", "depth"] + assert set(sent(route)["questions"]) == {"correct", "quality", "depth"} + assert scores[0].metadata["usage"]["input_tokens"] == 300 + assert scores[1].metadata["usage"] is None and scores[2].metadata["usage"] is None + assert scores[1].metadata["usage_counted_on"] == "correct" + with pytest.raises(ValueError, match="eval_many"): + scorer.eval("Paris") + + +def test_criteria_only_noul_sends_empty_instructions(drex): + route = drex.post(URL).respond(json=body({"DrexClassifier": NOUL})) + classifier(DrexQuestion.noul(true="grounded", false="invented")).eval("x") + question = sent(route)["questions"]["DrexClassifier"] + assert question["instructions"] == "" + assert question["criteria"] == {"true": "grounded", "false": "invented"} + + +def test_choice_null_description_is_kept(drex): + route = drex.post(URL).respond(json=body({"DrexClassifier": CHOICE})) + classifier(CHOICES).eval("x") + assert sent(route)["questions"]["DrexClassifier"]["criteria"]["ok"] is None + + +def test_state_is_built_from_the_case_fields(drex): + route = drex.post(URL).respond(json=body({"DrexClassifier": NOUL})) + classifier(YES_NO).eval("answer", "reference", input="question", context="docs") + assert sent(route)["state"] == { + "input": "question", + "output": "answer", + "expected": "reference", + "context": "docs", + } + classifier(YES_NO, state_fields=("output", "context")).eval("answer", "reference", context="docs") + assert sent(route)["state"] == {"output": "answer", "context": "docs"} + classifier(YES_NO).eval("answer") + assert sent(route)["state"] == "answer" + assert build_state({"a": 1}) == {"output": {"a": 1}} + + +def test_direction_reverse_and_threshold(drex): + drex.post(URL).respond(json=body({"DrexClassifier": NOUL})) + defect = DrexQuestion.noul("Does the output contain a defect?", yes_is="fail") + result = classifier(defect).eval("x") + assert result.score == pytest.approx(0.1) + assert result.metadata["yes_is"] == "fail" + assert result.metadata["probabilities"]["yes"] == 0.9 # the answer itself is not flipped + + drex.post(URL).respond(json=body({"DrexClassifier": SCORE})) + result = classifier(DrexQuestion.score(RUBRIC.criteria, reverse=True)).eval("x") + assert result.score == pytest.approx(0.25) + + drex.post(URL).respond(json=body({"DrexClassifier": NOUL})) + result = classifier(YES_NO, threshold=0.95).eval("x") + assert result.score == 0 and result.metadata["raw_value"] == pytest.approx(0.9) + assert classifier(YES_NO, threshold=0.5).eval("x").score == 1 + + +def test_attribution_headers(drex): + route = drex.post(URL).respond(json=body({"DrexClassifier": NOUL})) + classifier(YES_NO).eval("x") + headers = route.calls[0].request.headers + assert headers["authorization"] == "Bearer nace_sk_test" + assert headers["x-nace-client"] == CLIENT_NAME + assert headers["user-agent"] == CLIENT_NAME + + +def test_one_image_two_images_and_reversed_order(drex): + route = drex.post(URL).respond(json=body({"DrexClassifier": NOUL})) + scorer = classifier(YES_NO) + scorer.eval("Is Image 1 red?", images=[RED]) + assert len(sent(route)["images"]) == 1 + scorer.eval("Is Image 1 red?", images=[RED, BLUE]) + forward = sent(route)["images"] + scorer.eval("Is Image 1 red?", images=[BLUE, RED]) + reverse = sent(route)["images"] + assert len(forward) == 2 and forward == list(reversed(reverse)) + assert forward[0].startswith("data:image/png;base64,") + assert base64.b64decode(forward[0].split(",", 1)[1]) == RED + + +def test_images_on_text_only_model_are_refused_before_sending(drex): + route = drex.post(URL).respond(json=body({"DrexClassifier": NOUL})) + for model in ("drex-v2.0", "drex-v1.5", "drex-latest", "nace-ai/drex-v2.0"): + with pytest.raises(ValueError, match="reads text only"): + classifier(YES_NO, model=model).eval("x", images=[RED]) + assert route.call_count == 0 + assert is_text_only("drex-v2.0") and not is_text_only("drex-max") + + +def test_remote_image_urls_are_refused(drex): + with pytest.raises(ValueError, match="doesn't fetch image URLs"): + classifier(YES_NO).eval("x", images=["https://example.com/a.png"]) + + +def test_workspace_file_reference_passes_through(drex): + route = drex.post(URL).respond(json=body({"DrexClassifier": NOUL})) + ref = {"type": "workspace_file", "workspace_id": "ws_1", "file_id": "file_1"} + classifier(YES_NO).eval("x", images=[ref]) + assert sent(route)["images"] == [ref] + + +def test_more_than_16_questions_on_max_surfaces_drexs_422(drex): + drex.post(URL).respond( + 422, + json={ + "error": { + "type": "invalid_request_error", + "message": "invalid request", + "issues": [{"path": "questions", "message": "at most 16 questions allowed for drex-max"}], + } + }, + ) + questions = {f"q{i}": DrexQuestion.noul(f"Question {i}?") for i in range(17)} + with pytest.raises(DrexError) as caught: + classifier(questions, model="drex-max").eval_many("x") + assert caught.value.status == 422 + assert "questions: at most 16 questions allowed for drex-max" in str(caught.value) + + +def test_error_messages(drex): + drex.post(URL).respond(401, json={"error": {"type": "authentication_error", "message": "Invalid API key"}}) + with pytest.raises(DrexError, match="401") as caught: + classifier(YES_NO).eval("x") + assert caught.value.status == 401 + + drex.post(URL).respond(402, json={"error": {"type": "insufficient_credit", "message": "No available funds"}}) + with pytest.raises(DrexError) as caught: + classifier(YES_NO).eval("x") + assert f"{BASE}/dashboard/billing" in str(caught.value) + + +@pytest.mark.parametrize("status", [429, 529]) +def test_retries_after_retry_after_ms(drex, status, monkeypatch): + delays = [] + monkeypatch.setattr("time.sleep", delays.append) + route = drex.post(URL).mock( + side_effect=[ + httpx.Response( + status, + headers={"retry-after-ms": "5"}, + json={"error": {"type": "rate_limit_error", "message": "slow down"}}, + ), + httpx.Response(200, json=body({"DrexClassifier": NOUL})), + ] + ) + result = classifier(YES_NO).eval("x") + assert route.call_count == 2 and result.score == pytest.approx(0.9) + assert delays == [pytest.approx(0.005)] # the header's delay, not the SDK's own backoff + + +def test_exhausted_retries_are_an_error_not_a_score(drex): + route = drex.post(URL).respond( + 529, headers={"retry-after-ms": "1"}, json={"error": {"type": "overloaded_error", "message": "busy"}} + ) + with pytest.raises(DrexError) as caught: + classifier(YES_NO, max_retries=1).eval("x") + assert caught.value.status == 529 and route.call_count == 2 + + +def test_openrouter_payload_and_headers(drex): + url = "https://openrouter.ai/api/v1/systemone" + route = drex.post(url).respond(json=body({"DrexClassifier": NOUL})) + scorer = classifier( + DrexQuestion.noul(true="red"), + model="nace-ai/drex-max", + base_url="https://openrouter.ai/api/v1", + ) + scorer.eval("Is Image 1 red?", images=[RED]) + payload = sent(route) + assert "images" not in payload + parts = payload["state"] + assert parts[0] == {"type": "text", "text": "Is Image 1 red?"} + assert parts[1]["type"] == "image_url" and parts[1]["image_url"]["url"].startswith("data:image/png;base64,") + assert payload["questions"]["DrexClassifier"]["instructions"] == "" + headers = route.calls[0].request.headers + assert headers["http-referer"] == "https://nace.ai" + assert headers["x-openrouter-title"] == "Drex for Autoevals" + assert headers["x-nace-client"] == CLIENT_NAME + + +def test_openrouter_base_url_from_env(drex, monkeypatch): + monkeypatch.setenv("NACE_BASE_URL", "https://openrouter.ai/api/v1") + route = drex.post("https://openrouter.ai/api/v1/systemone").respond(json=body({"DrexClassifier": NOUL})) + DrexClassifier(DrexQuestion.noul(true="red"), api_key="nace_sk_test").eval("x") + assert route.calls[0].request.headers["x-openrouter-title"] == "Drex for Autoevals" + + +OPENROUTER = "https://openrouter.ai/api/v1" +OPENROUTER_URL = f"{OPENROUTER}/systemone" + + +@pytest.fixture +def clean_env(monkeypatch): + for name in ("NACE_API_KEY", "DREX_API_KEY", "OPENROUTER_API_KEY", "NACE_BASE_URL", "DREX_BASE_URL"): + monkeypatch.delenv(name, raising=False) + monkeypatch.delenv("NACE_VERCEL_PROTECTION_BYPASS", raising=False) + return monkeypatch + + +def test_openrouter_key_comes_from_api_key_or_openrouter_api_key_never_nace(drex, clean_env): + route = drex.post(OPENROUTER_URL).respond(json=body({"DrexClassifier": NOUL})) + clean_env.setenv("NACE_API_KEY", "nace_sk_must_not_leave") + with pytest.raises(DrexError, match="OPENROUTER_API_KEY"): + DrexClassifier(YES_NO, base_url=OPENROUTER).eval("x") + assert route.call_count == 0 + clean_env.setenv("OPENROUTER_API_KEY", "or_env_key") + DrexClassifier(YES_NO, base_url=OPENROUTER).eval("x") + assert route.calls[-1].request.headers["authorization"] == "Bearer or_env_key" + DrexClassifier(YES_NO, base_url=OPENROUTER, api_key="or_explicit").eval("x") + assert route.calls[-1].request.headers["authorization"] == "Bearer or_explicit" + + +def test_nace_host_never_uses_the_openrouter_key(drex, clean_env): + route = drex.post(URL).respond(json=body({"DrexClassifier": NOUL})) + clean_env.setenv("OPENROUTER_API_KEY", "or_must_not_leave") + with pytest.raises(DrexError, match="API key"): + DrexClassifier(YES_NO, base_url=BASE).eval("x") + assert route.call_count == 0 + clean_env.setenv("NACE_API_KEY", "nace_sk_env") + DrexClassifier(YES_NO, base_url=BASE).eval("x") + assert route.calls[0].request.headers["authorization"] == "Bearer nace_sk_env" + + +def test_vercel_bypass_secret_is_never_forwarded(drex, clean_env): + clean_env.setenv("NACE_VERCEL_PROTECTION_BYPASS", "bypass-secret") + nace = drex.post(URL).respond(json=body({"DrexClassifier": NOUL})) + openrouter = drex.post(OPENROUTER_URL).respond(json=body({"DrexClassifier": NOUL})) + classifier(YES_NO).eval("x") + classifier(YES_NO, base_url=OPENROUTER).eval("x") + assert "x-vercel-protection-bypass" not in nace.calls[0].request.headers + assert "x-vercel-protection-bypass" not in openrouter.calls[0].request.headers + + +def test_openrouter_host_match_is_exact(drex): + from autoevals.drex import _is_openrouter + + assert _is_openrouter("https://openrouter.ai/api/v1") + assert _is_openrouter("https://api.openrouter.ai/v1") + assert _is_openrouter("https://OpenRouter.ai") + assert _is_openrouter("https://openrouter.ai./api/v1") + for other in ( + "https://notopenrouter.ai/api/v1", + "https://openrouter.ai.evil.test", + "https://x.test/openrouter.ai", + ): + assert not _is_openrouter(other) + route = drex.post("https://notopenrouter.ai/v1/systemone").respond(json=body({"DrexClassifier": NOUL})) + classifier(YES_NO, base_url="https://notopenrouter.ai").eval("x") + assert "x-openrouter-title" not in route.calls[0].request.headers + + +def test_openrouter_refuses_workspace_files_and_links_its_own_credits(drex): + scorer = classifier(YES_NO, base_url="https://openrouter.ai/api/v1", model="nace-ai/drex-max") + ref = {"type": "workspace_file", "workspace_id": "ws_1", "file_id": "file_1"} + with pytest.raises(ValueError, match="inline images only"): + scorer.eval("x", images=[ref]) + drex.post("https://openrouter.ai/api/v1/systemone").respond( + 402, json={"error": {"type": "insufficient_credit", "message": "No credits"}} + ) + with pytest.raises(DrexError, match="openrouter.ai/settings/credits") as caught: + scorer.eval("x") + assert "/dashboard/billing" not in str(caught.value) + + +def test_inline_base64_images_get_their_mime_type_on_openrouter(drex): + route = drex.post("https://openrouter.ai/api/v1/systemone").respond(json=body({"DrexClassifier": NOUL})) + jpeg = base64.b64encode(b"\xff\xd8\xff\xe0" + bytes(20)).decode() + webp = base64.b64encode(b"RIFF\x00\x00\x00\x00WEBP" + bytes(20)).decode() + other = base64.b64encode(bytes(30)).decode() + scorer = classifier(YES_NO, base_url="https://openrouter.ai/api/v1", model="nace-ai/drex-max") + scorer.eval("x", images=[jpeg, webp, other]) + urls = [part["image_url"]["url"] for part in sent(route)["state"][1:]] + assert [u.split(";")[0] for u in urls] == ["data:image/jpeg", "data:image/webp", "data:application/octet-stream"] + + +def test_empty_output_is_scored_not_rejected(drex): + route = drex.post(URL).respond(json=body({"DrexClassifier": NOUL})) + classifier(YES_NO).eval("") + assert sent(route)["state"] == "" + with pytest.raises(ValueError, match="Nothing to evaluate"): + classifier(YES_NO, state_fields=("input",)).eval("x") + + +def test_a_response_without_a_request_id_is_still_scored(drex): + payload = body({"DrexClassifier": NOUL}) + del payload["request_id"] + drex.post(URL).respond(json=payload) + assert classifier(YES_NO).eval("x").metadata["request_id"] is None + + +def test_a_huge_retry_after_raises_instead_of_sleeping(drex, monkeypatch): + slept = [] + monkeypatch.setattr("time.sleep", slept.append) + route = drex.post(URL).respond( + 529, headers={"retry-after-ms": "86400000"}, json={"error": {"type": "overloaded_error", "message": "busy"}} + ) + with pytest.raises(DrexError) as caught: + classifier(YES_NO).eval("x") + assert caught.value.status == 529 + assert route.call_count == 1 + assert slept == [] + + +def test_timeouts_are_an_error_not_a_score(drex): + drex.post(URL).mock(side_effect=httpx.ReadTimeout("too slow")) + with pytest.raises(DrexError) as caught: + classifier(YES_NO, max_retries=0).eval("x") + assert caught.value.status is None + + +@pytest.mark.asyncio +async def test_async_eval(drex): + drex.post(URL).respond(json=body({"DrexClassifier": NOUL})) + result = await classifier(YES_NO).eval_async("x") + assert result.score == pytest.approx(0.9) + + +@pytest.mark.asyncio +async def test_async_errors_and_retries(drex, monkeypatch): + delays = [] + + async def record(delay): + delays.append(delay) + + monkeypatch.setattr("asyncio.sleep", record) + busy = {"error": {"type": "overloaded_error", "message": "busy"}} + route = drex.post(URL).mock( + side_effect=[ + httpx.Response(529, headers={"retry-after-ms": "5"}, json=busy), + httpx.Response(200, json=body({"DrexClassifier": NOUL})), + ] + ) + assert (await classifier(YES_NO).eval_async("x")).score == pytest.approx(0.9) + assert route.call_count == 2 and delays == [pytest.approx(0.005)] + + drex.post(URL).respond(402, json={"error": {"type": "insufficient_credit", "message": "No available funds"}}) + with pytest.raises(DrexError) as caught: + await classifier(YES_NO).eval_async("x") + assert caught.value.status == 402 and f"{BASE}/dashboard/billing" in str(caught.value) + + +def test_async_scorer_survives_several_event_loops(): + """A pooled connection kept from a closed loop fails with "Event loop is closed", so this needs a real socket.""" + import asyncio + import threading + from http.server import BaseHTTPRequestHandler, HTTPServer + + payload = json.dumps(body({"DrexClassifier": NOUL})).encode() + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_POST(self): + self.rfile.read(int(self.headers["content-length"])) + self.send_response(200) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(payload))) + self.send_header("x-request-id", "req_test123") + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *args): + pass + + server = HTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + scorer = classifier(YES_NO, base_url=f"http://127.0.0.1:{server.server_port}") + for _ in range(3): + assert asyncio.run(scorer.eval_async("x")).score == pytest.approx(0.9) + finally: + server.shutdown() + server.server_close() + + +def test_missing_answer_is_an_error(drex): + drex.post(URL).respond(json=body({})) + with pytest.raises(DrexError, match="no answer"): + classifier(YES_NO).eval("x") + + +def test_invalid_questions_fail_at_construction(drex): + route = drex.post(URL).respond(json=body({"DrexClassifier": NOUL})) + with pytest.raises(ValueError, match="cover exactly"): + classifier(DrexQuestion.choice({"a": None, "b": None}, scores={"a": 1})) + with pytest.raises(ValueError, match="at least two levels"): + classifier(DrexQuestion.score(["only"])) + with pytest.raises(ValueError, match="instructions or criteria"): + classifier(DrexQuestion.noul()) + with pytest.raises(ValueError, match="threshold"): + classifier(YES_NO, threshold=2) + assert route.call_count == 0 + + +def test_request_body_and_headers_follow_the_drex_contract(drex, clean_env): + clean_env.setenv("NACE_API_KEY", "nace_sk_env") + route = drex.post(URL).respond(json=body({"DrexClassifier": CHOICE})) + DrexClassifier(CHOICES, base_url=BASE).eval("Paris", input="Capital of France?") + request = route.calls[0].request + assert request.method == "POST" and request.url.path == "/v1/systemone" + assert sent(route) == { + "state": {"input": "Capital of France?", "output": "Paris"}, + "questions": { + "DrexClassifier": { + "type": "choice", + "instructions": "", + "criteria": {"good": "Fully right", "ok": None, "bad": "Wrong"}, + } + }, + "model": "drex-max", + } + assert request.headers["content-type"] == "application/json" + assert request.headers["accept"] == "application/json" + assert request.headers["authorization"] == "Bearer nace_sk_env" + assert request.headers["x-nace-client"] == request.headers["user-agent"] == CLIENT_NAME + + +def test_request_id_comes_from_the_header_else_the_body(drex): + payload = body({"DrexClassifier": NOUL}) + drex.post(URL).respond(json=payload, headers={"x-request-id": "req_header"}) + assert classifier(YES_NO).eval("x").metadata["request_id"] == "req_header" + drex.post(URL).respond(json=payload) + assert classifier(YES_NO).eval("x").metadata["request_id"] == "req_test123" + + +def test_request_id_reaches_errors(drex): + drex.post(URL).respond( + 401, headers={"x-request-id": "req_401"}, json={"error": {"type": "authentication_error", "message": "no"}} + ) + with pytest.raises(DrexError) as caught: + classifier(YES_NO).eval("x") + assert caught.value.request_id == "req_401" and "req_401" in str(caught.value) + + +def test_authentication_errors_are_not_retried(drex, monkeypatch): + monkeypatch.setattr("time.sleep", lambda _: pytest.fail("must not sleep")) + route = drex.post(URL).respond(401, json={"error": {"type": "authentication_error", "message": "no"}}) + with pytest.raises(DrexError): + classifier(YES_NO).eval("x") + assert route.call_count == 1 + + +def test_server_can_forbid_a_retry(drex, monkeypatch): + monkeypatch.setattr("time.sleep", lambda _: pytest.fail("must not sleep")) + route = drex.post(URL).respond( + 503, json={"error": {"type": "overloaded_error", "message": "stop", "retryable": False}} + ) + with pytest.raises(DrexError): + classifier(YES_NO).eval("x") + assert route.call_count == 1 + + +def test_retry_after_seconds_and_http_dates_are_honoured_and_backoff_is_the_fallback(drex, monkeypatch): + delays = [] + monkeypatch.setattr("time.sleep", delays.append) + ok = httpx.Response(200, json=body({"DrexClassifier": NOUL})) + busy = {"error": {"type": "overloaded_error", "message": "busy"}} + drex.post(URL).mock( + side_effect=[ + httpx.Response(429, headers={"retry-after": "2"}, json=busy), + httpx.Response( + 503, + headers={"retry-after": email.utils.formatdate(time.time() + 3, usegmt=True)}, + json=busy, + ), + ok, + ] + ) + classifier(YES_NO, max_retries=2).eval("x") + assert delays[0] == pytest.approx(2.0) + assert 1.5 < delays[1] <= 3.0 + delays.clear() + drex.post(URL).mock(side_effect=[httpx.Response(500, json=busy), httpx.Response(502, json=busy), ok]) + classifier(YES_NO, max_retries=2).eval("x") + assert 0.375 <= delays[0] <= 0.5 and 0.75 <= delays[1] <= 1.0 # 0.5 s and 1 s, minus up to 25% jitter + + +def test_retry_after_ms_wins_over_retry_after(drex, monkeypatch): + delays = [] + monkeypatch.setattr("time.sleep", delays.append) + drex.post(URL).mock( + side_effect=[ + httpx.Response( + 429, headers={"retry-after-ms": "7", "retry-after": "30"}, json={"error": {"message": "slow"}} + ), + httpx.Response(200, json=body({"DrexClassifier": NOUL})), + ] + ) + classifier(YES_NO).eval("x") + assert delays == [pytest.approx(0.007)] + + +def test_connection_errors_are_retried_then_reported(drex, monkeypatch): + delays = [] + monkeypatch.setattr("time.sleep", delays.append) + route = drex.post(URL).mock( + side_effect=[httpx.ConnectError("refused"), httpx.Response(200, json=body({"DrexClassifier": NOUL}))] + ) + assert classifier(YES_NO).eval("x").score == pytest.approx(0.9) + assert route.call_count == 2 and len(delays) == 1 + drex.post(URL).mock(side_effect=httpx.ConnectError("refused")) + with pytest.raises(DrexError, match="Connection error") as caught: + classifier(YES_NO, max_retries=1).eval("x") + assert caught.value.status is None + + +def test_timeouts_are_retried_and_the_default_waits_at_least_60_seconds(drex, monkeypatch): + monkeypatch.setattr("time.sleep", lambda _: None) + seen = [] + + def slow(request): + seen.append(request.extensions["timeout"]["read"]) + raise httpx.ReadTimeout("too slow", request=request) + + route = drex.post(URL).mock(side_effect=slow) + with pytest.raises(DrexError, match="timed out") as caught: + classifier(YES_NO).eval("x") + assert caught.value.status is None and route.call_count == 3 # the first try and two retries + assert seen and all(value >= 60 for value in seen) + + +def test_invalid_timeout_and_retry_settings_fail_at_construction(): + for bad in (0, -1, float("inf"), float("nan")): + with pytest.raises(ValueError, match="timeout"): + classifier(YES_NO, timeout=bad) + with pytest.raises(ValueError, match="max_retries"): + classifier(YES_NO, max_retries=-1) + + +def test_a_request_over_4_mib_is_refused_before_sending(drex): + route = drex.post(URL).respond(json=body({"DrexClassifier": NOUL})) + with pytest.raises(ValueError, match="4 MiB"): + classifier(YES_NO).eval("x" * (4 * 1024 * 1024 + 1)) + assert route.call_count == 0 + assert classifier(YES_NO).eval("y" * (3 * 1024 * 1024)).score == pytest.approx(0.9) + + +def test_422_issues_are_listed_as_path_and_message(drex): + drex.post(URL).respond( + 422, + json={ + "error": { + "type": "invalid_request_error", + "message": "invalid request", + "issues": [ + {"path": "questions.q1.criteria", "message": "must not be empty"}, + {"path": "images", "message": "at most 8 images"}, + ], + } + }, + ) + with pytest.raises(DrexError) as caught: + classifier(YES_NO).eval("x") + text = str(caught.value) + assert "422 invalid request" in text + assert "\n questions.q1.criteria: must not be empty" in text and "\n images: at most 8 images" in text + + +def test_error_bodies_that_are_not_drex_json_still_give_a_message(drex): + drex.post(URL).respond(502, text="bad gateway") + with pytest.raises(DrexError, match="502 bad gateway"): + classifier(YES_NO, max_retries=0).eval("x") + drex.post(URL).respond(500) + with pytest.raises(DrexError, match="500 status code \\(no body\\)"): + classifier(YES_NO, max_retries=0).eval("x") + + +def test_malformed_and_missing_answers_are_errors(drex): + drex.post(URL).respond(json={"model": "drex-max", "usage": {}}) + with pytest.raises(DrexError, match="unexpected response"): + classifier(YES_NO).eval("x") + drex.post(URL).respond(json=body({"DrexClassifier": {"type": "noul"}})) + with pytest.raises(DrexError, match="malformed answer"): + classifier(YES_NO).eval("x") + drex.post(URL).respond(json=body({"DrexClassifier": {"type": "noul", "noul": "high"}})) + with pytest.raises(DrexError, match="malformed answer"): + classifier(YES_NO).eval("x") + # An answer of another type, or one this client does not know, is no answer to the question asked. + drex.post(URL).respond(json=body({"DrexClassifier": CHOICE})) + with pytest.raises(DrexError, match="no answer"): + classifier(YES_NO).eval("x") + drex.post(URL).respond(json=body({"DrexClassifier": {"type": "future", "x": 1}})) + with pytest.raises(DrexError, match="no answer"): + classifier(YES_NO).eval("x") + + +def test_usage_fields_are_optional(drex): + payload = body({"DrexClassifier": NOUL}) + payload["usage"] = {"input_tokens": 5} + del payload["evaluation_time_ms"] + drex.post(URL).respond(json=payload) + metadata = classifier(YES_NO).eval("x").metadata + assert metadata["usage"] == {"input_tokens": 5, "image_tokens": None, "output_tokens": None} + assert metadata["evaluation_time_ms"] is None + + +def test_images_must_be_inline_and_are_never_dropped(drex): + route = drex.post(URL).respond(json=body({"DrexClassifier": NOUL})) + with pytest.raises(ValueError, match="PNG, JPEG and WebP"): + classifier(YES_NO).eval("x", images=[b"%PDF-1.7 not an image"]) + with pytest.raises(ValueError, match="neither base64 nor a data URL"): + classifier(YES_NO).eval("x", images=["/tmp/not-base64.png"]) + with pytest.raises(ValueError, match="images\\[0\\]"): + classifier(YES_NO).eval("x", images=[object()]) + with pytest.raises(ValueError, match="images\\[1\\]"): + classifier(YES_NO).eval("x", images=[RED, {"type": "url", "url": "https://x.test/a.png"}]) + assert route.call_count == 0 + # A base64 string, a data URL, bytes and an untyped uploaded-file mapping are all sent, in order. + b64 = base64.b64encode(RED).decode() + classifier(YES_NO).eval( + "x", images=[b64, f"data:image/png;base64,{b64}", BLUE, {"workspace_id": "ws_1", "file_id": "file_1"}] + ) + images = sent(route)["images"] + assert images[0] == b64 and images[1] == f"data:image/png;base64,{b64}" + assert base64.b64decode(images[2].split(",", 1)[1]) == BLUE + assert images[3] == {"type": "workspace_file", "workspace_id": "ws_1", "file_id": "file_1"} + + +# -- the standard-library transport, used when httpx is not installed --------------------------- + + +@pytest.fixture +def local_drex(monkeypatch): + """A real HTTP server whose scripted replies are `(status, headers, json)`; httpx is made unimportable.""" + import threading + from http.server import BaseHTTPRequestHandler, HTTPServer + + replies = [] + seen = [] + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_POST(self): + content = self.rfile.read(int(self.headers["content-length"])) + seen.append({"path": self.path, "headers": dict(self.headers), "body": json.loads(content)}) + status, headers, payload = replies.pop(0) if len(replies) > 1 else replies[0] + data = json.dumps(payload).encode() + self.send_response(status) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(data))) + for name, value in headers.items(): + self.send_header(name, value) + self.end_headers() + self.wfile.write(data) + + def log_message(self, *args): + pass + + server = HTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + monkeypatch.setitem(sys.modules, "httpx", None) + yield f"http://127.0.0.1:{server.server_port}", replies, seen + server.shutdown() + server.server_close() + + +def test_without_httpx_the_standard_library_transport_works(local_drex, monkeypatch): + base, replies, seen = local_drex + delays = [] + monkeypatch.setattr("time.sleep", delays.append) + busy = {"error": {"type": "rate_limit_error", "message": "slow down"}} + replies += [ + (429, {"retry-after-ms": "5"}, busy), + (200, {"x-request-id": "req_urllib"}, body({"DrexClassifier": NOUL})), + ] + result = classifier(YES_NO, base_url=base).eval("Paris", images=[RED]) + assert result.score == pytest.approx(0.9) and result.metadata["request_id"] == "req_urllib" + assert delays == [pytest.approx(0.005)] and len(seen) == 2 + assert seen[0]["path"] == "/v1/systemone" and seen[0]["body"]["model"] == "drex-max" + assert seen[0]["body"]["images"][0].startswith("data:image/png;base64,") + headers = {name.lower(): value for name, value in seen[0]["headers"].items()} + assert headers["authorization"] == "Bearer nace_sk_test" and headers["user-agent"] == CLIENT_NAME + assert headers["x-nace-client"] == CLIENT_NAME + + +def test_without_httpx_errors_and_connection_failures_are_mapped(local_drex, monkeypatch): + base, replies, seen = local_drex + replies.append( + ( + 402, + {"x-request-id": "req_402"}, + {"error": {"type": "insufficient_credit", "message": "No available funds"}}, + ) + ) + with pytest.raises(DrexError) as caught: + classifier(YES_NO, base_url=base).eval("x") + assert caught.value.status == 402 and f"{base}/dashboard/billing" in str(caught.value) + assert caught.value.request_id == "req_402" and len(seen) == 1 + replies[:] = [ + (422, {}, {"error": {"message": "invalid request", "issues": [{"path": "questions", "message": "too many"}]}}) + ] + with pytest.raises(DrexError, match="questions: too many"): + classifier(YES_NO, base_url=base).eval("x") + monkeypatch.setattr("time.sleep", lambda _: None) + with pytest.raises(DrexError, match="Connection error"): + classifier(YES_NO, base_url="http://127.0.0.1:9", max_retries=1).eval("x") + + +def test_without_httpx_async_runs_the_same_transport(local_drex): + import asyncio + + base, replies, seen = local_drex + replies.append((200, {}, body({"DrexClassifier": NOUL}))) + assert asyncio.run(classifier(YES_NO, base_url=base).eval_async("x")).score == pytest.approx(0.9) + assert len(seen) == 1 + + +def test_the_module_imports_without_httpx_or_a_sdk(): + import subprocess + + code = ( + "import sys; sys.modules['httpx'] = None; sys.modules['nace_sdk'] = None\n" + "from autoevals.drex import DrexClassifier, DrexQuestion\n" + "assert DrexClassifier(DrexQuestion.noul('ok?'), api_key='k').model == 'drex-max'" + ) + subprocess.run([sys.executable, "-c", code], check=True) + + +# -- live checks: need a real key, run with `-k live` --------------------------------- + +live = pytest.mark.skipif(not os.environ.get("NACE_API_KEY"), reason="NACE_API_KEY is not set") + + +@live +@pytest.mark.parametrize("model", ["drex-max", "drex-v2.0"]) +def test_live_text(model): + scorer = DrexClassifier( + DrexQuestion.noul("Does the output state that Paris is the capital of France?"), + model=model, + base_url=os.environ.get("NACE_BASE_URL"), + ) + assert scorer.eval("The capital of France is Paris.").score > 0.8 + assert scorer.eval("The capital of France is Berlin.").score < 0.2 + + +@live +def test_live_images_on_max(): + scorer = DrexClassifier( + DrexQuestion.noul("Is Image 1 solid red?"), model="drex-max", base_url=os.environ.get("NACE_BASE_URL") + ) + assert scorer.eval("Look at the images.", images=[RED, BLUE]).score > 0.8 + assert scorer.eval("Look at the images.", images=[BLUE, RED]).score < 0.2 + + +def test_invalid_noul_criteria_and_image_bytes_raise_value_error(drex): + with pytest.raises(ValueError, match="instructions or criteria"): + classifier(DrexQuestion.noul(true="")) + with pytest.raises(ValueError, match="strings or None"): + classifier(DrexQuestion("noul", "Is it right?", {"true": 1})) + classifier(DrexQuestion.noul("Is it right?", true="yes", false=None)) # a null outcome is allowed + with pytest.raises(ValueError, match="PNG, JPEG and WebP"): + classifier(YES_NO).eval("x", images=[b"not an image"]) + + +def test_openrouter_default_model_is_namespaced_and_bare_ids_are_refused(drex, clean_env): + route = drex.post(OPENROUTER_URL).respond(json=body({"DrexClassifier": NOUL})) + DrexClassifier(YES_NO, base_url=OPENROUTER, api_key="or_key").eval("x") + assert sent(route)["model"] == "nace-ai/drex-max" + DrexClassifier(YES_NO, base_url=OPENROUTER, api_key="or_key", model="nace-ai/drex-v2.0").eval("x") + assert sent(route)["model"] == "nace-ai/drex-v2.0" + for bare in ("drex-max", "drex-v2.0", "drex-latest"): + with pytest.raises(ValueError, match="namespaced"): + DrexClassifier(YES_NO, base_url=OPENROUTER, api_key="or_key", model=bare) + # Nace direct keeps the bare default. + drex.post(URL).respond(json=body({"DrexClassifier": NOUL})) + assert classifier(YES_NO).model == "drex-max" + + +def test_drex_latest_stays_text_only_on_openrouter(drex, clean_env): + scorer = DrexClassifier(YES_NO, base_url=OPENROUTER, api_key="or_key", model="nace-ai/drex-latest") + with pytest.raises(ValueError, match="reads text only"): + scorer.eval("x", images=[RED]) + + +ON_PREM = "https://drex.customer.example" + + +def test_on_prem_host_uses_the_nace_key_and_plain_nace_shaping(drex, clean_env): + clean_env.setenv("NACE_API_KEY", "nace_sk_onprem") + clean_env.setenv("OPENROUTER_API_KEY", "or_must_not_leave") + route = drex.post(f"{ON_PREM}/v1/systemone").respond(json=body({"DrexClassifier": NOUL})) + DrexClassifier(YES_NO, base_url=f"{ON_PREM}/").eval("Is Image 1 red?", images=[RED]) + request = route.calls[0].request + payload = json.loads(request.content) + assert request.headers["authorization"] == "Bearer nace_sk_onprem" + assert payload["model"] == "drex-max" + assert len(payload["images"]) == 1 and payload["state"] == "Is Image 1 red?" + assert "x-openrouter-title" not in request.headers and "http-referer" not in request.headers + assert request.headers["x-nace-client"] == CLIENT_NAME + # `NACE_BASE_URL` alone reaches the same host, and an explicit api_key wins over the environment. + clean_env.setenv("NACE_BASE_URL", ON_PREM) + DrexClassifier(YES_NO, api_key="nace_sk_explicit").eval("x") + assert route.calls[-1].request.headers["authorization"] == "Bearer nace_sk_explicit" + + +def test_billing_link_follows_the_configured_host(drex, clean_env): + insufficient = {"error": {"type": "insufficient_credit", "message": "No available funds"}} + drex.post(f"{ON_PREM}/v1/systemone").respond(402, json=insufficient) + with pytest.raises(DrexError) as caught: + DrexClassifier(YES_NO, base_url=ON_PREM, api_key="nace_sk_onprem").eval("x") + assert f"{ON_PREM}/dashboard/billing" in str(caught.value) + # With no base URL the link is Nace's console; OpenRouter never gets a Nace link. + drex.post("https://console.nace.ai/v1/systemone").respond(402, json=insufficient) + with pytest.raises(DrexError) as caught: + DrexClassifier(YES_NO, api_key="nace_sk_test").eval("x") + assert "https://console.nace.ai/dashboard/billing" in str(caught.value) + + +# -- live OpenRouter checks: need OPENROUTER_API_KEY, never NACE_API_KEY --------------- + +live_openrouter = pytest.mark.skipif(not os.environ.get("OPENROUTER_API_KEY"), reason="OPENROUTER_API_KEY is not set") + + +@pytest.fixture +def no_nace_key(monkeypatch): + for name in ("NACE_API_KEY", "DREX_API_KEY", "NACE_BASE_URL", "DREX_BASE_URL", "NACE_VERCEL_PROTECTION_BYPASS"): + monkeypatch.delenv(name, raising=False) + + +@live_openrouter +@pytest.mark.parametrize("model", [None, "nace-ai/drex-v2.0", "nace-ai/drex-max"]) +def test_live_openrouter_text(no_nace_key, model): + scorer = DrexClassifier( + DrexQuestion.noul("Does the output state that Paris is the capital of France?"), + base_url="https://openrouter.ai/api/v1", + model=model, + ) + assert scorer.eval("The capital of France is Paris.").score > 0.8 + assert scorer.eval("The capital of France is Berlin.").score < 0.2 + + +@live_openrouter +def test_live_openrouter_inline_image_on_max(no_nace_key): + scorer = DrexClassifier( + DrexQuestion.noul("Is Image 1 solid red?"), + base_url="https://openrouter.ai/api/v1", + model="nace-ai/drex-max", + ) + assert scorer.eval("Look at the images.", images=[RED, BLUE]).score > 0.8 + assert scorer.eval("Look at the images.", images=[BLUE, RED]).score < 0.2