diff --git a/.env.example b/.env.example index f6e4823..f32666f 100644 --- a/.env.example +++ b/.env.example @@ -244,6 +244,17 @@ BAIDU_ENDPOINT=https://qianfan.baidubce.com/v2/chat/completions # DESCRIPTION: Default Baidu ERNIE model. BAIDU_MODEL=glm-5.2 +# ------------------------------------------------------------------------------ +# Fireworks AI (serverless open models) +# ------------------------------------------------------------------------------ +# DESCRIPTION: Fireworks API key (format: fw-...). Get one at fireworks.ai. +# FIREWORKS_API_KEY=your-fireworks-api-key +# DESCRIPTION: Fireworks OpenAI-compatible chat completions endpoint. +FIREWORKS_ENDPOINT=https://api.fireworks.ai/inference/v1/chat/completions +# DESCRIPTION: Default Fireworks model (long-form serverless id). +# Dated suffixes rotate — check https://app.fireworks.ai/models for current ids. +FIREWORKS_MODEL=accounts/fireworks/models/kimi-k2-instruct-0905 + # ------------------------------------------------------------------------------ # Codex (uses your ChatGPT subscription via local codex CLI) # ------------------------------------------------------------------------------ diff --git a/README.md b/README.md index 8d136b8..68f92fe 100644 --- a/README.md +++ b/README.md @@ -221,6 +221,7 @@ Claude Code / Cursor / Codex / Cline / Continue | **Z.ai** | Cloud | GLM-4.7, GLM-4.5-Air | $ | | **Moonshot AI** | Cloud | Kimi K2.6, Kimi K3 | $ | | **Baidu Qianfan** | Cloud | ERNIE 4.5 Turbo, ERNIE X1.1 | $ (unverified — not yet probed against a live key) | +| **Fireworks AI** | Cloud | Kimi K2, GLM-5, DeepSeek V3 (serverless) | $ (unverified — not yet probed against a live key) | **4 local providers** for 100% offline, free usage. **14+ cloud providers** for scale. diff --git a/bin/lynkr-init.js b/bin/lynkr-init.js index 0280d01..e063c3d 100644 --- a/bin/lynkr-init.js +++ b/bin/lynkr-init.js @@ -179,12 +179,21 @@ const PROVIDERS = { extras: [], defaultModel: 'ernie-4.5-turbo-128k', }, + fireworks: { + label: 'Fireworks AI (serverless open models)', + local: false, + creds: [ + { key: 'FIREWORKS_API_KEY', label: 'Fireworks API key (fw-...)', secret: true }, + ], + extras: [], + defaultModel: 'accounts/fireworks/models/kimi-k2-instruct-0905', + }, }; const PROVIDER_ORDER = [ 'ollama', 'llamacpp', 'lmstudio', 'azure-anthropic', 'azure-openai', 'openai', 'atlas', 'openrouter', 'edenai', - 'databricks', 'bedrock', 'vertex', 'zai', 'moonshot', 'baidu', + 'databricks', 'bedrock', 'vertex', 'zai', 'moonshot', 'baidu', 'fireworks', ]; const TIERS = ['SIMPLE', 'MEDIUM', 'COMPLEX', 'REASONING']; @@ -342,6 +351,9 @@ const BASELINE_ENV = { BAIDU_API_KEY: '', BAIDU_ENDPOINT: 'https://qianfan.baidubce.com/v2/chat/completions', BAIDU_MODEL: 'ernie-4.5-turbo-128k', + FIREWORKS_API_KEY: '', + FIREWORKS_ENDPOINT: 'https://api.fireworks.ai/inference/v1/chat/completions', + FIREWORKS_MODEL: 'accounts/fireworks/models/kimi-k2-instruct-0905', LLAMACPP_ENDPOINT: 'http://localhost:8080', LLAMACPP_MODEL: 'default', LLAMACPP_TIMEOUT_MS: '120000', diff --git a/config/model-capabilities.json b/config/model-capabilities.json new file mode 100644 index 0000000..ae22fc6 --- /dev/null +++ b/config/model-capabilities.json @@ -0,0 +1,20 @@ +{ + "version": 1, + "heads": ["reasoning", "codegen", "debugging", "tool_use"], + "notes": "Capability profiles are decoupled from the predictor (see src/routing/capabilities.js). Editing this file re-routes traffic with zero retraining — the HyDRA shortfall-matching port. Tier profiles are seeded to mirror the legacy scalar bands (SIMPLE 0-19, MEDIUM 20-50, COMPLEX 51-75, REASONING 76-100). This file is read once at boot (same as config/model-tiers.json) — restart to pick up edits. No env vars: everything shortfall needs lives here.", + "enabled": true, + "tau": 0.24, + "weights": { + "reasoning": 0.25, + "codegen": 0.25, + "debugging": 0.25, + "tool_use": 0.25 + }, + "tierProfiles": { + "SIMPLE": { "reasoning": 0.2, "codegen": 0.3, "debugging": 0.2, "tool_use": 0.3 }, + "MEDIUM": { "reasoning": 0.45, "codegen": 0.55, "debugging": 0.45, "tool_use": 0.55 }, + "COMPLEX": { "reasoning": 0.7, "codegen": 0.75, "debugging": 0.7, "tool_use": 0.7 }, + "REASONING": { "reasoning": 0.9, "codegen": 0.9, "debugging": 0.9, "tool_use": 0.9 } + }, + "modelOverrides": {} +} diff --git a/config/model-capability-seeds.json b/config/model-capability-seeds.json new file mode 100644 index 0000000..71f307c --- /dev/null +++ b/config/model-capability-seeds.json @@ -0,0 +1,75 @@ +{ + "version": 1, + "updatedAt": "2026-09-08", + "notes": "Reviewed per-family capability seeds. Values are curated estimates triangulated from SWE-Bench Verified, TerminalBench, LiveCodeBench/BigCodeBench and Artificial Analysis coding index (see scripts/seed-capabilities.js to regenerate from live sources). Heads: reasoning/codegen/debugging/tool_use in [0,1]. Keys are normalized family ids (see capability-seeds/family.js) with trailing-* wildcards matched longest-first. Quantized self-hosted servings take an automatic -0.02 haircut at resolve time. Unknown families fall through to the family heuristic, then tier-slot caps.", + "seeds": { + "claude-opus-4*": { "caps": { "reasoning": 0.9, "codegen": 0.9, "debugging": 0.9, "tool_use": 0.9 }, "sources": ["swe-bench-verified", "artificial-analysis"], "note": "curated-estimate: flagship tier" }, + "claude-opus-3*": { "caps": { "reasoning": 0.8, "codegen": 0.82, "debugging": 0.8, "tool_use": 0.78 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" }, + "claude-sonnet-4*": { "caps": { "reasoning": 0.78, "codegen": 0.8, "debugging": 0.78, "tool_use": 0.78 }, "sources": ["swe-bench-verified", "artificial-analysis"], "note": "curated-estimate" }, + "claude-sonnet-3.5": { "caps": { "reasoning": 0.72, "codegen": 0.75, "debugging": 0.72, "tool_use": 0.7 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" }, + "claude-haiku*": { "caps": { "reasoning": 0.42, "codegen": 0.48, "debugging": 0.42, "tool_use": 0.45 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" }, + "gpt-5.6*": { "caps": { "reasoning": 0.85, "codegen": 0.85, "debugging": 0.85, "tool_use": 0.82 }, "sources": ["swe-bench-verified"], "note": "curated-estimate: flagship tier" }, + "gpt-5.4*": { "caps": { "reasoning": 0.85, "codegen": 0.85, "debugging": 0.85, "tool_use": 0.82 }, "sources": ["swe-bench-verified"], "note": "curated-estimate: flagship tier" }, + "gpt-5.3*": { "caps": { "reasoning": 0.82, "codegen": 0.85, "debugging": 0.82, "tool_use": 0.8 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" }, + "gpt-5.2*": { "caps": { "reasoning": 0.8, "codegen": 0.83, "debugging": 0.8, "tool_use": 0.78 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" }, + "gpt-5.1*": { "caps": { "reasoning": 0.7, "codegen": 0.72, "debugging": 0.7, "tool_use": 0.68 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" }, + "gpt-5-mini*": { "caps": { "reasoning": 0.6, "codegen": 0.65, "debugging": 0.6, "tool_use": 0.6 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" }, + "gpt-5-nano*": { "caps": { "reasoning": 0.45, "codegen": 0.5, "debugging": 0.45, "tool_use": 0.45 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" }, + "gpt-4o": { "caps": { "reasoning": 0.65, "codegen": 0.7, "debugging": 0.65, "tool_use": 0.68 }, "sources": ["swe-bench-verified", "livecodebench"], "note": "curated-estimate" }, + "gpt-4o-mini": { "caps": { "reasoning": 0.45, "codegen": 0.5, "debugging": 0.45, "tool_use": 0.48 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" }, + "gpt-4.1": { "caps": { "reasoning": 0.68, "codegen": 0.72, "debugging": 0.68, "tool_use": 0.7 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" }, + "gpt-4.1-mini": { "caps": { "reasoning": 0.5, "codegen": 0.55, "debugging": 0.5, "tool_use": 0.52 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" }, + "o3": { "caps": { "reasoning": 0.88, "codegen": 0.8, "debugging": 0.85, "tool_use": 0.7 }, "sources": ["swe-bench-verified"], "note": "curated-estimate: reasoning tilt" }, + "o3-mini": { "caps": { "reasoning": 0.72, "codegen": 0.68, "debugging": 0.7, "tool_use": 0.6 }, "sources": ["swe-bench-verified"], "note": "curated-estimate: reasoning tilt" }, + "o4-mini": { "caps": { "reasoning": 0.7, "codegen": 0.66, "debugging": 0.68, "tool_use": 0.6 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" }, + "o1": { "caps": { "reasoning": 0.82, "codegen": 0.72, "debugging": 0.78, "tool_use": 0.6 }, "sources": ["swe-bench-verified"], "note": "curated-estimate: reasoning tilt" }, + "o1-mini": { "caps": { "reasoning": 0.62, "codegen": 0.6, "debugging": 0.6, "tool_use": 0.5 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" }, + "gemini-2.5-pro": { "caps": { "reasoning": 0.85, "codegen": 0.83, "debugging": 0.85, "tool_use": 0.78 }, "sources": ["swe-bench-verified", "artificial-analysis"], "note": "curated-estimate" }, + "gemini-3-pro*": { "caps": { "reasoning": 0.85, "codegen": 0.83, "debugging": 0.85, "tool_use": 0.78 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" }, + "gemini-2.5-flash": { "caps": { "reasoning": 0.55, "codegen": 0.6, "debugging": 0.55, "tool_use": 0.58 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" }, + "gemini-2.0-flash": { "caps": { "reasoning": 0.45, "codegen": 0.5, "debugging": 0.45, "tool_use": 0.5 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" }, + "gemini-1.5-pro": { "caps": { "reasoning": 0.6, "codegen": 0.62, "debugging": 0.6, "tool_use": 0.6 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" }, + "deepseek-r1": { "caps": { "reasoning": 0.82, "codegen": 0.78, "debugging": 0.8, "tool_use": 0.6 }, "sources": ["swe-bench-verified", "livecodebench"], "note": "curated-estimate: reasoning tilt, weak tool-use" }, + "deepseek-reasoner": { "caps": { "reasoning": 0.82, "codegen": 0.78, "debugging": 0.8, "tool_use": 0.6 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" }, + "deepseek-chat": { "caps": { "reasoning": 0.65, "codegen": 0.7, "debugging": 0.65, "tool_use": 0.62 }, "sources": ["livecodebench"], "note": "curated-estimate" }, + "deepseek-v3*": { "caps": { "reasoning": 0.68, "codegen": 0.72, "debugging": 0.68, "tool_use": 0.62 }, "sources": ["swe-bench-verified", "livecodebench"], "note": "curated-estimate" }, + "qwen3-max": { "caps": { "reasoning": 0.78, "codegen": 0.8, "debugging": 0.78, "tool_use": 0.72 }, "sources": ["swe-bench-verified", "livecodebench"], "note": "curated-estimate" }, + "qwen3-235b*": { "caps": { "reasoning": 0.78, "codegen": 0.8, "debugging": 0.78, "tool_use": 0.7 }, "sources": ["livecodebench"], "note": "curated-estimate" }, + "qwen3-32b": { "caps": { "reasoning": 0.65, "codegen": 0.7, "debugging": 0.65, "tool_use": 0.6 }, "sources": ["livecodebench"], "note": "curated-estimate" }, + "qwen3-coder*": { "caps": { "reasoning": 0.68, "codegen": 0.78, "debugging": 0.72, "tool_use": 0.62 }, "sources": ["swe-bench-verified", "livecodebench"], "note": "curated-estimate: coder tilt" }, + "qwen2.5-coder-32b*": { "caps": { "reasoning": 0.55, "codegen": 0.68, "debugging": 0.6, "tool_use": 0.55 }, "sources": ["livecodebench"], "note": "curated-estimate: coder tilt" }, + "qwen2.5-coder-14b*": { "caps": { "reasoning": 0.45, "codegen": 0.58, "debugging": 0.5, "tool_use": 0.45 }, "sources": ["livecodebench"], "note": "curated-estimate" }, + "qwen2.5-coder-7b*": { "caps": { "reasoning": 0.35, "codegen": 0.48, "debugging": 0.4, "tool_use": 0.38 }, "sources": ["livecodebench"], "note": "curated-estimate" }, + "qwen3-8b": { "caps": { "reasoning": 0.45, "codegen": 0.5, "debugging": 0.45, "tool_use": 0.45 }, "sources": ["livecodebench"], "note": "curated-estimate" }, + "qwen3-4b": { "caps": { "reasoning": 0.35, "codegen": 0.42, "debugging": 0.38, "tool_use": 0.38 }, "sources": ["livecodebench"], "note": "curated-estimate" }, + "glm-5.2": { "caps": { "reasoning": 0.68, "codegen": 0.72, "debugging": 0.68, "tool_use": 0.66 }, "sources": ["swe-bench-verified", "artificial-analysis"], "note": "curated-estimate: provider-agnostic (z.ai/Baidu/local share this entry)" }, + "glm-5*": { "caps": { "reasoning": 0.65, "codegen": 0.68, "debugging": 0.65, "tool_use": 0.62 }, "sources": ["artificial-analysis"], "note": "curated-estimate: GLM-5 family default" }, + "glm-4.7": { "caps": { "reasoning": 0.62, "codegen": 0.66, "debugging": 0.62, "tool_use": 0.6 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" }, + "glm-4.6": { "caps": { "reasoning": 0.6, "codegen": 0.64, "debugging": 0.6, "tool_use": 0.58 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" }, + "glm-4.5*": { "caps": { "reasoning": 0.5, "codegen": 0.55, "debugging": 0.5, "tool_use": 0.52 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" }, + "glm-4-flash": { "caps": { "reasoning": 0.35, "codegen": 0.4, "debugging": 0.35, "tool_use": 0.38 }, "sources": ["artificial-analysis"], "note": "curated-estimate" }, + "kimi-k3": { "caps": { "reasoning": 0.72, "codegen": 0.75, "debugging": 0.72, "tool_use": 0.7 }, "sources": ["swe-bench-verified", "artificial-analysis"], "note": "curated-estimate" }, + "kimi-k2.6": { "caps": { "reasoning": 0.62, "codegen": 0.66, "debugging": 0.62, "tool_use": 0.62 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" }, + "kimi-k2.5": { "caps": { "reasoning": 0.6, "codegen": 0.64, "debugging": 0.6, "tool_use": 0.6 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" }, + "kimi-k2*": { "caps": { "reasoning": 0.6, "codegen": 0.63, "debugging": 0.6, "tool_use": 0.6 }, "sources": ["swe-bench-verified"], "note": "curated-estimate: Kimi-K2 family default" }, + "llama-3.3-70b*": { "caps": { "reasoning": 0.6, "codegen": 0.65, "debugging": 0.6, "tool_use": 0.58 }, "sources": ["swe-bench-verified", "livecodebench"], "note": "curated-estimate" }, + "llama-3.1-70b*": { "caps": { "reasoning": 0.55, "codegen": 0.6, "debugging": 0.55, "tool_use": 0.52 }, "sources": ["livecodebench"], "note": "curated-estimate" }, + "llama-4-maverick*": { "caps": { "reasoning": 0.6, "codegen": 0.62, "debugging": 0.6, "tool_use": 0.58 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" }, + "llama-4-scout*": { "caps": { "reasoning": 0.5, "codegen": 0.52, "debugging": 0.5, "tool_use": 0.5 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" }, + "llama-3.2*": { "caps": { "reasoning": 0.25, "codegen": 0.3, "debugging": 0.25, "tool_use": 0.28 }, "sources": ["livecodebench"], "note": "curated-estimate" }, + "mistral-large*": { "caps": { "reasoning": 0.6, "codegen": 0.65, "debugging": 0.6, "tool_use": 0.62 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" }, + "codestral*": { "caps": { "reasoning": 0.5, "codegen": 0.62, "debugging": 0.55, "tool_use": 0.5 }, "sources": ["livecodebench"], "note": "curated-estimate: coder tilt" }, + "mistral-small*": { "caps": { "reasoning": 0.45, "codegen": 0.5, "debugging": 0.45, "tool_use": 0.48 }, "sources": ["artificial-analysis"], "note": "curated-estimate" }, + "devstral*": { "caps": { "reasoning": 0.5, "codegen": 0.6, "debugging": 0.55, "tool_use": 0.5 }, "sources": ["swe-bench-verified"], "note": "curated-estimate: coder tilt" }, + "gpt-oss-120b": { "caps": { "reasoning": 0.65, "codegen": 0.68, "debugging": 0.65, "tool_use": 0.55 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" }, + "gpt-oss-20b": { "caps": { "reasoning": 0.45, "codegen": 0.5, "debugging": 0.45, "tool_use": 0.42 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" }, + "gemma-3-27b": { "caps": { "reasoning": 0.45, "codegen": 0.48, "debugging": 0.45, "tool_use": 0.45 }, "sources": ["livecodebench"], "note": "curated-estimate" }, + "gemma-2*": { "caps": { "reasoning": 0.28, "codegen": 0.32, "debugging": 0.28, "tool_use": 0.3 }, "sources": ["livecodebench"], "note": "curated-estimate" }, + "phi-4": { "caps": { "reasoning": 0.45, "codegen": 0.48, "debugging": 0.45, "tool_use": 0.4 }, "sources": ["livecodebench"], "note": "curated-estimate" }, + "phi-3*": { "caps": { "reasoning": 0.28, "codegen": 0.32, "debugging": 0.28, "tool_use": 0.28 }, "sources": ["livecodebench"], "note": "curated-estimate" }, + "minimax-m2*": { "caps": { "reasoning": 0.65, "codegen": 0.68, "debugging": 0.65, "tool_use": 0.65 }, "sources": ["swe-bench-verified"], "note": "curated-estimate" }, + "muse-spark-1.3": { "caps": { "reasoning": 0.85, "codegen": 0.87, "debugging": 0.85, "tool_use": 0.75 }, "sources": ["deepswe-1.1@75.4% (vendor-reported, max setting)", "terminal-bench-2.1@88.8% (vendor-reported)"], "note": "curated-estimate: full 1.3 at max reasoning. Vendor self-reported; absent from public DeepSWE leaderboard (only 1.2 listed); hands-on gap reported by third parties. tool_use capped — no direct tool-use data. Revisit on independent rows." }, + "muse-spark-1.2*": { "caps": { "reasoning": 0.42, "codegen": 0.46, "debugging": 0.42, "tool_use": 0.44 }, "sources": ["lynkr-telemetry:4176 scored rows, avg quality 52.7 @ complexity 51.7"], "note": "measured: operator's own traffic. Below the 70 evidence bar — economize trivial turns only." }, + "muse-spark-1.3-contributor*": { "caps": { "reasoning": 0.4, "codegen": 0.45, "debugging": 0.4, "tool_use": 0.45 }, "sources": ["lynkr-telemetry:29 scored rows (thin) + 1.2 family history"], "note": "provisional: free/contributor serving grades far below full-1.3 published scores (variant, effort setting, or benchmark inflation — undetermined). Do NOT cover with muse-spark*: the full model and the free serving must never share caps." } + } +} diff --git a/config/model-tiers.json b/config/model-tiers.json index 360bd95..b4e8730 100644 --- a/config/model-tiers.json +++ b/config/model-tiers.json @@ -54,6 +54,9 @@ ], "moonshot": [ "kimi-k2.6" + ], + "fireworks": [ + "accounts/fireworks/models/llama-3.1-8b-instruct" ] } }, @@ -109,6 +112,9 @@ ], "moonshot": [ "kimi-k2.6" + ], + "fireworks": [ + "accounts/fireworks/models/kimi-k2-instruct-0905" ] } }, @@ -158,6 +164,9 @@ ], "moonshot": [ "kimi-k2.6" + ], + "fireworks": [ + "accounts/fireworks/models/deepseek-v3p1" ] } }, @@ -205,6 +214,9 @@ "moonshot": [ "kimi-k3", "kimi-k2.6" + ], + "fireworks": [ + "accounts/fireworks/models/glm-5p2" ] } } @@ -233,4 +245,4 @@ "kimi": "moonshot", "z-ai": "zai" } -} \ No newline at end of file +} diff --git a/docs/routing-intelligence.md b/docs/routing-intelligence.md index e03e533..85acf3a 100644 --- a/docs/routing-intelligence.md +++ b/docs/routing-intelligence.md @@ -109,6 +109,26 @@ response → quality score → telemetry (SQLite, .lynkr/telemetry.db) (> `LYNKR_KNN_CONFIDENCE_HIGH`) override the heuristic; ambiguous ones escalate only when telemetry shows cheap tiers actually failing. - The bandit explores only within `TIER_*`-configured models. +- Capability-decoupled shortfall routing (HyDRA port, off by default): + `src/routing/capabilities.js` maps the 15 weighted dims onto 4 heads + (`reasoning/codegen/debugging/tool_use`), and `src/routing/shortfall.js` + serves the cheapest `TIER_*`-configured model covering them within τ. + Switch, tolerance, weights, and model capabilities all live in + `config/model-capabilities.json` (no env vars; restart to pick up edits), + so catalog changes re-route with zero retraining. When off, shortfall still + shadow-computes and logs the legacy-vs-shortfall comparison; when on, a + disagreement serves the shortfall pick with `method+=+shortfall` (upward + moves also record an `escalations[]` entry). Force paths (risk/force + phrases), static mode, and downstream guards (context/vision/kNN/bandit/ + deadline/tenant) all dominate shortfall. +- Model capability is a property of the WEIGHTS, not the provider: + `capability-seeds/family.js` normalizes `zai:glm-5.2`, `baidu:glm-5.2` + and `ollama:glm-5.2` to one family id, so all three servings share caps + (quantized GGUF servings take an automatic −0.02 haircut). Resolution + order: operator `modelOverrides` → fetched snapshot (`data/`, gitignored) + → shipped seeds (`config/model-capability-seeds.json`, reviewed) → family + ladder heuristic → tier-slot caps. Every pick carries a `source` tag + (override|seed:snapshot|seed:shipped|family|tier) for audit. ## Key environment knobs @@ -122,6 +142,29 @@ response → quality score → telemetry (SQLite, .lynkr/telemetry.db) | `LYNKR_KNN_MIN_INDEX_SIZE` | 100 | entries before kNN advises | | `LYNKR_KNN_CONFIDENCE_HIGH` / `_LOW` | 0.7 / 0.4 | override / ambiguous bands | +Shortfall has no env vars — `enabled`, `tau` (~0.01 peak quality, 0.24 +default, ~0.6 aggressive savings), `weights`, tier profiles, and +`modelOverrides` all live in `config/model-capabilities.json` (restart to +pick up edits). + +## Seeding model capabilities (new-model runbook) + +`npm run seed:capabilities` (→ `scripts/seed-capabilities.js --refresh`): + +1. Fetches SWE-Bench Verified + models.dev flags (7-day file cache in + `data/`, offline-safe) plus any `data/capability-benchmarks/*.json` + drop-ins (TerminalBench/LiveCodeBench/AA exports). +2. Maps bare-model scores to families (scaffold+model composites like + "live-swe-agent + claude" are quarantined — they measure harnesses, not + weights — and weak matches stay unmapped for review). +3. Writes `data/capability-seeds.snapshot.json` and prints the diff, + candidate graduations (wildcard matches worth an exact entry), and the + review lists. `--dry-run` prints without writing; `--check` fails CI + when shipped seeds go stale (>90d). +4. Promote by copying reviewed entries into + `config/model-capability-seeds.json` (bump `updatedAt`) and restarting. + Unknown families keep working throughout via heuristic → tier fallback. + Auto-calibration and the telemetry DB location are deliberately **not** configurable — calibration self-gates on sample count, and telemetry lives at `.lynkr/telemetry.db`. diff --git a/documentation/providers.md b/documentation/providers.md index b593405..4dc3a85 100644 --- a/documentation/providers.md +++ b/documentation/providers.md @@ -21,6 +21,7 @@ Lynkr supports multiple AI model providers, giving you flexibility in choosing t | **OpenAI** | Cloud | GPT-4o, o1, o3 | $$$ | Cloud | Easy | | **Atlas Cloud** | Cloud | Qwen, DeepSeek, and other OpenAI-compatible models | $-$$$ | Cloud | Easy | | **Moonshot AI (Kimi)** | Cloud | Kimi K2 (thinking + turbo) | $ | Cloud | Easy | +| **Fireworks AI** | Cloud | Llama, DeepSeek, Qwen, Kimi, GLM (serverless) | $ | Cloud | Easy | | **LM Studio** | Local | Local models with GUI | **FREE** | 🔒 100% Local | Easy | | **MLX OpenAI Server** | Local | Apple Silicon optimized | **FREE** | 🔒 100% Local | Easy | @@ -872,6 +873,69 @@ curl -X POST https://api.moonshot.ai/v1/chat/completions \ --- +### 10a. Fireworks AI (OpenAI-Compatible) + +**Best for:** Fast serverless open-weight models (Llama, DeepSeek, Qwen, Kimi, GLM), cheap mid-tier routing + +#### Configuration + +```env +MODEL_PROVIDER=fireworks +FIREWORKS_API_KEY=fw-your-fireworks-api-key +FIREWORKS_ENDPOINT=https://api.fireworks.ai/inference/v1/chat/completions +FIREWORKS_MODEL=accounts/fireworks/models/kimi-k2-instruct-0905 +``` + +#### Getting a Fireworks API Key + +1. Visit [app.fireworks.ai](https://app.fireworks.ai) +2. Sign up or log in +3. Navigate to API Keys section +4. Create a new key (`fw-...`) + +#### Available Models + +Model ids are long-form serverless paths. Tier-selected ids reach the wire unchanged: + +```env +FIREWORKS_MODEL=accounts/fireworks/models/kimi-k2-instruct-0905 # default, tool calling +FIREWORKS_MODEL=accounts/fireworks/models/glm-5p2 # strong general +FIREWORKS_MODEL=accounts/fireworks/models/deepseek-v3p1 # strong coding +FIREWORKS_MODEL=accounts/fireworks/models/llama-3.1-8b-instruct # cheap/fast +# Fast serving routers (higher speed, select models): +# accounts/fireworks/routers/glm-5p2-fast, accounts/fireworks/routers/kimi-k2p6-fast +``` + +**Note:** dated suffixes (e.g. `-0905`) rotate — check https://app.fireworks.ai/models for current ids. + +#### How It Works + +Fireworks uses an **OpenAI-compatible** chat completions API. Lynkr handles all format conversion automatically: + +1. Claude Code CLI sends Anthropic-format request to Lynkr +2. Lynkr converts Anthropic messages → OpenAI chat completions format +3. Request is sent to Fireworks' `/inference/v1/chat/completions` endpoint +4. Fireworks response is converted back to Anthropic format +5. Claude Code CLI receives a standard Anthropic response + +#### Important Notes (E2E-unverified — probed from docs, not a live key yet) + +- **Streaming:** wired through the SSE transformer like Moonshot/Baidu; confirm chunk shape against a live key before trusting streamed tool calls. +- **Reasoning models** (R1/GLM): reasoning output shares the answer token budget; the buffered path lifts `reasoning_content` into thinking blocks. +- **Tool calling:** full OpenAI function-calling support; Fireworks recommends low temperature (0.0–0.3) for deterministic tool selection — a future tuning knob, not yet pinned. +- **Rate limits:** standard per-minute limits — Lynkr retries with backoff, then tier-fallback climbs on persistent 429s. + +#### Test Connection + +```bash +curl https://api.fireworks.ai/inference/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $FIREWORKS_API_KEY" \ + -d '{"model":"accounts/fireworks/models/kimi-k2-instruct-0905","messages":[{"role":"user","content":"Hello"}]}' +``` + +--- + ### 11. MLX OpenAI Server (Apple Silicon) **Best for:** Maximum performance on Apple Silicon Macs (M1/M2/M3/M4) diff --git a/package.json b/package.json index 4c3404b..066e7ce 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,8 @@ "test:performance": "LYNKR_KNN_DIR=/tmp/lynkr-test-knn DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com node test/hybrid-routing-performance.test.js && DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com node test/performance-tests.js", "test:benchmark": "LYNKR_KNN_DIR=/tmp/lynkr-test-knn DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com node test/performance-benchmark.js", "test:quick": "LYNKR_KNN_DIR=/tmp/lynkr-test-knn DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com node --test test/routing.test.js", - "test:all": "npm run test:unit && npm run test:performance && npm run test:benchmark" + "test:all": "npm run test:unit && npm run test:performance && npm run test:benchmark", + "seed:capabilities": "node scripts/seed-capabilities.js --refresh" }, "keywords": [ "llm", diff --git a/scripts/seed-capabilities.js b/scripts/seed-capabilities.js new file mode 100644 index 0000000..e5e8429 --- /dev/null +++ b/scripts/seed-capabilities.js @@ -0,0 +1,230 @@ +#!/usr/bin/env node +/** + * Seed per-family capability profiles from online benchmark data. + * + * Reads (all optional, all fail-soft): SWE-Bench Verified leaderboard, + * data/capability-benchmarks/*.json drop-ins, models.dev boolean flags. + * Maps scores to families, normalizes to 0-1 caps, and writes + * data/capability-seeds.snapshot.json for the shortfall resolver. + * Shipped seeds (config/model-capability-seeds.json) are only ever changed + * by human review of the printed diff — this script never touches them. + * + * Request-path routing never calls this (zero latency impact) and never + * needs it: unknown families fall back to heuristic → tier caps. + * + * Usage: + * node scripts/seed-capabilities.js [--refresh] [--model provider:model] + * [--dry-run] [--check] [--stale-days 90] + * + * --refresh fetch remote sources (default: cache-only) + * --model limit to one family (e.g. --model z.ai:glm-5.2) + * --dry-run print diff, write nothing + * --check exit 2 if shipped seeds are older than --stale-days (CI) + */ + +const fs = require('fs'); +const path = require('path'); + +const { fetchSweBench } = require('../src/routing/capability-seeds/swebench'); +const { fetchModelsDevFlags, NO_TOOLCALL_TOOL_USE_CAP } = require('../src/routing/capability-seeds/models-dev'); +const { loadBenchmarksDir } = require('../src/routing/capability-seeds/benchmarks-dir'); +const { mapEntryToFamily, scoresToCaps, normalizeFamily } = require('../src/routing/capability-seeds/normalize'); +const { SHIPPED_PATH, SNAPSHOT_PATH } = require('../src/routing/capability-seeds/registry'); + +const STALE_DAYS_DEFAULT = 90; + +function _parseArgs(argv) { + const out = { refresh: false, model: null, dryRun: false, check: false, staleDays: STALE_DAYS_DEFAULT }; + for (let i = 0; i < argv.length; i++) { + if (argv[i] === '--refresh') out.refresh = true; + else if (argv[i] === '--dry-run') out.dryRun = true; + else if (argv[i] === '--check') out.check = true; + else if (argv[i] === '--model') out.model = String(argv[++i] || ''); + else if (argv[i] === '--stale-days') out.staleDays = Number(argv[++i]) || STALE_DAYS_DEFAULT; + } + return out; +} + +function _readJson(p) { + try { + if (!fs.existsSync(p)) return null; + return JSON.parse(fs.readFileSync(p, 'utf8')); + } catch { + return null; + } +} + +// Drop-in source name → normalized score slot. +function _slotFor(source) { + const s = String(source || '').toLowerCase(); + if (s.includes('terminal')) return 'terminal'; + if (s.includes('livecode') || s.includes('bigcode')) return 'livecode'; + if (s.includes('arena')) return 'arena'; + if (s.includes('swe')) return 'swe'; + return null; +} + +async function main() { + const args = _parseArgs(process.argv.slice(2)); + + if (args.check) { + const shipped = _readJson(SHIPPED_PATH); + const updatedAt = shipped?.updatedAt ? new Date(shipped.updatedAt).getTime() : NaN; + const ageDays = Number.isFinite(updatedAt) ? (Date.now() - updatedAt) / 86400000 : Infinity; + if (!(ageDays <= args.staleDays)) { + console.error(`STALE: shipped seeds updatedAt=${shipped?.updatedAt ?? 'missing'} (> ${args.staleDays}d). Run --refresh and review the diff.`); + process.exit(2); + } + console.log(`OK: shipped seeds fresh (${shipped.updatedAt}, ${ageDays.toFixed(0)}d old).`); + return; + } + + const shipped = _readJson(SHIPPED_PATH); + const shippedSeeds = shipped?.seeds && typeof shipped.seeds === 'object' ? shipped.seeds : {}; + const knownFamilies = Object.keys(shippedSeeds).filter((k) => !k.endsWith('*')); + const wildcardFamilies = Object.keys(shippedSeeds).filter((k) => k.endsWith('*')); + + let onlyFamily = null; + if (args.model) { + const [provider, ...rest] = args.model.split(':'); + const { family } = normalizeFamily(provider, rest.join(':') || provider); + onlyFamily = family; + console.log(`Limiting to family: ${onlyFamily}`); + } + + // 1. Gather score entries per source. + const scored = {}; // family -> { swe, terminal, livecode, arena } + const unmapped = []; + const scaffolded = []; + const candidates = []; + const statuses = []; + const addScore = (family, slot, fraction, source) => { + if (onlyFamily && family !== onlyFamily) return; + scored[family] = scored[family] || {}; + // Keep the max when several entries map to one family. + if (scored[family][slot] === undefined || fraction > scored[family][slot]) { + scored[family][slot] = fraction; + scored[family][`${slot}:source`] = source; + } + }; + // Bare-model scores only: a "+" joins scaffold + model ("live-swe-agent + + // claude 4.5 opus") and measures the harness, not the weights. Those go to + // a separate review list and never seed caps. + const triage = (name, score, slot, source) => { + if (String(name).includes('+')) { + scaffolded.push({ source, name, score }); + return; + } + const m = mapEntryToFamily(name, knownFamilies); + if (m.family) { + addScore(m.family, slot, score, source); + return; + } + const w = mapEntryToFamily(name, wildcardFamilies); + if (w.family) { + candidates.push({ entry: name, score, slot, prefix: w.family.replace(/\*$/, '') }); + return; + } + unmapped.push({ source, name, score }); + }; + + const swe = await fetchSweBench({ refresh: args.refresh }); + statuses.push(`swebench[${swe.board || '?'}]: ${swe.status}${swe.cached ? ' (cache)' : ''}${swe.reason ? ` — ${swe.reason}` : ''}`); + for (const e of swe.entries || []) { + triage(e.name, e.resolved / 100, 'swe', `swebench-verified@${e.resolved}%`); + } + + const dir = loadBenchmarksDir(); + for (const f of dir.files || []) { + const slot = _slotFor(f.source); + statuses.push(`${f.source}: ${f.status}${f.reason ? ` — ${f.reason}` : ''}${f.status === 'ok' && !slot ? ' (unknown slot — skipped)' : ''}`); + if (f.status !== 'ok' || !slot) continue; + for (const e of f.entries || []) { + const fraction = f.scale === 'fraction' ? e.score : e.score / 100; + triage(e.name, fraction, slot, `${f.source}@${e.score}`); + } + } + + const md = await fetchModelsDevFlags({ refresh: args.refresh }); + statuses.push(`models.dev: ${md.status}${md.cached ? ' (cache)' : ''}${md.reason ? ` — ${md.reason}` : ''}`); + + // 2. Normalize to caps. + const proposed = {}; + for (const [family, slots] of Object.entries(scored)) { + const scores = {}; + for (const k of ['swe', 'terminal', 'livecode', 'arena']) { + if (Number.isFinite(slots[k])) scores[k] = slots[k]; + } + const caps = scoresToCaps(scores); + if (!caps) continue; + const sources = Object.keys(scores).map((k) => slots[`${k}:source`]).filter(Boolean); + const flag = md.flags?.[family]; + if (flag && flag.toolCall === false && caps.tool_use > NO_TOOLCALL_TOOL_USE_CAP) { + caps.tool_use = NO_TOOLCALL_TOOL_USE_CAP; + sources.push('models.dev:no-tool_call-cap'); + } + proposed[family] = { caps, sources, fetchedAt: new Date().toISOString().slice(0, 10) }; + } + + // 3. Diff vs snapshot + shipped. + const snapshot = _readJson(SNAPSHOT_PATH); + const prevSeeds = snapshot?.seeds && typeof snapshot.seeds === 'object' ? snapshot.seeds : {}; + const moves = []; + for (const [family, entry] of Object.entries(proposed)) { + const prev = prevSeeds[family]; + const shippedCaps = shippedSeeds[family]?.caps; + const sameAs = (a, b) => a && b && ['reasoning', 'codegen', 'debugging', 'tool_use'].every((h) => a[h] === b[h]); + if (!prev && !shippedCaps) moves.push({ family, type: 'added', caps: entry.caps }); + else if (!sameAs(prev?.caps, entry.caps)) moves.push({ family, type: 'changed', from: prev?.caps ?? null, to: entry.caps }); + } + + console.log('Sources:'); + for (const s of statuses) console.log(` - ${s}`); + console.log(`Families scored: ${Object.keys(proposed).length}, changed vs snapshot: ${moves.length}`); + for (const m of moves.slice(0, 30)) { + console.log(` ${m.type} ${m.family}: ${JSON.stringify(m.from ?? null)} → ${JSON.stringify(m.to ?? m.caps)}`); + } + if (moves.length > 30) console.log(` … and ${moves.length - 30} more`); + const topUnmapped = unmapped.sort((a, b) => b.score - a.score).slice(0, 10); + if (topUnmapped.length > 0) { + console.log('Unmapped (no family match — review, add alias/seed if real):'); + for (const u of topUnmapped) console.log(` - [${u.source}] ${u.name} (${u.score})`); + } + const topCandidates = candidates.sort((a, b) => b.score - a.score).slice(0, 15); + if (topCandidates.length > 0) { + console.log('Candidate new seeds (matched a shipped wildcard — add an exact entry to graduate):'); + for (const c of topCandidates) { + console.log(` - "${c.entry}" (${c.slot} ${c.score}) → under prefix "${c.prefix}"`); + } + } + const topScaffolded = scaffolded.sort((a, b) => b.score - a.score).slice(0, 5); + if (topScaffolded.length > 0) { + console.log('Scaffolded entries (harness+model composites — never seeded, shown for context):'); + for (const u of topScaffolded) console.log(` - [${u.source}] ${u.name} (${u.score})`); + } + + if (args.dryRun) { + console.log('dry-run: wrote nothing.'); + return; + } + const next = { + version: 1, + fetchedAt: new Date().toISOString(), + seeds: { ...prevSeeds }, + }; + for (const [family, entry] of Object.entries(proposed)) next.seeds[family] = entry; + try { + fs.mkdirSync(path.dirname(SNAPSHOT_PATH), { recursive: true }); + fs.writeFileSync(SNAPSHOT_PATH, `${JSON.stringify(next, null, 2)}\n`); + console.log(`Wrote ${SNAPSHOT_PATH} (${Object.keys(next.seeds).length} families).`); + console.log('Review the diff above; to promote entries to shipped seeds, copy them into config/model-capability-seeds.json and bump updatedAt.'); + } catch (err) { + console.error(`Write failed: ${err.message}`); + process.exit(1); + } +} + +main().catch((err) => { + console.error(`seed-capabilities failed: ${err?.message || err}`); + process.exit(1); +}); diff --git a/src/api/openai-router.js b/src/api/openai-router.js index f2aea0a..4db1d6e 100644 --- a/src/api/openai-router.js +++ b/src/api/openai-router.js @@ -1143,6 +1143,19 @@ function getConfiguredProviders() { }); } + if (config.fireworks?.apiKey) { + providers.push({ + name: "fireworks", + type: "fireworks-ai", + models: [ + config.fireworks.model || "accounts/fireworks/models/kimi-k2-instruct-0905", + "accounts/fireworks/models/kimi-k2-instruct-0905", + "accounts/fireworks/models/glm-5p2", + "accounts/fireworks/models/deepseek-v3p1" + ] + }); + } + if (config.vertex?.projectId) { providers.push({ name: "vertex", diff --git a/src/api/providers-handler.js b/src/api/providers-handler.js index ecab69c..214c86e 100644 --- a/src/api/providers-handler.js +++ b/src/api/providers-handler.js @@ -237,6 +237,22 @@ function getConfiguredProviders() { }); } + // Check Fireworks AI (serverless open models) + if (config.fireworks?.apiKey) { + providers.push({ + name: "fireworks", + type: "fireworks-ai", + baseUrl: config.fireworks.endpoint || "https://api.fireworks.ai/inference/v1", + enabled: true, + models: [ + { id: config.fireworks.model || "accounts/fireworks/models/kimi-k2-instruct-0905", name: "Configured Model" }, + { id: "accounts/fireworks/models/kimi-k2-instruct-0905", name: "Kimi K2 Instruct" }, + { id: "accounts/fireworks/models/glm-5p2", name: "GLM 5.2" }, + { id: "accounts/fireworks/models/deepseek-v3p1", name: "DeepSeek V3.1" }, + ] + }); + } + // Check Vertex AI (Google Cloud) if (config.vertex?.projectId) { const region = config.vertex.region || "us-east5"; diff --git a/src/clients/databricks.js b/src/clients/databricks.js index 99f0a53..e2a3a45 100644 --- a/src/clients/databricks.js +++ b/src/clients/databricks.js @@ -2552,6 +2552,147 @@ async function invokeBaidu(body, _incomingHeaders = {}) { return response; } +/** + * Fireworks AI Provider (serverless inference) + * + * Fireworks exposes an OpenAI-compatible Chat Completions API + * (https://api.fireworks.ai/inference/v1/chat/completions, bearer-token + * auth). Modeled on invokeBaidu: request side reuses the shared + * openrouter-utils converters, response is converted to Anthropic shape + * locally so the orchestrator branch is a plain passthrough. + * + * Model ids are long-form serverless paths + * (accounts/fireworks/models/, accounts/fireworks/routers/). + * The modelMap below covers bare Anthropic names; tier-selected Fireworks + * ids (e.g. TIER_COMPLEX=fireworks:accounts/fireworks/models/glm-5p2) + * reach the wire unchanged. + * + * NOTE (E2E-unverified as of this addition, same caveat Baidu shipped + * with): sampling quirks, reasoning_content emission on reasoning models, + * finish_reason shape on tool calls, and 429 retryability are best-effort + * from public docs, not yet probed against a live key. Revisit once real + * traffic surfaces 400s — see Moonshot's kimi-k* precedent for the shape + * such fixes take. + */ +async function invokeFireworks(body, _incomingHeaders = {}) { + if (!config.fireworks?.apiKey) { + throw new Error("Fireworks API key is not configured. Set FIREWORKS_API_KEY in your .env file."); + } + + const { + convertAnthropicToolsToOpenRouter, + convertAnthropicMessagesToOpenRouter + } = require("./openrouter-utils"); + + const endpoint = config.fireworks.endpoint || "https://api.fireworks.ai/inference/v1/chat/completions"; + + // Model mapping: Anthropic names → Fireworks serverless ids. + const modelMap = { + "claude-sonnet-4-5-20250929": "accounts/fireworks/models/kimi-k2-instruct-0905", + "claude-sonnet-4-5": "accounts/fireworks/models/kimi-k2-instruct-0905", + "claude-sonnet-4.5": "accounts/fireworks/models/kimi-k2-instruct-0905", + "claude-3-5-sonnet": "accounts/fireworks/models/kimi-k2-instruct-0905", + "claude-opus-4-5": "accounts/fireworks/models/glm-5p2", + "claude-opus-4.5": "accounts/fireworks/models/glm-5p2", + "claude-3-opus": "accounts/fireworks/models/glm-5p2", + "claude-haiku-4-5-20251001": "accounts/fireworks/models/llama-3.1-8b-instruct", + "claude-haiku-4-5": "accounts/fireworks/models/llama-3.1-8b-instruct", + "claude-3-haiku": "accounts/fireworks/models/llama-3.1-8b-instruct", + }; + + const requestedModel = body._tierModel || body.model || config.fireworks.model; + + // Honor tier-selected Fireworks ids instead of silently swapping in the + // .env default model. Accepts full serverless paths as well as well-known + // open-weight family slugs hosted on Fireworks. + const FIREWORKS_ID_RE = /^(accounts\/|kimi-|glm-|deepseek-|qwen|llama-|mistral-|mixtral-|phi-|gemma-|grok-|solar-|yi-|fireworks-)/i; + const mappedModel = modelMap[requestedModel] + || (FIREWORKS_ID_RE.test(requestedModel || "") ? requestedModel : null) + || config.fireworks.model + || "accounts/fireworks/models/kimi-k2-instruct-0905"; + + const messages = convertAnthropicMessagesToOpenRouter(body.messages || []); + + // Fireworks supports the system role natively. + if (body.system) { + const systemContent = Array.isArray(body.system) + ? body.system.map(s => s.text || s).join("\n") + : body.system; + messages.unshift({ role: "system", content: systemContent }); + } + + const { resolveThinkingParam } = require("./provider-capabilities"); + const fireworksBody = { + model: mappedModel, + messages, + max_tokens: body.max_tokens || 16384, + temperature: body.temperature ?? 0.7, + top_p: body.top_p ?? 1.0, + // Reasoning models (R1/GLM) share the answer budget with thinking output; + // the buffered path lifts reasoning_content into thinking blocks. + thinking: resolveThinkingParam(body), + // Streaming honored once "fireworks" joins DEFAULT_OPENAI_SSE_PROVIDERS + // (sse-transformer.js). Buffered requests use the Anthropic conversion + // path below regardless. + stream: body.stream ?? false, + }; + + if (Array.isArray(body.tools) && body.tools.length > 0) { + fireworksBody.tools = convertAnthropicToolsToOpenRouter(body.tools); + fireworksBody.tool_choice = "auto"; + fireworksBody.parallel_tool_calls = false; + } + + const headers = { + "Content-Type": "application/json", + "Authorization": `Bearer ${config.fireworks.apiKey}`, + }; + + logger.debug({ + endpoint, + model: fireworksBody.model, + originalModel: requestedModel, + messageCount: fireworksBody.messages?.length || 0, + hasTools: !!fireworksBody.tools, + toolCount: fireworksBody.tools?.length || 0, + }, "=== Fireworks REQUEST ==="); + + // No retryableStatusesOverride: Fireworks serverless 429s are standard + // per-minute rate limits (unlike Moonshot's persistent org quotas), so the + // default retry-with-backoff applies. A 429 that survives retries throws + // below with status set so tier-fallback climbs instead of hanging. + const response = await performJsonRequest(endpoint, { + headers, + body: fireworksBody, + }, "Fireworks"); + + if (!response.ok && response.status === 429) { + const err = new Error(`Fireworks rate-limited: ${String(response.json?.error?.message || '').slice(0, 120)}`); + err.status = 429; + throw err; + } + + // Streaming request: hand the raw stream to the orchestrator's stream + // branch. The Anthropic conversion below is buffered-only. + if (response?.stream) { + return response; + } + + if (response?.ok && response?.json) { + const anthropicJson = convertOpenAIToAnthropic(response.json); + return { + ok: response.ok, + status: response.status, + json: anthropicJson, + text: JSON.stringify(anthropicJson), + contentType: "application/json", + headers: response.headers, + }; + } + + return response; +} + /** * Convert OpenAI response to Anthropic format */ @@ -3144,6 +3285,7 @@ const PROVIDER_INVOKERS = { moonshot: invokeMoonshot, codex: invokeCodex, baidu: invokeBaidu, + fireworks: invokeFireworks, }; function invokeProvider(provider, body, incomingHeaders) { @@ -4013,6 +4155,7 @@ module.exports = { invokeOllama, invokeMoonshot, invokeBaidu, + invokeFireworks, invokeAtlas, PROVIDER_INVOKERS, stripLynkrBadges, diff --git a/src/config/index.js b/src/config/index.js index aafc1d9..35ecedd 100644 --- a/src/config/index.js +++ b/src/config/index.js @@ -62,7 +62,7 @@ function resolveConfigPath(targetPath) { return path.resolve(normalised); } -const SUPPORTED_MODEL_PROVIDERS = new Set(["databricks", "azure-anthropic", "ollama", "openrouter", "edenai", "azure-openai", "openai", "atlas", "llamacpp", "lmstudio", "bedrock", "zai", "vertex", "moonshot", "baidu"]); +const SUPPORTED_MODEL_PROVIDERS = new Set(["databricks", "azure-anthropic", "ollama", "openrouter", "edenai", "azure-openai", "openai", "atlas", "llamacpp", "lmstudio", "bedrock", "zai", "vertex", "moonshot", "baidu", "fireworks"]); const rawModelProvider = (process.env.MODEL_PROVIDER ?? "databricks").toLowerCase(); // Validate MODEL_PROVIDER early with a clear error message @@ -153,6 +153,11 @@ const baiduApiKey = process.env.BAIDU_API_KEY?.trim() || null; const baiduEndpoint = process.env.BAIDU_ENDPOINT?.trim() || "https://qianfan.baidubce.com/v2/chat/completions"; const baiduModel = process.env.BAIDU_MODEL?.trim() || "ernie-4.5-turbo-128k"; +// Fireworks AI configuration - OpenAI-compatible serverless inference API +const fireworksApiKey = process.env.FIREWORKS_API_KEY?.trim() || null; +const fireworksEndpoint = process.env.FIREWORKS_ENDPOINT?.trim() || "https://api.fireworks.ai/inference/v1/chat/completions"; +const fireworksModel = process.env.FIREWORKS_MODEL?.trim() || "accounts/fireworks/models/kimi-k2-instruct-0905"; + // Vertex AI (Google Gemini) configuration const vertexApiKey = process.env.VERTEX_API_KEY?.trim() || process.env.GOOGLE_API_KEY?.trim() || null; const vertexModel = process.env.VERTEX_MODEL?.trim() || "gemini-2.0-flash"; @@ -662,6 +667,11 @@ var config = { endpoint: baiduEndpoint, model: baiduModel, }, + fireworks: { + apiKey: fireworksApiKey, + endpoint: fireworksEndpoint, + model: fireworksModel, + }, codex: { enabled: process.env.CODEX_ENABLED !== "false", binaryPath: process.env.CODEX_BINARY_PATH?.trim() || null, @@ -1125,6 +1135,8 @@ function reloadConfig() { config.moonshot.model = process.env.MOONSHOT_MODEL?.trim() || "kimi-k2-turbo-preview"; config.baidu.apiKey = process.env.BAIDU_API_KEY?.trim() || null; config.baidu.model = process.env.BAIDU_MODEL?.trim() || "ernie-4.5-turbo-128k"; + config.fireworks.apiKey = process.env.FIREWORKS_API_KEY?.trim() || null; + config.fireworks.model = process.env.FIREWORKS_MODEL?.trim() || "accounts/fireworks/models/kimi-k2-instruct-0905"; // Model provider settings const newProvider = (process.env.MODEL_PROVIDER ?? "databricks").toLowerCase(); diff --git a/src/dashboard/api.js b/src/dashboard/api.js index 66cb006..2dda910 100644 --- a/src/dashboard/api.js +++ b/src/dashboard/api.js @@ -17,6 +17,7 @@ function providerMeta() { 'azure-openai': { type: 'cloud', configured: !!(c.azureOpenAI?.endpoint && c.azureOpenAI?.apiKey) }, vertex: { type: 'cloud', configured: !!c.vertex?.projectId }, moonshot: { type: 'cloud', configured: !!c.moonshot?.apiKey }, + fireworks: { type: 'cloud', configured: !!c.fireworks?.apiKey }, ollama: { type: 'local', configured: !!c.ollama?.endpoint }, llamacpp: { type: 'local', configured: !!c.llamacpp?.endpoint }, lmstudio: { type: 'local', configured: !!c.lmstudio?.endpoint }, diff --git a/src/orchestrator/index.js b/src/orchestrator/index.js index c24995e..b2afe97 100644 --- a/src/orchestrator/index.js +++ b/src/orchestrator/index.js @@ -59,6 +59,8 @@ function getDestinationUrl(providerType) { return config.moonshot?.endpoint ?? 'unknown'; case 'baidu': return config.baidu?.endpoint ?? 'unknown'; + case 'fireworks': + return config.fireworks?.endpoint ?? 'unknown'; case 'codex': return 'codex://app-server (local process)'; default: @@ -1148,6 +1150,14 @@ function sanitizePayload(payload) { } else { clean.tools = ensureAnthropicToolFormat(clean.tools); } + } else if (providerType === "fireworks") { + // Fireworks supports OpenAI-style tools - keep them in Anthropic format + // They will be converted to OpenAI format in invokeFireworks + if (!Array.isArray(clean.tools) || clean.tools.length === 0) { + delete clean.tools; + } else { + clean.tools = ensureAnthropicToolFormat(clean.tools); + } } else if (providerType === "azure-openai" || providerType === "openai" || providerType === "atlas") { // Azure OpenAI / OpenAI-compatible providers support tools — keep Anthropic format; the // client converts to Chat Completions / Responses format. Without this @@ -2705,6 +2715,12 @@ IMPORTANT TOOL USAGE RULES: if (Array.isArray(anthropicPayload?.content)) { anthropicPayload.content = policy.sanitiseContent(anthropicPayload.content); } + } else if (actualProvider === "fireworks") { + // Fireworks responses are already converted to Anthropic format in invokeFireworks + anthropicPayload = databricksResponse.json; + if (Array.isArray(anthropicPayload?.content)) { + anthropicPayload.content = policy.sanitiseContent(anthropicPayload.content); + } } else if (actualProvider === "codex") { // Codex responses are already in Anthropic format from invokeCodex anthropicPayload = databricksResponse.json; diff --git a/src/orchestrator/sse-transformer.js b/src/orchestrator/sse-transformer.js index 1905fdd..e1bdfc6 100644 --- a/src/orchestrator/sse-transformer.js +++ b/src/orchestrator/sse-transformer.js @@ -46,6 +46,12 @@ const DEFAULT_OPENAI_SSE_PROVIDERS = [ "llamacpp", "moonshot", "baidu", + // fireworks (api.fireworks.ai/inference/v1/chat/completions) is documented + // as OpenAI-compatible SSE — same baidu caveat applies: E2E-unverified + // against a live key as of this addition. If its deltas turn out not to + // match choices[0].delta exactly, remove it from this list rather than + // patching the shared transformer for one provider's quirk. + "fireworks", ]; // llama.cpp specific: reasoning-capable local builds (live-confirmed on this diff --git a/src/routing/capabilities.js b/src/routing/capabilities.js new file mode 100644 index 0000000..eaf9029 --- /dev/null +++ b/src/routing/capabilities.js @@ -0,0 +1,88 @@ +/** + * Capability requirement vector (item 1: HyDRA port, phase 1). + * + * Maps the existing 15-dim weighted analysis (complexity-analyzer.js + * calculateWeightedScore dimensions, each 0-100) onto 4 independent + * capability heads in [0,1]: + * + * - reasoning : multi-step / planning / tradeoff analysis load + * - codegen : code-writing demand (generation + technical + tool depth) + * - debugging : diagnostic load (analysis + technical + domain breadth) + * - tool_use : tool-orchestration load (count, complexity, chaining, history) + * + * Deliberately decoupled from the model catalog: this module never names a + * provider/model/tier. Model capabilities live in + * config/model-capabilities.json and matching lives in shortfall.js, so a + * catalog change is a config edit with zero retraining. + * + * Deliberately excludes risk/agentic overrides: risk-high forces REASONING + * and AUTONOMOUS sets a REASONING floor upstream (routing/index.js). Those + * stay as deterministic post-passes (like HyDRA's health veto), not as + * learned heads — keeping this predictor language- and catalog-invariant. + * The agentic flag only floors tool_use (agentic work always needs tools), + * it never lowers a requirement. + * + * Pure function, no I/O. Never throws: bad input yields the neutral prior + * (all 0.2 ≈ trivial) so callers fail open to cheap routing. + */ + +const HEADS = ['reasoning', 'codegen', 'debugging', 'tool_use']; + +function _clamp01(v) { + const n = Number(v); + if (!Number.isFinite(n)) return 0; + return Math.max(0, Math.min(1, n)); +} + +function _dim01(dimensions, name) { + const v = Number(dimensions?.[name]); + if (!Number.isFinite(v)) return 0; + return Math.max(0, Math.min(1, v / 100)); +} + +/** + * @param {object} args + * @param {object} [args.dimensions] — calculateWeightedScore dimensions (0-100 each) + * @param {object} [args.agenticResult] — { isAgentic } (floors tool_use only) + * @returns {{ reasoning:number, codegen:number, debugging:number, tool_use:number }} + */ +function buildRequirementVector({ dimensions = {}, agenticResult = null } = {}) { + try { + const d = (name) => _dim01(dimensions, name); + + let reasoning = d('multiStepReasoning') * 0.4 + + d('analysisDepth') * 0.4 + + d('promptComplexity') * 0.2; + + let codegen = d('codeGeneration') * 0.5 + + d('technicalDepth') * 0.3 + + d('toolComplexity') * 0.2; + + let debugging = d('analysisDepth') * 0.3 + + d('technicalDepth') * 0.2 + + d('domainSpecificity') * 0.3 + + d('promptComplexity') * 0.2; + + let toolUse = d('toolCount') * 0.3 + + d('toolComplexity') * 0.3 + + d('toolChainPotential') * 0.2 + + d('priorToolUsage') * 0.2; + + // Agentic work always needs tool orchestration — floor only, never lower. + if (agenticResult?.isAgentic) { + toolUse = Math.max(toolUse, 0.6); + } + + const round3 = (v) => Math.round(_clamp01(v) * 1000) / 1000; + return { + reasoning: round3(reasoning), + codegen: round3(codegen), + debugging: round3(debugging), + tool_use: round3(toolUse), + }; + } catch { + return { reasoning: 0.2, codegen: 0.2, debugging: 0.2, tool_use: 0.2 }; + } +} + +module.exports = { HEADS, buildRequirementVector }; diff --git a/src/routing/capability-seeds/benchmarks-dir.js b/src/routing/capability-seeds/benchmarks-dir.js new file mode 100644 index 0000000..8f40d46 --- /dev/null +++ b/src/routing/capability-seeds/benchmarks-dir.js @@ -0,0 +1,58 @@ +/** + * Drop-in benchmark directory loader: data/capability-benchmarks/*.json. + * + * For leaderboards with no stable raw endpoint (TerminalBench, LiveCodeBench + * snapshots, Artificial Analysis exports): download the export once, drop it + * here, and the seed script picks it up. Format per file: + * + * { "source": "terminalbench-2.1", "scale": "percent", + * "results": { "Kimi K2.5": 42.1, "GPT 5.2": 38.0 } } + * + * scale "percent" (0-100) or "fraction" (0-1). Model names are cleaned with + * the SWE-bench cleaner and family-mapped in normalize.js; unmapped names + * surface for review instead of being silently seeded. Directory absent or + * empty → { files: [] } (not an error). + */ + +const fs = require('fs'); +const path = require('path'); +const { cleanName } = require('./swebench'); + +const BENCHMARKS_DIR = path.join(__dirname, '../../../data/capability-benchmarks'); + +function loadBenchmarksDir(dir = BENCHMARKS_DIR) { + try { + if (!fs.existsSync(dir)) return { files: [] }; + const files = []; + for (const name of fs.readdirSync(dir)) { + if (!name.endsWith('.json')) continue; + try { + const raw = JSON.parse(fs.readFileSync(path.join(dir, name), 'utf8')); + if (!raw || typeof raw !== 'object' || !raw.results || typeof raw.results !== 'object') { + files.push({ file: name, status: 'skipped', reason: 'missing results object' }); + continue; + } + const entries = []; + for (const [modelName, score] of Object.entries(raw.results)) { + const cleaned = cleanName(modelName); + const s = Number(score); + if (cleaned && Number.isFinite(s)) entries.push({ name: cleaned, score: s }); + } + files.push({ + file: name, + status: 'ok', + source: String(raw.source || name.replace(/\.json$/, '')), + scale: raw.scale === 'fraction' ? 'fraction' : 'percent', + entries, + }); + } catch (err) { + files.push({ file: name, status: 'skipped', reason: err.message }); + } + } + return { files }; + } catch (err) { + return { files: [], error: err.message }; + } +} + +module.exports = { loadBenchmarksDir, BENCHMARKS_DIR }; diff --git a/src/routing/capability-seeds/family-heuristics.js b/src/routing/capability-seeds/family-heuristics.js new file mode 100644 index 0000000..df4c468 --- /dev/null +++ b/src/routing/capability-seeds/family-heuristics.js @@ -0,0 +1,77 @@ +/** + * Family-ladder heuristics: zero-network capability estimates from model + * family names. Vendors ship in ladders (haiku= 100) return 0.8; + if (gb >= 30) return 0.65; + if (gb >= 10) return 0.5; + if (gb >= 4) return 0.35; + return 0.25; +} + +/** + * @param {string} family — normalized family id (see family.js) + * @returns {null | { reasoning:number, codegen:number, debugging:number, tool_use:number }} + */ +function heuristicCaps(family) { + try { + const f = String(family || '').toLowerCase().trim(); + if (!f || f === 'unknown') return null; + let base = null; + for (const { re, base: b } of LADDER) { + if (re.test(f)) { + base = b; + break; + } + } + if (base === null) base = _sizeBase(f); + if (base === null) return null; + const round3 = (v) => Math.max(FLOOR, Math.min(CAP, Math.round(v * 1000) / 1000)); + const codegen = base + (CODER_RE.test(f) ? 0.05 : 0); + const reasoning = base + (REASONER_RE.test(f) ? 0.05 : 0); + return { + reasoning: round3(reasoning), + codegen: round3(codegen), + debugging: round3(base), + tool_use: round3(base), + }; + } catch { + return null; + } +} + +module.exports = { heuristicCaps }; diff --git a/src/routing/capability-seeds/family.js b/src/routing/capability-seeds/family.js new file mode 100644 index 0000000..d637bcc --- /dev/null +++ b/src/routing/capability-seeds/family.js @@ -0,0 +1,105 @@ +/** + * Model-family normalization for capability seeding. + * + * Capability is a property of the MODEL (weights), not the provider serving + * it: zai:glm-5.2, baidu:glm-5.2 and ollama:glm-5.2 must resolve to one + * family id ("glm-5.2") and therefore identical caps. Provider stays on the + * cost/invocation path only. + * + * Normalization (order matters): + * 1. lowercase + trim, drop @digest suffixes + * 2. last `/`-segment wins (strips org prefixes: zai-org/glm-5.2, + * openai/gpt-4o-mini, anthropic/claude-...) + * 3. Ollama `:tag` becomes `-tag` (tags often carry SIZE: qwen2.5-coder:7b + * → qwen2.5-coder-7b; :latest/:cloud are noise but harmless post-strip) + * 4. strip serving prefixes (databricks-, anthropic., bedrock/) + * 5. `_` → `-`, digit-dash-digit → digit.dot.digit + * (gpt-3-5-turbo → gpt-3.5-turbo, llama-3-1-70b → llama-3.1-70b), + * collapse repeats + * + * Quantization is detected, not normalized away: Q4_K_M / gguf / int4 / awq + * style markers set quant:true so the resolver can apply the small + * same-brain-smaller-body haircut. Full-precision self-hosted hits the same + * caps as the API. + * + * Pure functions, no I/O. Never throws. + */ + +const SERVING_PREFIXES = ['databricks-', 'anthropic.', 'bedrock/']; + +// Quant markers: matched against the full "model:tag" string BEFORE tag +// folding, and against the folded family after (tags like Q4_K_M survive as +// -q4-k-m, so one pass after folding suffices — but pre-tag names like +// "model-Q4" exist too, hence match post-fold only, on the whole string). +const QUANT_RE = /(q[23468](_|-|$)|_k_[sm]|[-_]q8_0|gguf|int[48]|fp[148]|awq|gptq|bnb|mlx|quantized)/i; + +// Tags that carry no signal even as suffixes. +const NOISE_TAGS = new Set(['latest', 'cloud', 'instruct']); + +function detectQuant(s) { + try { + return QUANT_RE.test(String(s || '')); + } catch { + return false; + } +} + +/** + * @param {string} provider + * @param {string} model + * @returns {{ family:string, quant:boolean }} + */ +function normalizeFamily(provider, model) { + try { + let s = `${String(model || '').trim()}`; + // @sha256:... digests + s = s.split('@')[0]; + // org prefix: last segment wins + if (s.includes('/')) s = s.split('/').pop(); + const quant = detectQuant(`${provider || ''}/${model || ''}`) || detectQuant(s); + // Ollama :tag → -tag (size tags preserved: :7b → -7b) + if (s.includes(':')) { + const [base, ...tags] = s.split(':'); + const kept = tags + .map((t) => t.trim().toLowerCase()) + .filter((t) => t && !NOISE_TAGS.has(t)); + s = kept.length > 0 ? `${base}-${kept.join('-')}` : base; + } + s = s.toLowerCase().trim(); + for (const p of SERVING_PREFIXES) { + if (s.startsWith(p)) { + s = s.slice(p.length); + break; + } + } + s = s.replace(/_/g, '-'); + // digit-dash-digit → digit.dot.digit, but only for version runs: the + // second digit must be followed by another dash or end-of-string, so + // size suffixes survive (llama-3-1-70b → llama-3.1-70b, but + // qwen3-32b and gemma-3-27b keep their dash). + s = s.replace(/(\d)-(?=\d(?:-|$))/g, '$1.'); + s = s.replace(/-+/g, '-').replace(/^-|-$/g, ''); + return { family: s || 'unknown', quant }; + } catch { + return { family: 'unknown', quant: false }; + } +} + +/** + * Quant haircut: same brain, smaller body. Applied by the resolver to + * seed/family/tier caps — never to explicit operator overrides (an exact + * provider:model override wins verbatim). + */ +const QUANT_HAIRCUT = 0.02; +const QUANT_FLOOR = 0.1; + +function applyQuantHaircut(caps) { + const out = {}; + for (const [k, v] of Object.entries(caps || {})) { + const n = Number(v); + out[k] = Number.isFinite(n) ? Math.max(QUANT_FLOOR, Math.round((n - QUANT_HAIRCUT) * 1000) / 1000) : v; + } + return out; +} + +module.exports = { normalizeFamily, detectQuant, applyQuantHaircut, QUANT_HAIRCUT }; diff --git a/src/routing/capability-seeds/models-dev.js b/src/routing/capability-seeds/models-dev.js new file mode 100644 index 0000000..69222bb --- /dev/null +++ b/src/routing/capability-seeds/models-dev.js @@ -0,0 +1,64 @@ +/** + * models.dev adapter: boolean capability flags per model (tool_call, + * vision/image input, reasoning, context window). Same public endpoint the + * pricing registry already consumes (https://models.dev/api.json), same + * shape ({ providerId: { models: { modelId: {...} } } }). + * + * Flags don't set caps — they GATE them: a model with tool_call=false gets + * tool_use capped (it can still drive tools via XML extraction, which is why + * this is a cap, not a zero). Everything else passes through as provenance + * for operator review. + * + * Fail-soft: { status:'skipped', reason } on any problem. + */ + +const { fetchJson, getCached, setCached, skipped } = require('./sources'); +const { normalizeFamily } = require('./family'); + +const MODELS_DEV_URL = 'https://models.dev/api.json'; + +// Conservative ceiling for models without function-calling. XML-extracted +// tool calls (see xml-tool-extractor.js) keep them usable, hence 0.4 not 0. +const NO_TOOLCALL_TOOL_USE_CAP = 0.4; + +/** + * @param {object} [opts] — { refresh:boolean } + * @returns {Promise<{ status, flags?:Object, reason? }>} + */ +async function fetchModelsDevFlags({ refresh = true } = {}) { + const parse = (data) => { + const out = {}; + for (const [providerId, providerData] of Object.entries(data || {})) { + if (!providerData?.models) continue; + for (const [modelId, info] of Object.entries(providerData.models)) { + if (!info || typeof info !== 'object') continue; + const { family } = normalizeFamily(providerId, modelId); + if (family === 'unknown' || out[family]) continue; // first provider wins + out[family] = { + toolCall: info.tool_call ?? null, + vision: Array.isArray(info.input) ? info.input.includes('image') : null, + reasoning: info.reasoning ?? null, + context: Number(info.context) || null, + }; + } + } + return out; + }; + + if (!refresh) { + const cached = getCached('modelsdev'); + if (cached) return { status: 'ok', flags: cached, cached: true }; + return skipped('no cache (run with --refresh)'); + } + try { + const flags = parse(await fetchJson(MODELS_DEV_URL)); + setCached('modelsdev', flags); + return { status: 'ok', flags }; + } catch (err) { + const cached = getCached('modelsdev'); + if (cached) return { status: 'ok', flags: cached, cached: true, stale: err.message }; + return skipped(`fetch failed and no cache: ${err.message}`); + } +} + +module.exports = { fetchModelsDevFlags, MODELS_DEV_URL, NO_TOOLCALL_TOOL_USE_CAP }; diff --git a/src/routing/capability-seeds/normalize.js b/src/routing/capability-seeds/normalize.js new file mode 100644 index 0000000..b47c24d --- /dev/null +++ b/src/routing/capability-seeds/normalize.js @@ -0,0 +1,160 @@ +/** + * Score → capability-cap normalization + leaderboard-name → family mapping. + * + * Anchors (fixed permanently — τ absorbs residual error, consistency matters + * more than precision): benchmark fraction 0.8 (≈ flagship SWE-Verified + * territory) → caps 0.9; fraction 0 → 0.15 (never zero: even weak models do + * trivial turns). Linear between, clamped to [0.15, 0.9]. + * + * Per-head source weights (v1 heuristic, documented): + * debugging = 0.6·swe + 0.4·terminal + * codegen = 0.5·livecode + 0.3·swe + 0.2·arena + * reasoning = 0.5·swe + 0.3·arena + 0.2·livecode + * tool_use = 0.6·terminal + 0.4·swe + * Missing sources renormalize over the present ones. A pessimistic haircut + * (-0.03) applies to all normalized caps: underestimating a new model costs + * money (extra escalations), overestimating costs quality. + * + * Family mapping: order-insensitive token overlap between the normalized + * family id and the leaderboard name. Every family token must hit an entry + * token exactly or as a version prefix ("4" matches "4.5", "k2" matches + * "k2.5" — but "32b" never matches "480b"). Best overlap wins; weak matches + * land in `unmapped` for operator review instead of being seeded silently. + * Pure functions, never throw. + */ + +const { normalizeFamily } = require('./family'); + +const CAP_FLOOR = 0.15; +const CAP_CEIL = 0.9; +const NORMALIZE_HAIRCUT = 0.03; + +// score fraction (0-1) at which caps hit the ceiling — flagship territory. +const ANCHOR_SCORE = 0.8; + +const HEAD_WEIGHTS = { + debugging: { swe: 0.6, terminal: 0.4, livecode: 0, arena: 0 }, + codegen: { swe: 0.3, terminal: 0, livecode: 0.5, arena: 0.2 }, + reasoning: { swe: 0.5, terminal: 0, livecode: 0.2, arena: 0.3 }, + tool_use: { swe: 0.4, terminal: 0.6, livecode: 0, arena: 0 }, +}; + +const STOPWORDS = new Set([ + 'the', 'model', 'preview', 'high', 'medium', 'low', + 'agent', 'instruct', 'thinking', + '2024', '2025', '2026', +]); + +function entryTokens(name) { + return String(name || '') + .toLowerCase() + .split(/[^a-z0-9.]+/) + .map((t) => t.replace(/^\.+|\.+$/g, '')) + .filter((t) => t && !STOPWORDS.has(t)); +} + +function _tokenHit(familyTok, entryToks) { + for (const tok of entryToks) { + if (tok === familyTok) return true; + // version prefix: family "4" hits entry "4.5"; family "k2" hits "k2.5". + // The char after the prefix must be a dot (never a letter/digit, so + // "32b" can't match "480b" and "gpt-5" can't match "gpt-50"). + if (tok.length > familyTok.length && tok.startsWith(familyTok) && tok[familyTok.length] === '.') { + return true; + } + // reverse: entry "4" satisfies family "4.5"? No — a bare major never + // proves the minor. Skip. + } + return false; +} + +function familyTokens(family) { + return String(family || '') + .toLowerCase() + .split(/[^a-z0-9.]+/) + .map((t) => t.replace(/^\.+|\.+$/g, '')) + .filter((t) => t && !STOPWORDS.has(t)); +} + +/** + * @param {string} entryName — cleaned leaderboard name + * @param {string[]} knownFamilies — candidate family ids (seed keys w/o wildcards) + * @returns {{ family:string|null, hits:number, of:number }} + */ +function mapEntryToFamily(entryName, knownFamilies) { + try { + const toks = entryTokens(entryName); + if (toks.length === 0 || !Array.isArray(knownFamilies)) { + return { family: null, hits: 0, of: 0 }; + } + let best = { family: null, hits: 0, of: 0, ratio: 0 }; + for (const fam of knownFamilies) { + const ftoks = familyTokens(fam); + if (ftoks.length === 0) continue; + let hits = 0; + for (const ft of ftoks) { + if (_tokenHit(ft, toks)) hits++; + } + const ratio = hits / ftoks.length; + // Full coverage required; ties prefer the more specific family. + if (ratio === 1 && (best.family === null || ftoks.length > best.of)) { + best = { family: fam, hits, of: ftoks.length, ratio }; + } + } + return best.family ? best : { family: null, hits: 0, of: 0 }; + } catch { + return { family: null, hits: 0, of: 0 }; + } +} + +function scoreToCap(score) { + const s = Math.max(0, Math.min(1, Number(score))); + if (!Number.isFinite(s)) return null; + const cap = CAP_FLOOR + (s / ANCHOR_SCORE) * (CAP_CEIL - CAP_FLOOR); + return Math.max(CAP_FLOOR, Math.min(CAP_CEIL, cap)); +} + +/** + * @param {object} scores — { swe?, terminal?, livecode?, arena? } fractions 0-1 + * @returns {null | { reasoning, codegen, debugging, tool_use }} + */ +function scoresToCaps(scores) { + try { + if (!scores || typeof scores !== 'object') return null; + const present = {}; + for (const k of ['swe', 'terminal', 'livecode', 'arena']) { + const v = Number(scores[k]); + if (Number.isFinite(v)) present[k] = Math.max(0, Math.min(1, v)); + } + if (Object.keys(present).length === 0) return null; + const caps = {}; + for (const [head, weights] of Object.entries(HEAD_WEIGHTS)) { + let total = 0; + let wsum = 0; + for (const [src, w] of Object.entries(weights)) { + if (present[src] !== undefined && w > 0) { + total += scoreToCap(present[src]) * w; + wsum += w; + } + } + if (wsum === 0) return null; + caps[head] = Math.round((total / wsum - NORMALIZE_HAIRCUT) * 1000) / 1000; + caps[head] = Math.max(CAP_FLOOR, Math.min(CAP_CEIL, caps[head])); + } + return caps; + } catch { + return null; + } +} + +module.exports = { + mapEntryToFamily, + scoresToCaps, + scoreToCap, + entryTokens, + familyTokens, + normalizeFamily, + CAP_FLOOR, + CAP_CEIL, + NORMALIZE_HAIRCUT, +}; diff --git a/src/routing/capability-seeds/registry.js b/src/routing/capability-seeds/registry.js new file mode 100644 index 0000000..24801e2 --- /dev/null +++ b/src/routing/capability-seeds/registry.js @@ -0,0 +1,125 @@ +/** + * Seed registry: per-FAMILY capability profiles from online-derived data. + * + * Layers (first hit wins in shortfall.js resolveCapabilities): + * operator modelOverrides (provider:model, manual — supreme, lives in + * model-capabilities.json) → snapshot (data/, fetched, `source:'seed:snapshot'`) + * → shipped (config/, reviewed, `source:'seed:shipped'`) → family + * heuristic (`source:'family'`) → tier slot (`source:'tier'`). + * + * Keys are normalized family ids (see family.js), lowercased, with optional + * trailing `*` wildcards matched longest-first (gpt-5* , qwen3-*). Provider + * never appears in keys — zai/baidu/ollama servings of glm-5.2 share one entry. + * + * File lifecycles mirror model-tiers.json: read once at boot, restart to + * pick up edits. data/ is gitignored (operator-local), config/ is versioned. + * Pure lookups after load; load never throws (missing/malformed → empty). + */ + +const fs = require('fs'); +const path = require('path'); +const logger = require('../../logger'); +const { HEADS } = require('../capabilities'); + +const SHIPPED_PATH = path.join(__dirname, '../../../config/model-capability-seeds.json'); +const SNAPSHOT_PATH = path.join(__dirname, '../../../data/capability-seeds.snapshot.json'); + +let _cache = null; + +function sanitizeCaps(raw) { + const caps = {}; + for (const h of HEADS) { + const v = Number(raw?.[h]); + caps[h] = Number.isFinite(v) ? Math.max(0, Math.min(1, v)) : 0.5; + } + return caps; +} + +function _loadSeedsFile(filePath, label) { + try { + if (!fs.existsSync(filePath)) return {}; + const raw = JSON.parse(fs.readFileSync(filePath, 'utf8')); + const seeds = raw?.seeds && typeof raw.seeds === 'object' ? raw.seeds : {}; + const out = {}; + for (const [key, entry] of Object.entries(seeds)) { + const caps = entry?.caps && typeof entry.caps === 'object' ? entry.caps : entry; + if (!caps || typeof caps !== 'object') continue; + out[String(key).toLowerCase()] = { + caps: sanitizeCaps(caps), + sources: Array.isArray(entry?.sources) ? entry.sources : [], + note: typeof entry?.note === 'string' ? entry.note : '', + }; + } + return out; + } catch (err) { + logger.debug({ err: err.message, label }, '[SeedRegistry] seed file load failed — skipping layer'); + return {}; + } +} + +function loadSeedLayers() { + if (_cache) return _cache; + _cache = { + snapshot: _loadSeedsFile(SNAPSHOT_PATH, 'snapshot'), + shipped: _loadSeedsFile(SHIPPED_PATH, 'shipped'), + }; + return _cache; +} + +function _resetSeedsCache() { + _cache = null; +} + +// Test hook: inject layers directly (Operations on the same cached object +// the resolver reads). +function _setSeedsForTests({ snapshot, shipped } = {}) { + const base = loadSeedLayers(); + _cache = { + snapshot: snapshot ?? base.snapshot, + shipped: shipped ?? base.shipped, + }; + return _cache; +} + +function _matchWildcard(layer, family) { + let best = null; + for (const [key, entry] of Object.entries(layer)) { + if (!key.endsWith('*')) continue; + const prefix = key.slice(0, -1); + if (prefix && family.startsWith(prefix) && (!best || prefix.length > best.prefix.length)) { + best = { prefix, entry }; + } + } + return best?.entry ?? null; +} + +/** + * @param {string} family — normalized family id + * @returns {null | { caps, source:'seed:snapshot'|'seed:shipped', sources, note }} + */ +function resolveSeedCaps(family) { + try { + const f = String(family || '').toLowerCase().trim(); + if (!f || f === 'unknown') return null; + const { snapshot, shipped } = loadSeedLayers(); + if (snapshot[f]) return { ...snapshot[f], source: 'seed:snapshot' }; + if (shipped[f]) return { ...shipped[f], source: 'seed:shipped' }; + const snapWild = _matchWildcard(snapshot, f); + if (snapWild) return { ...snapWild, source: 'seed:snapshot' }; + const shipWild = _matchWildcard(shipped, f); + if (shipWild) return { ...shipWild, source: 'seed:shipped' }; + return null; + } catch { + return null; + } +} + +module.exports = { + loadSeedLayers, + _resetSeedsCache, + _setSeedsForTests, + resolveSeedCaps, + sanitizeCaps, + SHIPPED_PATH, + SNAPSHOT_PATH, +}; diff --git a/src/routing/capability-seeds/sources.js b/src/routing/capability-seeds/sources.js new file mode 100644 index 0000000..60334ac --- /dev/null +++ b/src/routing/capability-seeds/sources.js @@ -0,0 +1,69 @@ +/** + * Shared plumbing for capability-seed sources (online benchmark data). + * + * Every adapter is optional and fail-soft: no network, bad payload, or + * unknown shape → { status:'skipped', reason } — never throws, never blocks + * the script, and never touches request-path routing (this only runs in + * scripts/seed-capabilities.js). Fetched payloads are cached under + * data/capability-sources-cache.json with a 7-day TTL so refreshes are + * cheap and CI stays hermetic (tests use fixtures, never the network). + */ + +const fs = require('fs'); +const path = require('path'); + +const CACHE_PATH = path.join(__dirname, '../../../data/capability-sources-cache.json'); +const CACHE_TTL_MS = 7 * 24 * 60 * 60 * 1000; +const FETCH_TIMEOUT_MS = 20000; + +function _readCache() { + try { + if (!fs.existsSync(CACHE_PATH)) return {}; + return JSON.parse(fs.readFileSync(CACHE_PATH, 'utf8')); + } catch { + return {}; + } +} + +function _writeCache(cache) { + try { + fs.mkdirSync(path.dirname(CACHE_PATH), { recursive: true }); + fs.writeFileSync(CACHE_PATH, JSON.stringify(cache, null, 2)); + } catch { + // cache is best-effort; a failed write must not fail seeding + } +} + +function getCached(source) { + const cache = _readCache(); + const entry = cache[source]; + if (!entry || typeof entry !== 'object') return null; + if (Date.now() - (entry.fetchedAt || 0) > CACHE_TTL_MS) return null; + return entry.payload ?? null; +} + +function setCached(source, payload) { + const cache = _readCache(); + cache[source] = { fetchedAt: Date.now(), payload }; + _writeCache(cache); +} + +async function fetchJson(url, { timeoutMs = FETCH_TIMEOUT_MS } = {}) { + const response = await fetch(url, { + signal: AbortSignal.timeout(timeoutMs), + headers: { Accept: 'application/json' }, + }); + if (!response.ok) throw new Error(`HTTP ${response.status}`); + return response.json(); +} + +const skipped = (reason) => ({ status: 'skipped', reason }); + +module.exports = { + CACHE_PATH, + CACHE_TTL_MS, + getCached, + setCached, + fetchJson, + skipped, +}; diff --git a/src/routing/capability-seeds/swebench.js b/src/routing/capability-seeds/swebench.js new file mode 100644 index 0000000..bfcd5f7 --- /dev/null +++ b/src/routing/capability-seeds/swebench.js @@ -0,0 +1,66 @@ +/** + * SWE-Bench Verified adapter: per-model % resolved from the public + * leaderboard raw data. + * + * Source: https://raw.githubusercontent.com/SWE-bench/swe-bench.github.io/master/data/leaderboards.json + * Shape: { leaderboards: [{ name: "Verified"|"Lite"|..., results: [{ name, resolved, date, ... }] }] } + * Entry names look like "Claude 4.5 Opus (high) · mini-SWE-agent · 2026-02-17" + * (model + effort qualifier + scaffold + date). We keep the best resolved% + * per cleaned model name; family mapping happens in normalize.js + * (token-overlap against known families, unmapped surfaced for review). + * + * Fail-soft: any fetch/parse problem → { status:'skipped', reason }. + */ + +const { fetchJson, getCached, setCached, skipped } = require('./sources'); + +const LEADERBOARDS_URL = 'https://raw.githubusercontent.com/SWE-bench/swe-bench.github.io/master/data/leaderboards.json'; +const BOARD = 'Verified'; + +function cleanName(name) { + return String(name || '') + .split('·')[0] // drop scaffold + date segments + .replace(/\((high|medium|low)\)/gi, '') // effort qualifier (we keep max across them) + .trim() + .replace(/\s+/g, ' ') + .toLowerCase(); +} + +/** + * @param {object} [opts] — { refresh:boolean } (false = cache-only) + * @returns {Promise<{ status, board?, entries?:Array<{name,resolved,date}>, reason? }>} + */ +async function fetchSweBench({ refresh = true } = {}) { + if (!refresh) { + const cached = getCached('swebench'); + if (cached) return { status: 'ok', board: BOARD, entries: cached, cached: true }; + return skipped('no cache (run with --refresh)'); + } + try { + const data = await fetchJson(LEADERBOARDS_URL); + const boards = Array.isArray(data?.leaderboards) ? data.leaderboards : []; + const verified = boards.find((b) => String(b?.name || '').toLowerCase() === BOARD.toLowerCase()); + if (!verified || !Array.isArray(verified.results)) { + return skipped(`board "${BOARD}" not found in payload`); + } + const best = new Map(); + for (const r of verified.results) { + const name = cleanName(r?.name); + const resolved = Number(r?.resolved); + if (!name || !Number.isFinite(resolved)) continue; + const prev = best.get(name); + if (!prev || resolved > prev.resolved) { + best.set(name, { name, resolved, date: r?.date ?? null }); + } + } + const entries = [...best.values()]; + setCached('swebench', entries); + return { status: 'ok', board: BOARD, entries }; + } catch (err) { + const cached = getCached('swebench'); + if (cached) return { status: 'ok', board: BOARD, entries: cached, cached: true, stale: err.message }; + return skipped(`fetch failed and no cache: ${err.message}`); + } +} + +module.exports = { fetchSweBench, cleanName, LEADERBOARDS_URL, BOARD }; diff --git a/src/routing/index.js b/src/routing/index.js index 355f750..ff375a1 100644 --- a/src/routing/index.js +++ b/src/routing/index.js @@ -1230,6 +1230,91 @@ async function _determineProviderSmartInner(payload, options = {}) { selectedModel = modelSelection.model; logger.debug({ tier, provider, model: selectedModel }, '[Routing] Using tier config'); + // Item 1 — capability-decoupled shortfall matching (HyDRA port). + // Shadow-computes whenever possible (weighted mode only); serves only when + // config/model-capabilities.json has enabled:true. Never overrides risk-high (returned earlier + // upstream), static mode (tier null), or legacy scorer mode. Downstream + // guards (de-escalation, context, vision, kNN, bandit, deadline, tenant) + // still run after and dominate — shortfall only replaces the tier_config pick. + let shortfallInfo = null; + try { + if (tier && config.modelTiers?.enabled && risk?.level !== 'high' + && analysis?.mode === 'weighted' && analysis?.breakdown) { + const { buildRequirementVector } = require('./capabilities'); + const sf = require('./shortfall'); + const req = buildRequirementVector({ dimensions: analysis.breakdown, agenticResult }); + // Candidates constrained to the user's TIER_* (same eligibility rule + // as the bandit in decide.js) with tier labels attached for capability + // resolution. Dedupe identical provider:model keeping the highest tier. + const seen = new Map(); + for (const t of ['SIMPLE', 'MEDIUM', 'COMPLEX', 'REASONING']) { + for (const m of selector.getModelsForTier(t)) { + const key = `${m.provider}:${m.model}`; + const prev = seen.get(key); + if (!prev || (TIER_DEFINITIONS[t]?.priority || 0) > (TIER_DEFINITIONS[prev.tier]?.priority || 0)) { + seen.set(key, { provider: m.provider, model: m.model, tier: t }); + } + } + } + let registry = null; + try { registry = require('./model-registry').getModelRegistrySync(); } catch { registry = null; } + let optimizer = null; + try { optimizer = require('./cost-optimizer').getCostOptimizer(); } catch { optimizer = null; } + const candidates = [...seen.values()].map((c) => { + let cost = Number.POSITIVE_INFINITY; // unknown price never wins on cheapness + try { + const info = registry?.getCost?.(c.model); + if (info && !info.unknown) { + const est = optimizer?.estimateCost?.(c.model, 1000); + cost = Number.isFinite(est?.totalEstimate) + ? est.totalEstimate + : (Number(info.input) || 0) + (Number(info.output) || 0); + } + } catch { /* keep Infinity */ } + return { ...c, cost }; + }); + const result = sf.selectByShortfall(req, candidates); + if (result) { + const agreed = result.selected.provider === provider && result.selected.model === selectedModel; + shortfallInfo = { + req, + tau: result.tau, + selected: result.selected, + agreed, + legacy: { provider, model: selectedModel, tier }, + }; + logger.debug({ + req, + tau: result.tau, + legacy: `${tier}:${provider}:${selectedModel}`, + shortfall: `${result.selected.tier}:${result.selected.provider}:${result.selected.model}`, + agreed, + }, '[Routing] Shortfall shadow compare'); + if (sf.isEnabled() && !agreed) { + const fromTier = tier; + const fromModel = selectedModel; + provider = result.selected.provider; + selectedModel = result.selected.model; + tier = result.selected.tier; + analysis.tier = tier; + method = method + '+shortfall'; + if ((TIER_DEFINITIONS[tier]?.priority || 0) > (TIER_DEFINITIONS[fromTier]?.priority || 0)) { + escalations.push({ + source: 'shortfall', + fromTier, + toTier: tier, + fromModel, + toModel: selectedModel, + }); + } + logger.info({ from: `${fromTier}:${fromModel}`, to: `${tier}:${selectedModel}` }, '[Routing] Shortfall override'); + } + } + } + } catch (err) { + degradation.record('shortfall', err); + } + // WS2.3 — evidence-based de-escalation. // // The check is intentionally gated by evidence, not a feature flag: the @@ -1599,6 +1684,7 @@ async function _determineProviderSmartInner(payload, options = {}) { knnResult, base_tier: baseTier, escalations, + shortfall: shortfallInfo, // Upward escalations take precedence in the source label; a demotion is // only surfaced when nothing else escalated (guarded by the wire above, // but re-checked here for clarity). diff --git a/src/routing/model-tiers.js b/src/routing/model-tiers.js index 7b89fbd..f098909 100644 --- a/src/routing/model-tiers.js +++ b/src/routing/model-tiers.js @@ -353,6 +353,8 @@ class ModelTierSelector { return config.moonshot?.model || null; case 'baidu': return config.baidu?.model || null; + case 'fireworks': + return config.fireworks?.model || null; case 'codex': return config.codex?.model || null; case 'vertex': diff --git a/src/routing/shortfall.js b/src/routing/shortfall.js new file mode 100644 index 0000000..c72e41c --- /dev/null +++ b/src/routing/shortfall.js @@ -0,0 +1,290 @@ +/** + * Shortfall matching (item 1: HyDRA port, phase 1). + * + * Selects the cheapest candidate whose capabilities cover the predicted + * requirement vector within tolerance τ: + * + * shortfall(m) = Σ w_k · max(0, req_k − cap_mk) + * + * All four heads share the [0,1] bandwidth, so v1 weights are uniform + * (band compensation is a no-op until heads diverge — noted for phase 2). + * + * Contracts: + * - Catalog-decoupled: candidates arrive as [{provider, model, tier}]; + * capabilities resolve per MODEL FAMILY (provider never matters: + * zai/baidu/ollama servings of glm-5.2 share caps) via operator + * modelOverrides → seed snapshot/shipped → family heuristic → tier + * profile. Adding, removing, or repricing a model re-routes via config + * with zero retraining. + * - Self-contained config: enabled/tau/weights also live in + * config/model-capabilities.json — no env vars. The file is read once at + * boot (same lifecycle as config/model-tiers.json); restart to pick up + * edits. + * - Cost comes from the caller (model-registry) or per-1k blended estimate; + * unknown cost sorts last (conservative — never wins on cheapness alone). + * - No covering candidate (all shortfalls > τ) → minimal shortfall wins, + * tiebreak higher tier (correctness over cost, matches tier-fallback.js + * escalate-then-demote bias). Never returns null on valid input; returns + * null only on malformed input so callers fail open to legacy routing. + * - Pure except for config load (cached). Never throws. + */ + +const fs = require('fs'); +const path = require('path'); +const logger = require('../logger'); +const { HEADS } = require('./capabilities'); + +const PROFILES_PATH = path.join(__dirname, '../../config/model-capabilities.json'); + +const DEFAULT_TAU = 0.24; // HyDRA iso-quality operating point +const TIER_PRIORITY = { SIMPLE: 1, MEDIUM: 2, COMPLEX: 3, REASONING: 4 }; + +let _profilesCache = null; + +function _defaultWeights() { + const w = {}; + for (const h of HEADS) w[h] = 1 / HEADS.length; + return w; +} + +function _sanitizeCaps(raw) { + const caps = {}; + for (const h of HEADS) { + const v = Number(raw?.[h]); + caps[h] = Number.isFinite(v) ? Math.max(0, Math.min(1, v)) : 0.5; + } + return caps; +} + +function loadProfiles() { + if (_profilesCache) return _profilesCache; + try { + const raw = JSON.parse(fs.readFileSync(PROFILES_PATH, 'utf8')); + const tierProfiles = {}; + const fileTiers = raw?.tierProfiles && typeof raw.tierProfiles === 'object' + ? raw.tierProfiles + : {}; + for (const [tier, caps] of Object.entries(fileTiers)) { + if (caps && typeof caps === 'object') tierProfiles[tier] = _sanitizeCaps(caps); + } + const modelOverrides = {}; + const fileOverrides = raw?.modelOverrides && typeof raw.modelOverrides === 'object' + ? raw.modelOverrides + : {}; + for (const [key, caps] of Object.entries(fileOverrides)) { + if (caps && typeof caps === 'object') modelOverrides[String(key).toLowerCase()] = _sanitizeCaps(caps); + } + const tau = Number(raw?.tau); + const weights = _normalizeWeights(raw?.weights); + _profilesCache = { + tierProfiles, + modelOverrides, + enabled: raw?.enabled === true, + tau: Number.isFinite(tau) && tau >= 0 ? tau : DEFAULT_TAU, + weights, + }; + } catch (err) { + logger.debug({ err: err.message }, '[Shortfall] profiles load failed — disabled with tier fallbacks'); + _profilesCache = { tierProfiles: {}, modelOverrides: {}, enabled: false, tau: DEFAULT_TAU, weights: _defaultWeights() }; + } + return _profilesCache; +} + +// Exposed for tests (cache reset / profile injection). +function _resetProfilesCache() { + _profilesCache = null; +} + +function _setProfilesForTests(profiles) { + const base = loadProfiles(); + _profilesCache = { + tierProfiles: profiles?.tierProfiles ?? base.tierProfiles, + modelOverrides: profiles?.modelOverrides ?? base.modelOverrides, + enabled: profiles?.enabled ?? base.enabled, + tau: profiles?.tau ?? base.tau, + weights: profiles?.weights ?? base.weights, + }; + return _profilesCache; +} + +function _normalizeWeights(raw) { + const w = _defaultWeights(); + if (!raw || typeof raw !== 'object') return w; + let touched = false; + for (const h of HEADS) { + const v = Number(raw[h]); + if (Number.isFinite(v) && v >= 0) { + w[h] = v; + touched = true; + } + } + if (!touched) return _defaultWeights(); + const sum = Object.values(w).reduce((a, b) => a + b, 0); + if (sum <= 0) return _defaultWeights(); + for (const h of HEADS) w[h] = w[h] / sum; + return w; +} + +function getTau() { + return loadProfiles().tau; +} + +function getWeights() { + return { ...loadProfiles().weights }; +} + +function isEnabled() { + return loadProfiles().enabled === true; +} + +/** + * Resolve capabilities for one candidate. Precedence (first hit wins): + * 1. operator "provider:model" / "provider:*" override (manual, supreme, + * returned verbatim — not even the quant haircut applies) + * 2. seed snapshot / shipped seeds by normalized MODEL FAMILY + * (zai/baidu/ollama servings of glm-5.2 share one entry) + * 3. family-ladder heuristic (zero-network estimate, never frontier) + * 4. tier profile → tier-priority fallback (SIMPLE 0.2 … REASONING 0.9) + * + * Quantized self-hosted servings (Q4_K_M, gguf, …) take a small haircut at + * every level except 1: same brain, smaller body. + * + * @returns {{ caps, source }} — source is override|seed:snapshot| + * seed:shipped|family|tier|tier-fallback (telemetry provenance). + */ +function resolveCapabilitiesWithSource({ provider, model, tier }) { + const { tierProfiles, modelOverrides } = loadProfiles(); + const key = `${String(provider || '').toLowerCase()}:${String(model || '').toLowerCase()}`; + const wild = `${String(provider || '').toLowerCase()}:*`; + const pick = modelOverrides[key] || modelOverrides[wild]; + if (pick) return { caps: { ...pick }, source: 'override' }; + + let family = 'unknown'; + let quant = false; + try { + const fam = require('./capability-seeds/family'); + ({ family, quant } = fam.normalizeFamily(provider, model)); + } catch { /* family helpers unavailable — tier fallback below */ } + + try { + const { resolveSeedCaps } = require('./capability-seeds/registry'); + const seed = resolveSeedCaps(family); + if (seed) { + const caps = quant + ? require('./capability-seeds/family').applyQuantHaircut(seed.caps) + : { ...seed.caps }; + return { caps, source: seed.source }; + } + } catch { /* seed layers unavailable — keep falling through */ } + + try { + const { heuristicCaps } = require('./capability-seeds/family-heuristics'); + const heur = heuristicCaps(family); + if (heur) { + const caps = quant + ? require('./capability-seeds/family').applyQuantHaircut(heur) + : heur; + return { caps, source: 'family' }; + } + } catch { /* heuristic unavailable — tier fallback below */ } + + const tp = tierProfiles[tier]; + if (tp) { + const caps = quant + ? require('./capability-seeds/family').applyQuantHaircut(tp) + : { ...tp }; + return { caps, source: 'tier' }; + } + const p = (TIER_PRIORITY[tier] || 1) / 4; + const caps = {}; + for (const h of HEADS) caps[h] = Math.round(p * 1000) / 1000; + return { caps, source: 'tier-fallback' }; +} + +function resolveCapabilities(candidate) { + return resolveCapabilitiesWithSource(candidate).caps; +} + +function shortfall(req, caps, weights = null) { + const w = weights || _defaultWeights(); + let s = 0; + for (const h of HEADS) { + const r = Math.max(0, Math.min(1, Number(req?.[h]) || 0)); + const c = Math.max(0, Math.min(1, Number(caps?.[h]) || 0)); + s += (w[h] ?? 0) * Math.max(0, r - c); + } + return Math.round(s * 10000) / 10000; +} + +function _costValue(c) { + const n = Number(c); + return Number.isFinite(n) && n >= 0 ? n : Number.POSITIVE_INFINITY; +} + +/** + * @param {object} req — requirement vector {reasoning, codegen, debugging, tool_use} in [0,1] + * @param {Array<{provider, model, tier, cost?}>} candidates — catalog-constrained set + * (callers pass getAllConfiguredModels() + model-registry costs) + * @param {object} [opts] — { tau, weights } + * @returns {null | { selected, shortfalls: Array<{provider, model, tier, shortfall, cost, source}>, tau }} + */ +function selectByShortfall(req, candidates, opts = {}) { + try { + if (!req || typeof req !== 'object') return null; + if (!Array.isArray(candidates) || candidates.length === 0) return null; + const tau = opts.tau ?? getTau(); + const weights = opts.weights ?? getWeights(); + + const rows = candidates + .filter((c) => c && c.provider && c.model) + .map((c) => { + const { caps, source } = resolveCapabilitiesWithSource(c); + return { + provider: c.provider, + model: c.model, + tier: c.tier || 'MEDIUM', + cost: _costValue(c.cost), + shortfall: shortfall(req, caps, weights), + source, + }; + }); + if (rows.length === 0) return null; + + const covering = rows.filter((r) => r.shortfall <= tau); + const pool = covering.length > 0 ? covering : rows; + pool.sort((a, b) => { + if (covering.length > 0) { + // Cheapest covering wins; cost tie (incl. all-unknown) breaks toward + // the LOWER tier — a covering lower tier is sufficient by definition, + // so prefer it over excess headroom (avoids over-provisioning). + if (a.cost !== b.cost) return a.cost - b.cost; + return (TIER_PRIORITY[a.tier] || 0) - (TIER_PRIORITY[b.tier] || 0); + } + // Nothing covers: minimal shortfall wins, tiebreak higher tier then cheaper. + if (a.shortfall !== b.shortfall) return a.shortfall - b.shortfall; + const tp = (TIER_PRIORITY[b.tier] || 0) - (TIER_PRIORITY[a.tier] || 0); + if (tp !== 0) return tp; + return a.cost - b.cost; + }); + + return { selected: pool[0], shortfalls: rows, tau }; + } catch (err) { + logger.debug({ err: err.message }, '[Shortfall] select failed — failing open'); + return null; + } +} + +module.exports = { + HEADS, + DEFAULT_TAU, + loadProfiles, + _resetProfilesCache, + _setProfilesForTests, + getTau, + getWeights, + isEnabled, + resolveCapabilities, + resolveCapabilitiesWithSource, + shortfall, + selectByShortfall, +}; diff --git a/test/capability-family.test.js b/test/capability-family.test.js new file mode 100644 index 0000000..d7202d0 --- /dev/null +++ b/test/capability-family.test.js @@ -0,0 +1,71 @@ +const assert = require('assert'); +const { describe, it } = require('node:test'); +const { normalizeFamily, detectQuant, applyQuantHaircut } = require('../src/routing/capability-seeds/family'); +const { heuristicCaps } = require('../src/routing/capability-seeds/family-heuristics'); + +describe('family normalization', () => { + it('strips provider prefixes and org segments', () => { + assert.strictEqual(normalizeFamily('zai', 'GLM-5.2').family, 'glm-5.2'); + assert.strictEqual(normalizeFamily('baidu', 'glm-5.2').family, 'glm-5.2'); + assert.strictEqual(normalizeFamily('ollama', 'glm-5.2').family, 'glm-5.2'); + assert.strictEqual(normalizeFamily('openai', 'openai/gpt-4o-mini').family, 'gpt-4o-mini'); + assert.strictEqual(normalizeFamily('x', 'zai-org/GLM-5.2').family, 'glm-5.2'); + assert.strictEqual(normalizeFamily('databricks', 'databricks-claude-sonnet-4-5').family, 'claude-sonnet-4.5'); + }); + + it('folds ollama tags, keeping size and dropping noise', () => { + assert.strictEqual(normalizeFamily('ollama', 'qwen2.5-coder:7b').family, 'qwen2.5-coder-7b'); + assert.strictEqual(normalizeFamily('ollama', 'minimax-m3:cloud').family, 'minimax-m3'); + assert.strictEqual(normalizeFamily('ollama', 'llama3.2:latest').family, 'llama3.2'); + }); + + it('unifies separators and version runs', () => { + assert.strictEqual(normalizeFamily('x', 'gpt_3_5_turbo').family, 'gpt-3.5-turbo'); + assert.strictEqual(normalizeFamily('x', 'llama-3-1-70b').family, 'llama-3.1-70b'); + assert.strictEqual(normalizeFamily('x', 'Qwen3-32B').family, 'qwen3-32b'); + }); + + it('detects quantization without changing identity matching', () => { + const q = normalizeFamily('ollama', 'glm-5.2:Q4_K_M'); + assert.strictEqual(q.family, 'glm-5.2-q4-k-m'); + assert.strictEqual(q.quant, true); + assert.strictEqual(normalizeFamily('zai', 'glm-5.2').quant, false); + assert.strictEqual(detectQuant('model.gguf'), true); + assert.strictEqual(detectQuant('gpt-4o'), false); + }); + + it('applies a small floor-bounded haircut', () => { + const out = applyQuantHaircut({ reasoning: 0.7, codegen: 0.1 }); + assert.strictEqual(out.reasoning, 0.68); + assert.strictEqual(out.codegen, 0.1); // floor holds, never negative push below 0.1 + }); +}); + +describe('family-ladder heuristics', () => { + it('orders flagship > mid > small', () => { + const opus = heuristicCaps('claude-opus-4-6'); + const sonnet = heuristicCaps('claude-sonnet-4-5'); + const haiku = heuristicCaps('claude-haiku-4-5'); + assert.ok(opus.reasoning > sonnet.reasoning && sonnet.reasoning > haiku.reasoning); + }); + + it('never claims frontier and returns null on no signal', () => { + assert.ok(heuristicCaps('gpt-5.4').reasoning <= 0.85); + assert.strictEqual(heuristicCaps('muse-spark-1.3-contributor-free'), null); + assert.strictEqual(heuristicCaps('unknown'), null); + assert.strictEqual(heuristicCaps(''), null); + }); + + it('tilts coders and reasoners', () => { + const coder = heuristicCaps('qwen2.5-coder-32b'); + assert.ok(coder.codegen >= coder.reasoning); + const reasoner = heuristicCaps('deepseek-r1'); + assert.ok(reasoner.reasoning >= reasoner.codegen); + }); + + it('sizes unknown numbered models monotonically', () => { + const small = heuristicCaps('somemodel-7b'); + const big = heuristicCaps('somemodel-70b'); + assert.ok(big.reasoning > small.reasoning); + }); +}); diff --git a/test/capability-seeds.test.js b/test/capability-seeds.test.js new file mode 100644 index 0000000..54bfb14 --- /dev/null +++ b/test/capability-seeds.test.js @@ -0,0 +1,131 @@ +const assert = require('assert'); +const { describe, it, beforeEach, afterEach } = require('node:test'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +const { cleanName } = require('../src/routing/capability-seeds/swebench'); +const { loadBenchmarksDir } = require('../src/routing/capability-seeds/benchmarks-dir'); +const { mapEntryToFamily, scoresToCaps, scoreToCap } = require('../src/routing/capability-seeds/normalize'); +const registry = require('../src/routing/capability-seeds/registry'); +const shortfall = require('../src/routing/shortfall'); + +const KNOWN = ['claude-opus-4.5', 'gpt-5.2', 'qwen3-coder-480b', 'kimi-k2.5', 'devstral-small', 'gemini-2.5-flash', 'gpt-5*']; + +describe('leaderboard cleaning and family mapping', () => { + it('cleans scaffold, date and effort qualifiers', () => { + assert.strictEqual(cleanName('Claude 4.5 Opus (high) · mini-SWE-agent · 2026-02-17'), 'claude 4.5 opus'); + assert.strictEqual(cleanName('GPT 5.2 · mini-SWE-agent · 2025-12-11'), 'gpt 5.2'); + }); + + it('maps entries to families order-insensitively', () => { + assert.strictEqual(mapEntryToFamily('claude 4.5 opus', KNOWN).family, 'claude-opus-4.5'); + assert.strictEqual(mapEntryToFamily('gpt 5.2', KNOWN).family, 'gpt-5.2'); + assert.strictEqual(mapEntryToFamily('qwen3-coder 480b-a35b instruct', KNOWN).family, 'qwen3-coder-480b'); + assert.strictEqual(mapEntryToFamily('kimi k2.5', KNOWN).family, 'kimi-k2.5'); + }); + + it('refuses weak matches instead of seeding silently', () => { + // "4" must not match "4.5" backwards; size tokens must match exactly. + assert.strictEqual(mapEntryToFamily('claude 4 opus', ['claude-opus-4.5']).family, null); + assert.strictEqual(mapEntryToFamily('qwen3-coder 32b', ['qwen3-coder-480b']).family, null); + assert.strictEqual(mapEntryToFamily('completely unknown model 9000', KNOWN).family, null); + }); + + it('loads drop-in benchmark dirs and skips bad files', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'bench-')); + fs.writeFileSync(path.join(dir, 'good.json'), JSON.stringify({ source: 's', results: { 'A B': 10 } })); + fs.writeFileSync(path.join(dir, 'bad.json'), JSON.stringify({ nope: true })); + fs.writeFileSync(path.join(dir, 'ignore.txt'), 'x'); + const { files } = loadBenchmarksDir(dir); + assert.strictEqual(files.length, 2); + assert.ok(files.some((f) => f.status === 'ok' && f.entries.length === 1)); + assert.ok(files.some((f) => f.status === 'skipped')); + fs.rmSync(dir, { recursive: true, force: true }); + }); + + it('missing benchmarks dir is not an error', () => { + assert.deepStrictEqual(loadBenchmarksDir(path.join(os.tmpdir(), 'no-such-dir-xyz')).files, []); + }); +}); + +describe('score normalization', () => { + it('anchors flagship territory near the ceiling', () => { + assert.ok(scoreToCap(0.8) >= 0.85); + assert.ok(scoreToCap(0) === 0.15); + }); + + it('blends present sources and needs at least one', () => { + const caps = scoresToCaps({ swe: 0.74 }); + assert.ok(caps && caps.reasoning > 0.6 && caps.reasoning < 0.9); + assert.strictEqual(scoresToCaps({}), null); + assert.strictEqual(scoresToCaps(null), null); + }); + + it('applies the pessimistic haircut vs raw anchor', () => { + assert.ok(scoresToCaps({ swe: 0.8 }).reasoning < scoreToCap(0.8)); + }); +}); + +describe('seed registry precedence', () => { + beforeEach(() => { + registry._resetSeedsCache(); + shortfall._resetProfilesCache(); + }); + afterEach(() => { + registry._resetSeedsCache(); + shortfall._resetProfilesCache(); + }); + + const seed = (caps) => ({ caps }); + + it('snapshot > shipped > family > tier, longest wildcard wins', () => { + registry._setSeedsForTests({ + snapshot: { 'glm-5.2': seed({ reasoning: 0.7, codegen: 0.7, debugging: 0.7, tool_use: 0.7 }) }, + shipped: { + 'glm-5.2': seed({ reasoning: 0.1, codegen: 0.1, debugging: 0.1, tool_use: 0.1 }), + 'glm-*': seed({ reasoning: 0.5, codegen: 0.5, debugging: 0.5, tool_use: 0.5 }), + 'g*': seed({ reasoning: 0.2, codegen: 0.2, debugging: 0.2, tool_use: 0.2 }), + }, + }); + assert.strictEqual(registry.resolveSeedCaps('glm-5.2').source, 'seed:snapshot'); + assert.strictEqual(registry.resolveSeedCaps('glm-4.7').source, 'seed:shipped'); + assert.strictEqual(registry.resolveSeedCaps('glm-4.7').caps.reasoning, 0.5); + // shortest wildcard still matches (longest-prefix rule is exercised above) + assert.strictEqual(registry.resolveSeedCaps('gpt-5.2').source, 'seed:shipped'); + assert.strictEqual(registry.resolveSeedCaps('totally-unknown-9z')?.source ?? null, null); + }); + + it('provider servings of one family share caps; quant takes haircut', () => { + const a = shortfall.resolveCapabilitiesWithSource({ provider: 'zai', model: 'glm-5.2', tier: 'SIMPLE' }); + const b = shortfall.resolveCapabilitiesWithSource({ provider: 'baidu', model: 'glm-5.2', tier: 'COMPLEX' }); + const c = shortfall.resolveCapabilitiesWithSource({ provider: 'ollama', model: 'glm-5.2', tier: 'SIMPLE' }); + assert.strictEqual(a.source, 'seed:shipped'); + assert.deepStrictEqual(a.caps, b.caps); + assert.deepStrictEqual(a.caps, c.caps); + const q = shortfall.resolveCapabilitiesWithSource({ provider: 'ollama', model: 'glm-5.2:Q4_K_M', tier: 'SIMPLE' }); + assert.ok(q.caps.reasoning < a.caps.reasoning); + }); + + it('operator override wins verbatim over seeds, even quantized', () => { + shortfall._setProfilesForTests({ + modelOverrides: { 'ollama:glm-5.2:q4_k_m': { reasoning: 0.9, codegen: 0.9, debugging: 0.9, tool_use: 0.9 } }, + }); + const r = shortfall.resolveCapabilitiesWithSource({ provider: 'ollama', model: 'glm-5.2:Q4_K_M', tier: 'SIMPLE' }); + assert.strictEqual(r.source, 'override'); + assert.strictEqual(r.caps.reasoning, 0.9); + }); + + it('unknown families fall back to tier caps, but muse-spark servings are seeded', () => { + const r = shortfall.resolveCapabilitiesWithSource({ provider: 'openai', model: 'muse-spark-1.3-contributor-free', tier: 'SIMPLE' }); + // Shipped provisional seed from operator telemetry — NOT tier caps, and + // NOT the full-1.3 flagship caps: variant servings stay separated. + assert.strictEqual(r.source, 'seed:shipped'); + assert.ok(r.caps.reasoning < 0.5); + const full = shortfall.resolveCapabilitiesWithSource({ provider: 'meta', model: 'muse-spark-1.3', tier: 'SIMPLE' }); + assert.strictEqual(full.source, 'seed:shipped'); + assert.ok(full.caps.reasoning > 0.8); + const unknown = shortfall.resolveCapabilitiesWithSource({ provider: 'x', model: 'never-heard-of-it-9z', tier: 'SIMPLE' }); + assert.strictEqual(unknown.source, 'tier'); + }); +}); diff --git a/test/fireworks-error-resilience.test.js b/test/fireworks-error-resilience.test.js new file mode 100644 index 0000000..f9ba306 --- /dev/null +++ b/test/fireworks-error-resilience.test.js @@ -0,0 +1,109 @@ +/** + * Fireworks AI error resilience. + * + * Fireworks rides the shared OpenAI↔Anthropic converters, so this file pins + * the converter behaviors the Fireworks path depends on most — especially + * the cases that differ across OpenAI-compatible upstreams in production: + * finish_reason "stop" arriving WITH tool_calls (the Moonshot precedent), + * missing/empty choices, and upstream error payloads passing through + * without being mistaken for completions. + */ + +const assert = require("assert"); +const { describe, it } = require("node:test"); + +const { convertOpenAIToAnthropic } = require("../src/clients/databricks"); + +function toolCallCompletion({ finishReason = "tool_calls", content = null } = {}) { + return { + id: "chatcmpl-fw", + object: "chat.completion", + created: 0, + model: "accounts/fireworks/models/kimi-k2-instruct-0905", + choices: [ + { + index: 0, + message: { + role: "assistant", + content, + tool_calls: [ + { + id: "call_fw1", + type: "function", + function: { name: "get_weather", arguments: '{"location":"SF"}' }, + }, + ], + }, + finish_reason: finishReason, + }, + ], + usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }, + }; +} + +describe("fireworks tool-call responses", () => { + it("marks stop_reason tool_use when finish_reason is tool_calls", () => { + const out = convertOpenAIToAnthropic(toolCallCompletion()); + assert.equal(out.stop_reason, "tool_use"); + const toolUse = out.content.find((b) => b.type === "tool_use"); + assert.ok(toolUse); + assert.equal(toolUse.name, "get_weather"); + assert.deepEqual(toolUse.input, { location: "SF" }); + }); + + it("marks stop_reason tool_use even when finish_reason is stop with tool_calls present", () => { + // The Moonshot precedent (databricks.js stop-reason comment): some + // OpenAI-compatible upstreams say "stop" while carrying tool_calls. + // The CLI only executes tools on stop_reason tool_use. + const out = convertOpenAIToAnthropic(toolCallCompletion({ finishReason: "stop" })); + assert.equal(out.stop_reason, "tool_use"); + }); + + it("keeps text content alongside tool calls", () => { + const out = convertOpenAIToAnthropic(toolCallCompletion({ content: "checking…" })); + assert.ok(out.content.some((b) => b.type === "text")); + assert.ok(out.content.some((b) => b.type === "tool_use")); + }); +}); + +describe("fireworks malformed responses", () => { + it("passes through a response with no choices instead of fabricating blocks", () => { + const errPayload = { error: { message: "model not found", type: "invalid_request_error" } }; + const out = convertOpenAIToAnthropic(errPayload); + assert.deepEqual(out, errPayload); + }); + + it("handles empty content without crashing", () => { + const out = convertOpenAIToAnthropic({ + id: "chatcmpl-fw", + object: "chat.completion", + created: 0, + model: "accounts/fireworks/models/kimi-k2-instruct-0905", + choices: [{ index: 0, message: { role: "assistant", content: "" }, finish_reason: "stop" }], + usage: { prompt_tokens: 1, completion_tokens: 0, total_tokens: 1 }, + }); + assert.equal(out.role, "assistant"); + assert.ok(Array.isArray(out.content)); + }); + + it("recovers tool calls degraded to XML/text content", () => { + const out = convertOpenAIToAnthropic({ + id: "chatcmpl-fw", + object: "chat.completion", + created: 0, + model: "accounts/fireworks/models/llama-3.1-8b-instruct", + choices: [{ + index: 0, + message: { + role: "assistant", + content: '\nSF\n', + }, + finish_reason: "stop", + }], + usage: { prompt_tokens: 10, completion_tokens: 12, total_tokens: 22 }, + }); + const toolUse = out.content.find((b) => b.type === "tool_use"); + assert.ok(toolUse, "expected XML degraded tool call to be extracted"); + assert.equal(toolUse.name, "get_weather"); + }); +}); diff --git a/test/fireworks-model-mapping.test.js b/test/fireworks-model-mapping.test.js new file mode 100644 index 0000000..ae32c2a --- /dev/null +++ b/test/fireworks-model-mapping.test.js @@ -0,0 +1,165 @@ +/** + * Tests for Fireworks AI model mapping (invokeFireworks). + * + * invokeFireworks is modeled on invokeBaidu: Anthropic model names map to + * Fireworks serverless ids, tier-selected ids (e.g. + * TIER_COMPLEX=fireworks:accounts/fireworks/models/glm-5p2) reach the wire + * unchanged, and the response is converted to Anthropic shape before + * returning. + * + * NOTE: the modelMap and sampling defaults in invokeFireworks are best-effort + * from public docs, not yet probed against a live key (see the NOTE at the + * top of invokeFireworks in src/clients/databricks.js). These tests pin + * current behavior, not confirmed-correct behavior. + */ + +process.env.DATABRICKS_API_KEY = process.env.DATABRICKS_API_KEY || "test-key"; +process.env.DATABRICKS_API_BASE = process.env.DATABRICKS_API_BASE || "http://test.com"; +process.env.FIREWORKS_API_KEY = process.env.FIREWORKS_API_KEY || "test-key"; +// Pin unconditionally: the test asserts the shipped default mapping, and a +// developer's real .env (e.g. FIREWORKS_MODEL=...) otherwise leaks into the +// config singleton and fails the fallback-model assertion. +process.env.FIREWORKS_MODEL = "accounts/fireworks/models/kimi-k2-instruct-0905"; + +const { describe, it, beforeEach, afterEach } = require("node:test"); +const assert = require("node:assert/strict"); + +const { invokeFireworks } = require("../src/clients/databricks"); + +let captured; +const realFetch = global.fetch; + +function okCompletion(model) { + return new Response( + JSON.stringify({ + id: "chatcmpl-test", + object: "chat.completion", + created: 0, + model, + choices: [ + { + index: 0, + message: { role: "assistant", content: "ok" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ); +} + +beforeEach(() => { + captured = null; + global.fetch = async (url, init) => { + captured = { url: String(url), body: JSON.parse(init.body), headers: init.headers }; + return okCompletion(captured.body.model); + }; +}); + +afterEach(() => { + global.fetch = realFetch; +}); + +const USER_MSG = [{ role: "user", content: "hi" }]; + +describe("fireworks model mapping", () => { + it("passes tier-selected serverless ids through instead of the .env default", async () => { + await invokeFireworks({ _tierModel: "accounts/fireworks/models/glm-5p2", model: "claude-sonnet-4-5", messages: USER_MSG }); + assert.equal(captured.body.model, "accounts/fireworks/models/glm-5p2"); + }); + + it("passes tier-selected family slugs through", async () => { + await invokeFireworks({ _tierModel: "deepseek-v3p1", model: "claude-sonnet-4-5", messages: USER_MSG }); + assert.equal(captured.body.model, "deepseek-v3p1"); + }); + + it("maps claude sonnet names to Kimi K2 Instruct", async () => { + await invokeFireworks({ model: "claude-sonnet-4-5", messages: USER_MSG }); + assert.equal(captured.body.model, "accounts/fireworks/models/kimi-k2-instruct-0905"); + }); + + it("maps claude opus names to GLM", async () => { + await invokeFireworks({ model: "claude-opus-4-5", messages: USER_MSG }); + assert.equal(captured.body.model, "accounts/fireworks/models/glm-5p2"); + }); + + it("maps claude haiku names to a small fast model", async () => { + await invokeFireworks({ model: "claude-haiku-4-5", messages: USER_MSG }); + assert.equal(captured.body.model, "accounts/fireworks/models/llama-3.1-8b-instruct"); + }); + + it("falls back to the .env default model for unrecognized names", async () => { + await invokeFireworks({ model: "some-unmapped-model", messages: USER_MSG }); + assert.equal(captured.body.model, "accounts/fireworks/models/kimi-k2-instruct-0905"); + }); + + it("posts to the Fireworks inference endpoint", async () => { + await invokeFireworks({ model: "claude-sonnet-4-5", messages: USER_MSG }); + assert.equal(captured.url, "https://api.fireworks.ai/inference/v1/chat/completions"); + }); +}); + +describe("fireworks request shape", () => { + it("sends a bearer auth header with the configured API key", async () => { + await invokeFireworks({ model: "claude-sonnet-4-5", messages: USER_MSG }); + assert.equal(captured.headers.Authorization, "Bearer test-key"); + }); + + it("prepends system content as a system-role message", async () => { + await invokeFireworks({ model: "claude-sonnet-4-5", system: "be terse", messages: USER_MSG }); + assert.equal(captured.body.messages[0].role, "system"); + assert.equal(captured.body.messages[0].content, "be terse"); + }); + + it("converts Anthropic tools to OpenAI function-calling shape", async () => { + const tools = [ + { name: "get_weather", description: "get weather", input_schema: { type: "object", properties: {} } }, + ]; + await invokeFireworks({ model: "claude-sonnet-4-5", messages: USER_MSG, tools }); + assert.equal(captured.body.tools[0].type, "function"); + assert.equal(captured.body.tools[0].function.name, "get_weather"); + assert.equal(captured.body.tool_choice, "auto"); + assert.equal(captured.body.parallel_tool_calls, false); + }); + + it("throws a clear error when FIREWORKS_API_KEY is not configured", async () => { + const config = require("../src/config"); + const original = config.fireworks.apiKey; + config.fireworks.apiKey = null; + try { + await assert.rejects( + invokeFireworks({ model: "claude-sonnet-4-5", messages: USER_MSG }), + /Fireworks API key is not configured/, + ); + } finally { + config.fireworks.apiKey = original; + } + }); + + it("throws a typed 429 so tier-fallback climbs instead of hanging", async () => { + const config = require("../src/config"); + const originalRetry = config.apiRetry; + config.apiRetry = { maxRetries: 1, initialDelay: 1, maxDelay: 5 }; + global.fetch = async () => new Response( + JSON.stringify({ error: { message: "Too Many Requests", type: "rate_limit_error" } }), + { status: 429, headers: { "content-type": "application/json" } }, + ); + try { + await assert.rejects( + invokeFireworks({ model: "claude-sonnet-4-5", messages: USER_MSG }), + (err) => err.status === 429 && /Fireworks rate-limited/.test(err.message), + ); + } finally { + config.apiRetry = originalRetry; + } + }); +}); + +describe("fireworks response conversion", () => { + it("converts the OpenAI-shaped completion to Anthropic content blocks", async () => { + const response = await invokeFireworks({ model: "claude-sonnet-4-5", messages: USER_MSG }); + assert.equal(response.json.content[0].type, "text"); + assert.equal(response.json.content[0].text, "ok"); + }); +}); diff --git a/test/fixtures/capability-sources/swebench-leaderboards.sample.json b/test/fixtures/capability-sources/swebench-leaderboards.sample.json new file mode 100644 index 0000000..a83841c --- /dev/null +++ b/test/fixtures/capability-sources/swebench-leaderboards.sample.json @@ -0,0 +1,18 @@ +{ + "leaderboards": [ + { + "name": "Verified", + "results": [ + { "name": "Claude 4.5 Opus (high) · mini-SWE-agent · 2026-02-17", "resolved": 74.2, "date": "2026-02-17" }, + { "name": "GPT 5.2 · mini-SWE-agent · 2025-12-11", "resolved": 71.0, "date": "2025-12-11" }, + { "name": "Kimi K2.5 (high) · mini-SWE-agent · 2026-02-17", "resolved": 62.4, "date": "2026-02-17" }, + { "name": "Qwen3-Coder 480B-A35B Instruct · mini-SWE-agent · 2025-08-02", "resolved": 69.1, "date": "2025-08-02" }, + { "name": "broken entry", "resolved": "NaN", "date": "2025-01-01" } + ] + }, + { + "name": "Lite", + "results": [{ "name": "Something Else", "resolved": 10.0 }] + } + ] +} diff --git a/test/fixtures/capability-sources/terminalbench.sample.json b/test/fixtures/capability-sources/terminalbench.sample.json new file mode 100644 index 0000000..9da4bb9 --- /dev/null +++ b/test/fixtures/capability-sources/terminalbench.sample.json @@ -0,0 +1,9 @@ +{ + "source": "terminalbench-2.1-sample", + "scale": "percent", + "results": { + "Claude 4.5 Opus": 55.0, + "Kimi K2.5": 41.2, + "Completely Unknown Model 9000": 5.0 + } +} diff --git a/test/shortfall-routing.test.js b/test/shortfall-routing.test.js new file mode 100644 index 0000000..386a188 --- /dev/null +++ b/test/shortfall-routing.test.js @@ -0,0 +1,96 @@ +const assert = require('assert'); +const { describe, it, beforeEach, afterEach } = require('node:test'); + +describe('shortfall routing integration', () => { + let originalEnv; + + beforeEach(() => { + for (const m of [ + '../src/config/index.js', + '../src/clients/routing', + '../src/routing/index.js', + '../src/routing/model-tiers', + '../src/routing/shortfall', + '../src/routing/capabilities', + ]) { + try { delete require.cache[require.resolve(m)]; } catch { /* not loaded */ } + } + originalEnv = { ...process.env }; + process.env.FALLBACK_PROVIDER = 'databricks'; + process.env.DATABRICKS_API_KEY = 'test-key'; + process.env.DATABRICKS_API_BASE = 'http://test.com'; + process.env.TIER_SIMPLE = 'openai:cheap-small'; + process.env.TIER_MEDIUM = 'openai:mid-model'; + process.env.TIER_COMPLEX = 'azure-openai:big-model'; + process.env.TIER_REASONING = 'azure-openai:big-model'; + }); + + afterEach(() => { + process.env = originalEnv; + }); + + // NOTE: force-path greetings ("hi") return before tier selection by design + // (force-local dominates shortfall) — use non-force asks so both paths run. + // NOTE 2: risk-high asks (auth/database/security) also return early via the + // risk guard, which dominates shortfall by design — keep HARD risk-free. + const TRIVIAL = 'Explain what a hash map is in one paragraph.'; + const HARD = 'Rewrite the entire data-visualization layer across a dozen files: redesign the chart architecture, untangle the tangled async rendering pipeline, then implement the new dashboard module with step-by-step tradeoffs analysis and write the integration tests.'; + + it('shadow-computes without changing the legacy decision when disabled', async () => { + const routing = require('../src/clients/routing'); + // Pin disabled via injection (hermetic — never depends on the operator's + // local config file, which may have enabled:true). + require('../src/routing/shortfall')._setProfilesForTests({ enabled: false }); + const result = await routing.determineProviderSmart({ messages: [{ role: 'user', content: TRIVIAL }] }); + assert.ok(result.provider, 'expected a provider'); + assert.ok(!String(result.method || '').includes('shortfall'), `method=${result.method}`); + assert.ok(result.shortfall && typeof result.shortfall === 'object', 'expected shadow shortfall info'); + assert.ok(result.shortfall.req, 'expected requirement vector'); + }); + + it('serves the shortfall pick with +shortfall method when enabled', async () => { + const routing = require('../src/clients/routing'); + // Toggle via config injection (no env vars) — same instance routing uses. + require('../src/routing/shortfall')._setProfilesForTests({ enabled: true }); + const result = await routing.determineProviderSmart({ messages: [{ role: 'user', content: TRIVIAL }] }); + assert.ok(result.shortfall && typeof result.shortfall === 'object'); + // Trivial ask: legacy SIMPLE (cheap-small) already covers → agree, no suffix. + // A hard multi-file refactor ask should escalate to the big model with suffix. + const hard = await routing.determineProviderSmart({ + messages: [{ role: 'user', content: HARD }], + }); + assert.ok(hard.shortfall, 'expected shortfall info on hard request'); + // Wiring contract (direction-agnostic: tier profiles are hand-seeded until + // calibrated on telemetry): when shortfall disagrees with legacy, the + // served model must be the shortfall pick and the method must say so. + if (hard.shortfall.agreed === false) { + assert.ok(String(hard.method).includes('shortfall'), `method=${hard.method}`); + assert.strictEqual(hard.model, hard.shortfall.selected.model); + } else { + assert.ok(!String(hard.method).includes('shortfall'), `method=${hard.method}`); + } + }); + + it('resolves one family identically across providers (z.ai/Baidu/local)', async () => { + // Same weights (glm-5.2) served three ways must route identically: + // provider decides cost/invocation only, family decides capability. + for (const simple of ['baidu:glm-5.2', 'zai:glm-5.2', 'ollama:glm-5.2']) { + process.env.TIER_SIMPLE = simple; + for (const m of [ + '../src/config/index.js', + '../src/clients/routing', + '../src/routing/index.js', + '../src/routing/model-tiers', + '../src/routing/shortfall', + '../src/routing/capabilities', + ]) { + try { delete require.cache[require.resolve(m)]; } catch { /* not loaded */ } + } + const routing = require('../src/clients/routing'); + require('../src/routing/shortfall')._setProfilesForTests({ enabled: true }); + const result = await routing.determineProviderSmart({ messages: [{ role: 'user', content: TRIVIAL }] }); + assert.strictEqual(result.model, 'glm-5.2', `simple=${simple}`); + assert.strictEqual(result.shortfall?.selected?.source, 'seed:shipped', `simple=${simple}`); + } + }); +}); diff --git a/test/shortfall.test.js b/test/shortfall.test.js new file mode 100644 index 0000000..cef1889 --- /dev/null +++ b/test/shortfall.test.js @@ -0,0 +1,123 @@ +const assert = require('assert'); +const { describe, it, beforeEach, afterEach } = require('node:test'); +const { buildRequirementVector } = require('../src/routing/capabilities'); +const shortfall = require('../src/routing/shortfall'); + +function dims(over = {}) { + return { + tokenCount: 10, + promptComplexity: 20, + technicalDepth: 20, + domainSpecificity: 20, + toolCount: 0, + toolComplexity: 0, + toolChainPotential: 20, + multiStepReasoning: 20, + codeGeneration: 20, + analysisDepth: 20, + conversationDepth: 10, + priorToolUsage: 10, + ambiguity: 40, + ...over, + }; +} + +describe('capability requirement vector', () => { + it('maps trivial dims to low requirements on every head', () => { + const v = buildRequirementVector({ dimensions: dims() }); + for (const h of ['reasoning', 'codegen', 'debugging', 'tool_use']) { + assert.ok(v[h] >= 0 && v[h] <= 0.35, `${h}=${v[h]}`); + } + }); + + it('maps code-heavy dims to high codegen, low reasoning', () => { + const v = buildRequirementVector({ + dimensions: dims({ codeGeneration: 80, technicalDepth: 80, toolComplexity: 80 }), + }); + assert.ok(v.codegen > 0.6, `codegen=${v.codegen}`); + assert.ok(v.reasoning < 0.4, `reasoning=${v.reasoning}`); + }); + + it('agentic floors tool_use without lowering other heads', () => { + const plain = buildRequirementVector({ dimensions: dims() }); + const agentic = buildRequirementVector({ dimensions: dims(), agenticResult: { isAgentic: true } }); + assert.ok(agentic.tool_use >= 0.6); + assert.strictEqual(agentic.reasoning, plain.reasoning); + }); + + it('fails open on garbage input', () => { + const v = buildRequirementVector({}); + assert.deepStrictEqual(Object.keys(v).sort(), ['codegen', 'debugging', 'reasoning', 'tool_use']); + }); +}); + +describe('shortfall matching', () => { + let env; + beforeEach(() => { + env = { ...process.env }; + shortfall._resetProfilesCache(); + }); + afterEach(() => { + process.env = env; + shortfall._resetProfilesCache(); + }); + + const cands = [ + { provider: 'openai', model: 'cheap', tier: 'SIMPLE', cost: 0.1 }, + { provider: 'openai', model: 'mid', tier: 'MEDIUM', cost: 1 }, + { provider: 'azure-openai', model: 'big', tier: 'REASONING', cost: 5 }, + ]; + + it('picks cheapest covering model for a trivial request', () => { + const req = { reasoning: 0.1, codegen: 0.1, debugging: 0.1, tool_use: 0.1 }; + const r = shortfall.selectByShortfall(req, cands, { tau: 0.24 }); + assert.strictEqual(r.selected.model, 'cheap'); + }); + + it('escalates past cheap when requirements exceed its caps', () => { + const req = { reasoning: 0.8, codegen: 0.8, debugging: 0.8, tool_use: 0.8 }; + const r = shortfall.selectByShortfall(req, cands, { tau: 0.24 }); + assert.strictEqual(r.selected.model, 'big'); + }); + + it('tau gates coverage: strict tau escalates, loose tau economizes', () => { + const req = { reasoning: 0.5, codegen: 0.5, debugging: 0.5, tool_use: 0.5 }; + const strict = shortfall.selectByShortfall(req, cands, { tau: 0.01 }); + const loose = shortfall.selectByShortfall(req, cands, { tau: 0.5 }); + assert.ok(['mid', 'big'].includes(strict.selected.model)); + assert.strictEqual(loose.selected.model, 'cheap'); + }); + + it('catalog change re-routes with zero retraining (model removal)', () => { + const req = { reasoning: 0.1, codegen: 0.1, debugging: 0.1, tool_use: 0.1 }; + const full = shortfall.selectByShortfall(req, cands, { tau: 0.24 }); + assert.strictEqual(full.selected.model, 'cheap'); + const withoutCheap = shortfall.selectByShortfall(req, cands.slice(1), { tau: 0.24 }); + assert.strictEqual(withoutCheap.selected.model, 'mid'); + }); + + it('unknown cost never wins on cheapness alone', () => { + const req = { reasoning: 0.1, codegen: 0.1, debugging: 0.1, tool_use: 0.1 }; + const rows = [ + { provider: 'x', model: 'mystery', tier: 'SIMPLE' }, // no cost + { provider: 'openai', model: 'cheap', tier: 'SIMPLE', cost: 0.1 }, + ]; + const r = shortfall.selectByShortfall(req, rows, { tau: 0.24 }); + assert.strictEqual(r.selected.model, 'cheap'); + }); + + it('cost tie breaks toward the lower covering tier (no over-provisioning)', () => { + const req = { reasoning: 0.1, codegen: 0.1, debugging: 0.1, tool_use: 0.1 }; + const rows = [ + { provider: 'azure-openai', model: 'big', tier: 'REASONING', cost: Number.POSITIVE_INFINITY }, + { provider: 'openai', model: 'cheap', tier: 'SIMPLE', cost: Number.POSITIVE_INFINITY }, + ]; + const r = shortfall.selectByShortfall(req, rows, { tau: 0.24 }); + assert.strictEqual(r.selected.model, 'cheap'); + }); + + it('fails open (null) on malformed input', () => { + assert.strictEqual(shortfall.selectByShortfall(null, cands), null); + assert.strictEqual(shortfall.selectByShortfall({ reasoning: 0.5 }, []), null); + }); +});