From eecd67894d5f4dc9a839aedd7c147fa8bf4a1fc0 Mon Sep 17 00:00:00 2001 From: 1bcMax Date: Tue, 20 Oct 2026 10:53:41 -0500 Subject: [PATCH 1/2] feat(router): re-sync Router Core to upstream 2cd5a10 Port router-core e38958b..2cd5a10 into blockrun_llm/router_core: - tokens.py (new): estimate_tokens counts CJK at ~1 token per char; scan_limit / sample_for_scan move here from strategy.py - rules.py: keywords match whole words with graded inflections (Latin/Cyrillic/Greek/digit boundary, CJK/Hangul/Arabic keep substring) - strategy.py: structured output only from requires_structured_output, never inferred from the system prompt - portfolio.py: stakes/latency/language read the user turn only; CamelCase error classes count as code; debug/code_edit with a code artifact floor at MEDIUM; stale speed fades toward 0.5; eleven probe-verified models join tool chains as priced, cost-ordered fallbacks - selector.py: an unpriced model reports 0 savings, not 100% - model_capabilities.py: regenerated from the 2026-10-20 catalog (89 models) - router.classify_by_rules defaults to estimate_tokens Tests: port the new upstream vitest cases and re-copy the decision snapshot fixture (88/88 decisions reproduce). --- CHANGELOG.md | 38 + blockrun_llm/router.py | 9 +- blockrun_llm/router_core/__init__.py | 4 +- blockrun_llm/router_core/config.py | 5 +- .../router_core/model_capabilities.py | 84 +- blockrun_llm/router_core/portfolio.py | 156 ++- blockrun_llm/router_core/rules.py | 165 ++- blockrun_llm/router_core/selector.py | 6 +- blockrun_llm/router_core/strategy.py | 40 +- blockrun_llm/router_core/tokens.py | 54 + .../unit/router_core_decisions.snapshot.json | 1189 ++++++++++------- tests/unit/test_router_core.py | 387 +++++- tests/unit/test_router_core_snapshot.py | 2 +- 13 files changed, 1549 insertions(+), 590 deletions(-) create mode 100644 blockrun_llm/router_core/tokens.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 06a84b0..ee7d1cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,44 @@ All notable changes to blockrun-llm will be documented in this file. +## Unreleased + +### Changed +- **Router Core re-synced to upstream `2cd5a10`** (was `e38958b`, five + upstream commits behind; the TypeScript SDK bundles `65a92c8`, which differs + from `2cd5a10` only in the capability snapshot). Auto routing now reads only + the user turn for stakes, latency and language cues, and structured output + is taken only from `requires_structured_output` (`response_format`): a + system prompt that lists tool schemas or says "be quick" no longer lifts a + request off the cheapest tier. CJK input is sized at about one token per + character (`router_core.estimate_tokens`, also the default for + `router.classify_by_rules`), so a long Chinese document reaches the + long-context models instead of overflowing a smaller window. Keyword scoring + matches whole words ("classic" is no longer code, "history" is no longer + creative, "prefix" is no longer agentic) while keeping ordinary inflections, + ASCII German/Romance endings, Cyrillic stems and CJK substrings. A stack + trace or CamelCase error class (`TypeError`, `NullPointerException`) now + floors a debug or code-edit turn at MEDIUM. A stale speed probe fades toward + neutral instead of toward zero, and a model with no catalog price no longer + reports 100% savings. + +- **Tool routing admits eleven newly probe-verified models as availability + fallbacks:** Claude Fable 5.1 / Opus 5.5 / Sonnet 5.5, Gemini 3.8 Flash, + GPT-5.1, GPT-6 Astra / Luna / Sol, Grok 4.6 / 4.7 and MiMo V2.5 now carry + `supports_tools=True` (all eleven were fail-closed here before). They join a + tool-required turn's chain after the configured and benchmark-backed models, + only when the live catalog prices them, cheapest first outside `premium`. + Primary selection is unchanged. + +- **Capability snapshot regenerated from the 2026-10-20 catalog: 89 text + models** (was 82). New: DeepSeek V4.1 Flash, GPT-6.1 Sol, Mistral Large 4, + Muse Glimmer 30B, Laguna S 2.1, Qwen 3.8 Max and Hunyuan 4 Preview, all + without probe evidence yet, so they serve plain chat but never a + tool-required turn. DeepSeek V4 Flash Vision Experimental and Qwen 3.8 Flash + failed the function-call probe and stay fail-closed. The cross-language + decision snapshot (`tests/unit/router_core_decisions.snapshot.json`) is + re-copied from upstream, and the port reproduces all 88 decisions exactly. + ## 1.20.0 — 2026-10-11 ### Added diff --git a/blockrun_llm/router.py b/blockrun_llm/router.py index 711dc27..279219a 100644 --- a/blockrun_llm/router.py +++ b/blockrun_llm/router.py @@ -36,7 +36,7 @@ ResolvedRoutingDecision, route_with_catalog, ) -from .router_core import DEFAULT_ROUTING_CONFIG +from .router_core import DEFAULT_ROUTING_CONFIG, estimate_tokens from .router_core import classify_by_rules as _classify_by_rules from .router_core.types import ModelPricing, ScoringResult, Tier, TierConfig from .types import RoutingProfile @@ -66,12 +66,11 @@ def classify_by_rules( ) -> ScoringResult: """Classify a prompt into a tier with the shared 15-dimension scorer. - ``estimated_tokens`` defaults to the ~4-chars-per-token estimate the router - itself uses. + ``estimated_tokens`` defaults to the estimate the router itself uses (about + four characters per token, one per CJK character). """ if estimated_tokens is None: - full_text = f"{system_prompt or ''} {prompt}" - estimated_tokens = -(-len(full_text) // 4) # ceil + estimated_tokens = estimate_tokens(f"{system_prompt or ''} {prompt}") return _classify_by_rules( prompt, system_prompt, estimated_tokens, DEFAULT_ROUTING_CONFIG["scoring"] ) diff --git a/blockrun_llm/router_core/__init__.py b/blockrun_llm/router_core/__init__.py index 3c8d105..36c1fb9 100644 --- a/blockrun_llm/router_core/__init__.py +++ b/blockrun_llm/router_core/__init__.py @@ -2,7 +2,7 @@ Router Core — deterministic, constraint-first model routing. Python port of `@blockrun/router-core `_ -(upstream commit ``e38958b``), the same routing engine the TypeScript SDK and +(upstream commit ``2cd5a10``), the same routing engine the TypeScript SDK and the BlockRun gateway use. The package is deliberately product-neutral: task classification, hard capability filtering, portfolio scoring, ordered fallbacks, and routing configuration. It contains no wallet, gateway client, @@ -51,6 +51,7 @@ get_strategy, register_strategy, ) +from .tokens import estimate_tokens from .tool_intent import infer_tool_requirement from .types import ( Capacity, @@ -108,6 +109,7 @@ def route( "calculate_model_cost", "classify_by_rules", "classify_task", + "estimate_tokens", "filter_by_exclude_list", "filter_by_tool_calling", "filter_by_vision", diff --git a/blockrun_llm/router_core/config.py b/blockrun_llm/router_core/config.py index 2068e0b..8915fde 100644 --- a/blockrun_llm/router_core/config.py +++ b/blockrun_llm/router_core/config.py @@ -2,8 +2,9 @@ Default Routing Config Python port of ``@blockrun/router-core`` ``config.ts`` (upstream commit -``e38958b``, 2026-09-29 — the same pin the TypeScript SDK -bundles). V3.5: every tier rung is a public catalog id. +``2cd5a10``; the file is unchanged since ``e38958b``, 2026-09-29, so it is the +same table the TypeScript SDK bundles). V3.5: every tier rung is a public +catalog id. All routing parameters as a module constant. Hosts override by passing their own ``RoutingConfig`` in ``RouterOptions["config"]``. diff --git a/blockrun_llm/router_core/model_capabilities.py b/blockrun_llm/router_core/model_capabilities.py index dd8d97d..f7722f7 100644 --- a/blockrun_llm/router_core/model_capabilities.py +++ b/blockrun_llm/router_core/model_capabilities.py @@ -8,7 +8,7 @@ product catalog is temporarily unavailable, without importing product code. GENERATED upstream by ``scripts/sync-model-capabilities.mjs`` from the public -catalog (GET https://blockrun.ai/api/v1/models) on 2026-10-02; ``supports_tools`` +catalog (GET https://blockrun.ai/api/v1/models) on 2026-10-20; ``supports_tools`` comes from a live function-calling probe. Re-sync from ``model-capabilities.ts`` rather than editing by hand — a hand edit is lost on the next sync. """ @@ -28,11 +28,10 @@ "supports_tools": True, "supports_vision": True, }, - # supportsTools: not probed — fails closed "anthropic/claude-fable-5.1": { "context_window": 1_000_000, "max_output_tokens": 128_000, - "supports_tools": False, + "supports_tools": True, "supports_vision": True, }, # override: The public catalog's `categories` omit "vision" for this Anthropic model even @@ -69,11 +68,10 @@ "supports_tools": True, "supports_vision": True, }, - # supportsTools: not probed — fails closed "anthropic/claude-opus-5.5": { "context_window": 1_000_000, "max_output_tokens": 128_000, - "supports_tools": False, + "supports_tools": True, "supports_vision": True, }, "anthropic/claude-sonnet-4.5": { @@ -98,11 +96,10 @@ "supports_tools": True, "supports_vision": True, }, - # supportsTools: not probed — fails closed "anthropic/claude-sonnet-5.5": { "context_window": 1_000_000, "max_output_tokens": 128_000, - "supports_tools": False, + "supports_tools": True, "supports_vision": True, }, # supportsTools: not probed — fails closed @@ -124,7 +121,6 @@ "supports_tools": True, "supports_vision": False, }, - # supportsTools: not probed — fails closed "deepseek/deepseek-v4-flash-vision-exp": { "context_window": 1_048_576, "max_output_tokens": 65_536, @@ -137,6 +133,13 @@ "supports_tools": True, "supports_vision": False, }, + # supportsTools: not probed — fails closed + "deepseek/deepseek-v4.1-flash": { + "context_window": 1_048_576, + "max_output_tokens": 65_536, + "supports_tools": False, + "supports_vision": False, + }, "google/gemini-2.5-flash": { "context_window": 1_048_576, "max_output_tokens": 65_536, @@ -191,11 +194,10 @@ "supports_tools": True, "supports_vision": True, }, - # supportsTools: not probed — fails closed "google/gemini-3.8-flash": { "context_window": 1_048_576, "max_output_tokens": 65_536, - "supports_tools": False, + "supports_tools": True, "supports_vision": True, }, "minimax/minimax-m2.7": { @@ -210,6 +212,13 @@ "supports_tools": True, "supports_vision": False, }, + # supportsTools: not probed — fails closed + "mistral/mistral-large-4": { + "context_window": 524_288, + "max_output_tokens": 262_144, + "supports_tools": False, + "supports_vision": True, + }, "moonshot/kimi-k3": { "context_window": 1_048_576, "max_output_tokens": 65_536, @@ -223,6 +232,13 @@ "supports_tools": False, "supports_vision": True, }, + # supportsTools: not probed — fails closed + "nvidia/muse-glimmer-30b": { + "context_window": 131_072, + "max_output_tokens": 16_384, + "supports_tools": False, + "supports_vision": False, + }, # override: The catalog tags this model "vision", but a correctly sized probe does not # hold up (ClawRouter, 2026-08-31): a 64x64 solid-red PNG was named correctly 1 of 4 # times on Base, and on Solana the image was silently dropped and a text model answered @@ -292,11 +308,10 @@ "supports_tools": True, "supports_vision": False, }, - # supportsTools: not probed — fails closed "openai/gpt-5.1": { "context_window": 400_000, "max_output_tokens": 128_000, - "supports_tools": False, + "supports_tools": True, "supports_vision": True, }, "openai/gpt-5.2": { @@ -398,22 +413,26 @@ "supports_tools": True, "supports_vision": True, }, - # supportsTools: not probed — fails closed "openai/gpt-6-astra": { "context_window": 1_050_000, "max_output_tokens": 128_000, - "supports_tools": False, + "supports_tools": True, "supports_vision": True, }, - # supportsTools: not probed — fails closed "openai/gpt-6-luna": { "context_window": 1_050_000, "max_output_tokens": 128_000, - "supports_tools": False, + "supports_tools": True, "supports_vision": True, }, - # supportsTools: not probed — fails closed "openai/gpt-6-sol": { + "context_window": 1_050_000, + "max_output_tokens": 128_000, + "supports_tools": True, + "supports_vision": True, + }, + # supportsTools: not probed — fails closed + "openai/gpt-6.1-sol": { "context_window": 1_050_000, "max_output_tokens": 128_000, "supports_tools": False, @@ -444,6 +463,13 @@ "supports_vision": False, }, # supportsTools: not probed — fails closed + "poolside/laguna-s-2.1": { + "context_window": 1_048_576, + "max_output_tokens": 131_072, + "supports_tools": False, + "supports_vision": False, + }, + # supportsTools: not probed — fails closed "poolside/laguna-xs-2.1": { "context_window": 131_072, "max_output_tokens": 16_384, @@ -468,13 +494,26 @@ "supports_tools": True, "supports_vision": False, }, - # supportsTools: not probed — fails closed "qwen/qwen3.8-flash": { "context_window": 1_000_000, "max_output_tokens": 131_072, "supports_tools": False, "supports_vision": True, }, + # supportsTools: not probed — fails closed + "qwen/qwen3.8-max": { + "context_window": 1_000_000, + "max_output_tokens": 131_072, + "supports_tools": False, + "supports_vision": False, + }, + # supportsTools: not probed — fails closed + "tencent/hy4-preview": { + "context_window": 1_048_576, + "max_output_tokens": 64_000, + "supports_tools": False, + "supports_vision": False, + }, "xai/grok-4.3": { "context_window": 1_000_000, "max_output_tokens": 16_384, @@ -487,18 +526,16 @@ "supports_tools": True, "supports_vision": True, }, - # supportsTools: not probed — fails closed "xai/grok-4.6": { "context_window": 500_000, "max_output_tokens": 16_384, - "supports_tools": False, + "supports_tools": True, "supports_vision": True, }, - # supportsTools: not probed — fails closed "xai/grok-4.7": { "context_window": 500_000, "max_output_tokens": 16_384, - "supports_tools": False, + "supports_tools": True, "supports_vision": True, }, "xai/grok-build-0.1": { @@ -507,11 +544,10 @@ "supports_tools": True, "supports_vision": False, }, - # supportsTools: not probed — fails closed "xiaomi/mimo-v2.5": { "context_window": 1_048_576, "max_output_tokens": 131_072, - "supports_tools": False, + "supports_tools": True, "supports_vision": True, }, "xiaomi/mimo-v2.5-pro": { diff --git a/blockrun_llm/router_core/portfolio.py b/blockrun_llm/router_core/portfolio.py index 77fd07f..1e74c00 100644 --- a/blockrun_llm/router_core/portfolio.py +++ b/blockrun_llm/router_core/portfolio.py @@ -19,7 +19,8 @@ from .model_capabilities import DEFAULT_MODEL_CAPABILITIES from .model_profiles import HISTORICAL_MODEL_PROFILES, LIVE_MODEL_PROFILES from .selector import get_fallback_chain, select_model -from .strategy import RulesStrategy, sample_prompt, scan_limit_for +from .strategy import RulesStrategy +from .tokens import estimate_tokens, sample_for_scan, scan_limit from .tool_intent import infer_tool_requirement from .types import ( CandidateScore, @@ -85,6 +86,8 @@ class TaskFeatures: terminal_tool_signal: bool terminal_safety_sensitive: bool implicit_terminal_code: bool + #: A code artifact or coding action is present, not merely a language name. + code_artifact: bool # ─── Compiled request features (ported 1:1 from the TypeScript regexes) ─── @@ -158,6 +161,12 @@ class TaskFeatures: r"\b(?:programmed|written|implemented?|code)\s+(?:in|using)\s+(?:c\+\+|c|rust|go)\b", ignorecase=True, ) +# A CamelCase error class (TypeError, NullPointerException) is a runtime +# artifact, not prose about errors; case-sensitive on purpose. +_ERROR_CLASS_SIGNAL = js_regex(r"\b[A-Z][A-Za-z]*(?:Error|Exception)\b") +_CODE_ARTIFACT_SIGNAL = js_regex( + r"```|\b(?:stack trace|traceback)\b|\.(?:ts|tsx|js|py|go|rs)\b", ignorecase=True +) _AIRLINE_TOOL = js_regex(r"(?:flight|reservation|airport|baggage|passenger)") _RETAIL_TOOL = js_regex(r"(?:order|product|item|return|exchange|address)") _WEB_RESEARCH_TOOL = js_regex(r"^(?:web_?search|web_?fetch)$") @@ -374,13 +383,15 @@ def _likely_needs_parallel_tool_calls( def classify_task(prompt: str, system_prompt: str | None, options: RouterOptions) -> TaskFeatures: """Extract the request-side features the portfolio scorer ranks against.""" full_text = f"{system_prompt or ''} {prompt}" - estimated_input_tokens = math.ceil(len(full_text) / 4) - # Feature regexes need request shape and intent, not the entire document. - # Sample both ends so a long pasted artifact keeps the task instruction at - # either boundary, while the full length still drives capacity decisions. - scan_limit = scan_limit_for(options) - scanned_prompt = sample_prompt(prompt, scan_limit) - scanned_system_prompt = sample_prompt(system_prompt or "", scan_limit) + estimated_input_tokens = estimate_tokens(full_text) + limit = scan_limit(options["config"]["classifier"]["prompt_truncation_chars"]) + scanned_prompt = sample_for_scan(prompt, limit) + scanned_system_prompt = sample_for_scan(system_prompt or "", limit) + # Only the deep-web-research clue detectors below read the system prompt, + # and they are gated on a web-research tool surface. Every other feature + # reads the user turn: a host's standing instructions ("be quick", "follow + # security best practices", a Mandarin persona) describe the assistant, not + # this request, and must not move its scoring or language band. scanned_full_text = f"{scanned_system_prompt} {scanned_prompt}" text = scanned_prompt.lower() @@ -391,7 +402,21 @@ def classify_task(prompt: str, system_prompt: str | None, options: RouterOptions # redirect an entire tool session to the code-agent portfolio. code_construct_signal = bool(_CODE_CONSTRUCT_SIGNAL.search(scanned_prompt)) native_code_signal = bool(_NATIVE_CODE_SIGNAL.search(scanned_prompt)) - has_code = explicit_code_signal or code_construct_signal or native_code_signal + # A CamelCase error class is a runtime artifact (see _ERROR_CLASS_SIGNAL). + error_class_signal = bool(_ERROR_CLASS_SIGNAL.search(scanned_prompt)) + has_code = ( + explicit_code_signal or code_construct_signal or native_code_signal or error_class_signal + ) + # Stronger than has_code: an actual code artifact or coding action, never a + # bare language name ("What is Python?" is trivia, not a code task) and + # never the English word "exception" on its own ("please make an + # exception"); a thrown exception shows up as a CamelCase class above. + code_artifact = ( + bool(_CODE_ARTIFACT_SIGNAL.search(scanned_prompt)) + or code_construct_signal + or native_code_signal + or error_class_signal + ) tools_available = options.get("has_tools", False) requires_tools = options.get("requires_tools") @@ -503,8 +528,8 @@ def classify_task(prompt: str, system_prompt: str | None, options: RouterOptions needs_vision = options.get("has_vision", False) needs_structured_output = options.get("requires_structured_output", False) - latency_sensitive = bool(_LATENCY_SENSITIVE.search(scanned_full_text)) - high_stakes = bool(_HIGH_STAKES.search(scanned_full_text)) + latency_sensitive = bool(_LATENCY_SENSITIVE.search(scanned_prompt)) + high_stakes = bool(_HIGH_STAKES.search(scanned_prompt)) # Terminal tasks often describe the desired artifact rather than naming a # programming language. Treat only small, deterministic local build/file @@ -569,7 +594,7 @@ def classify_task(prompt: str, system_prompt: str | None, options: RouterOptions and len(prompt) <= 1_000 and simple_terminal_artifact ) - language = "zh" if _HAN.search(scanned_full_text) else "other" + language = "zh" if _HAN.search(scanned_prompt) else "other" multiple_choice_signals = len(_MULTIPLE_CHOICE.findall(scanned_prompt)) numeric_signals = len(_NUMERIC.findall(scanned_prompt)) compact_math_problem = ( @@ -598,7 +623,7 @@ def classify_task(prompt: str, system_prompt: str | None, options: RouterOptions task_type = "reasoning_mcq" elif compact_math_problem: task_type = "reasoning_math" - elif _DEBUG_TASK.search(text): + elif error_class_signal or _DEBUG_TASK.search(text): task_type = "debug" elif has_code or _CODE_EDIT_TASK.search(text): task_type = "code_edit" @@ -626,6 +651,7 @@ def classify_task(prompt: str, system_prompt: str | None, options: RouterOptions terminal_tool_signal=terminal_tool_signal, terminal_safety_sensitive=terminal_safety_sensitive, implicit_terminal_code=implicit_terminal_code, + code_artifact=code_artifact, ) @@ -1063,6 +1089,29 @@ def evidence_candidates(task: TaskType) -> list[str]: return [] +#: Recently added catalog models with a verified structured function call. +#: +#: Tool compliance is enough to make these models safe availability fallbacks, +#: but it is not evidence that they should displace a trajectory-benchmarked +#: primary. Keep them after the configured and benchmark-backed chains until a +#: full agent evaluation earns a task-specific affinity above the neutral 0.68 +#: floor. ``is_eligible`` still applies every host capability override, context +#: limit, vision constraint, and unavailable-model guard before they are used. +VERIFIED_TOOL_FALLBACKS: tuple[str, ...] = ( + "anthropic/claude-fable-5.1", + "anthropic/claude-opus-5.5", + "anthropic/claude-sonnet-5.5", + "google/gemini-3.8-flash", + "openai/gpt-5.1", + "openai/gpt-6-astra", + "openai/gpt-6-luna", + "openai/gpt-6-sol", + "xai/grok-4.6", + "xai/grok-4.7", + "xiaomi/mimo-v2.5", +) + + def is_eligible( model_id: str, features: TaskFeatures, @@ -1168,6 +1217,11 @@ class PortfolioStrategy: name = "portfolio" + def __init__(self) -> None: + # Rules set the capability tier; the strategy is stateless, so one + # instance serves every request. + self._rules = RulesStrategy() + def route( self, prompt: str, @@ -1178,33 +1232,73 @@ def route( features = classify_task(prompt, system_prompt, options) rules_options: RouterOptions = dict(options) # type: ignore[assignment] rules_options["requires_tools"] = features.needs_tools - base = RulesStrategy().route(prompt, system_prompt, max_output_tokens, rules_options) + base = self._rules.route(prompt, system_prompt, max_output_tokens, rules_options) tier_configs = base.get("tier_configs") if not tier_configs: return base - target_tier: Tier = ( - "REASONING" - if features.task_type in ("reasoning_mcq", "reasoning_math") - and base["tier"] in ("SIMPLE", "MEDIUM") - else base["tier"] - ) + # The rules score sets the capability tier, but two task types carry a + # floor of their own: a compact reasoning problem belongs on the + # reasoning chain however short it is, and a code edit or stack trace + # belongs at least on the MEDIUM chain — the SIMPLE chain is curated for + # trivia and translation and has no model with code affinity. The code + # floor requires a code artifact or coding action: `debug` and + # `code_edit` are also assigned from bare verbs ("rewrite this + # sentence", "margin of error") and bare language names ("What is + # Python?"), and that prose must stay on the cheap chain. + target_tier: Tier + if features.task_type in ("reasoning_mcq", "reasoning_math") and base["tier"] in ( + "SIMPLE", + "MEDIUM", + ): + target_tier = "REASONING" + elif ( + features.task_type in ("debug", "code_edit") + and features.code_artifact + and base["tier"] == "SIMPLE" + ): + target_tier = "MEDIUM" + else: + target_tier = base["tier"] tier_config = tier_configs.get(target_tier) configured_candidates = get_fallback_chain(target_tier, tier_configs) if tier_config else [] # Evidence candidates join the configured chain, but the host's # unavailable set applies to both: the configured side arrives filtered # through RulesStrategy, and a dead evidence model must not re-enter here. unavailable = set(options.get("unavailable_models") or ()) + # `model_pricing` is the host's live catalog view. Do not introduce a + # built-in fallback that the current gateway does not advertise. These + # models carry no task evidence, so outside Premium the cheapest one is + # the first recovery hop; otherwise an outage of the curated chain would + # jump an Eco turn straight to the most expensive model in the list. + model_pricing = options["model_pricing"] + verified_tool_fallbacks = ( + [model for model in VERIFIED_TOOL_FALLBACKS if model in model_pricing] + if features.needs_tools + else [] + ) + if options.get("routing_profile") != "premium": + verified_tool_fallbacks.sort( + key=lambda model: estimated_cost( + model, options, features.estimated_input_tokens, max_output_tokens + ) + ) chain = [ model for model in dict.fromkeys( - [*configured_candidates, *evidence_candidates(features.task_type)] + [ + *configured_candidates, + *evidence_candidates(features.task_type), + *verified_tool_fallbacks, + ] ) if isinstance(model, str) and model and model not in unavailable ] eligible = [ model for model in chain if is_eligible(model, features, max_output_tokens, options) ] + eligible_set = set(eligible) + # When nothing is eligible the whole chain competes, scored as incapable. eligible_candidates = eligible if eligible else chain if not eligible_candidates: return base @@ -1279,9 +1373,7 @@ def route( if math.isfinite(cost) and max_cost > min_cost else 0.5 ) - capability_score = ( - 1.0 if is_eligible(model, features, max_output_tokens, options) else 0.0 - ) + capability_score = 1.0 if model in eligible_set else 0.0 profile = profile_score(model, options, now) # Fresh observations can refine affinity. Historical observations # fade quickly and never replace task-level RouterBench evidence. @@ -1292,8 +1384,20 @@ def route( observed_quality = ( model_affinity * (1 - profile.freshness) + profile.quality * profile.freshness ) - observed_speed = profile.speed * profile.freshness if profile else 0.5 - observed_tail_speed = profile.tail_speed * profile.freshness if profile else 0.5 + # Every observed dimension fades toward its neutral prior as the + # observation ages, exactly like reliability below. Fading speed + # toward zero instead ranked any model with a stale probe below a + # model nobody had measured, which the latter's 0.5 silently won. + observed_speed = ( + profile.speed * profile.freshness + 0.5 * (1 - profile.freshness) + if profile + else 0.5 + ) + observed_tail_speed = ( + profile.tail_speed * profile.freshness + 0.5 * (1 - profile.freshness) + if profile + else 0.5 + ) observed_reliability = ( profile.reliability * profile.freshness + (1 - profile.freshness) if profile diff --git a/blockrun_llm/router_core/rules.py b/blockrun_llm/router_core/rules.py index baebe18..29774d8 100644 --- a/blockrun_llm/router_core/rules.py +++ b/blockrun_llm/router_core/rules.py @@ -13,6 +13,9 @@ from __future__ import annotations import math +import re +import unicodedata +from dataclasses import dataclass from ._js import js_regex from .types import DimensionScore, ScoringConfig, ScoringResult, Tier, TokenCountThresholds @@ -25,6 +28,162 @@ _QUESTION_MARK = js_regex(r"\?") +# ─── Keyword Matching ─── +# +# A plain substring test counted `class` in "classic", `story` in "history", +# `edit` in "credit" and `fix` in "prefix", so everyday prose picked up code, +# creative and agentic signals it did not carry. Alphabetic keywords now have +# to start at a word boundary. Short ASCII keywords additionally have to end at +# one, after an optional inflection, graded by how much a stray suffix can +# change the word: a three-letter keyword takes only the English endings that +# cannot spell another word (fix → fixes, never pip → pipe or def → defer); a +# four-to-six-letter keyword also takes the common German and Romance endings, +# because the lists spell those languages in ASCII too (beweis → beweise, +# klasse → klassen); anything longer stays trailing-open like the substring +# test it replaces, since its false-positive surface is small and its +# inflection surface is large. Non-ASCII alphabetic keywords keep a +# trailing-open match because several are deliberate stems (`оптимизаци`, +# `рассуждени`). CJK, Hangul and Arabic keywords keep the substring test: those +# scripts have no word boundaries, or attach particles and articles directly +# to the word. + +_UNSEGMENTED_SCRIPT = re.compile( + "[\u0600-\u06ff\u3000-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uac00-\ud7af\uf900-\ufaff\uff00-\uffef]" +) +_ASCII_PHRASE = re.compile(r"[a-z0-9' -]+") +_JS_REGEX_SPECIALS = re.compile(r"[.*+?^${}()|[\]\\]") + +# A word boundary is a transition out of the segmented scripts. A CJK +# character next to a Latin keyword ("请step by step证明") is a boundary, not +# part of the word: mixed-script prompts carry no spaces between scripts. +# +# Upstream spells the class `[\p{Script=Latin}\p{Script=Cyrillic} +# \p{Script=Greek}\p{N}]`, which the `re` module cannot express. These are its +# code-point ranges (hex, inclusive), enumerated from Node 22's V8 (Unicode +# 16.0) so the boundary does not move with the interpreter's unicodedata. +_SEGMENTED_WORD_RANGES = ( + "30-39 41-5A 61-7A AA B2-B3 B9-BA BC-BE C0-D6 D8-F6 F8-2B8 2E0-2E4 370-373 375-377 " + "37A-37D 37F 384 386 388-38A 38C 38E-3A1 3A3-3E1 3F0-484 487-52F 660-669 6F0-6F9 " + "7C0-7C9 966-96F 9E6-9EF 9F4-9F9 A66-A6F AE6-AEF B66-B6F B72-B77 BE6-BF2 C66-C6F " + "C78-C7E CE6-CEF D58-D5E D66-D78 DE6-DEF E50-E59 ED0-ED9 F20-F33 1040-1049 1090-1099 " + "1369-137C 16EE-16F0 17E0-17E9 17F0-17F9 1810-1819 1946-194F 19D0-19DA 1A80-1A89 " + "1A90-1A99 1B50-1B59 1BB0-1BB9 1C40-1C49 1C50-1C59 1C80-1C8A 1D00-1DBF 1E00-1F15 " + "1F18-1F1D 1F20-1F45 1F48-1F4D 1F50-1F57 1F59 1F5B 1F5D 1F5F-1F7D 1F80-1FB4 1FB6-1FC4 " + "1FC6-1FD3 1FD6-1FDB 1FDD-1FEF 1FF2-1FF4 1FF6-1FFE 2070-2071 2074-2079 207F-2089 " + "2090-209C 2126 212A-212B 2132 214E 2150-2189 2460-249B 24EA-24FF 2776-2793 2C60-2C7F " + "2CFD 2DE0-2DFF 3007 3021-3029 3038-303A 3192-3195 3220-3229 3248-324F 3251-325F " + "3280-3289 32B1-32BF A620-A629 A640-A69F A6E6-A6EF A722-A787 A78B-A7CD A7D0-A7D1 A7D3 " + "A7D5-A7DC A7F2-A7FF A830-A835 A8D0-A8D9 A900-A909 A9D0-A9D9 A9F0-A9F9 AA50-AA59 " + "AB30-AB5A AB5C-AB69 ABF0-ABF9 FB00-FB06 FE2E-FE2F FF10-FF19 FF21-FF3A FF41-FF5A " + "10107-10133 10140-1018E 101A0 102E1-102FB 10320-10323 10341 1034A 103D1-103D5 " + "104A0-104A9 10780-10785 10787-107B0 107B2-107BA 10858-1085F 10879-1087F 108A7-108AF " + "108FB-108FF 10916-1091B 109BC-109BD 109C0-109CF 109D2-109FF 10A40-10A48 10A7D-10A7E " + "10A9D-10A9F 10AEB-10AEF 10B58-10B5F 10B78-10B7F 10BA9-10BAF 10CFA-10CFF 10D30-10D39 " + "10D40-10D49 10E60-10E7E 10F1D-10F26 10F51-10F54 10FC5-10FCB 11052-1106F 110F0-110F9 " + "11136-1113F 111D0-111D9 111E1-111F4 112F0-112F9 11450-11459 114D0-114D9 11650-11659 " + "116C0-116C9 116D0-116E3 11730-1173B 118E0-118F2 11950-11959 11BF0-11BF9 11C50-11C6C " + "11D50-11D59 11DA0-11DA9 11F50-11F59 11FC0-11FD4 12400-1246E 16130-16139 16A60-16A69 " + "16AC0-16AC9 16B50-16B59 16B5B-16B61 16D70-16D79 16E80-16E96 1CCF0-1CCF9 1D200-1D245 " + "1D2C0-1D2D3 1D2E0-1D2F3 1D360-1D378 1D7CE-1D7FF 1DF00-1DF1E 1DF25-1DF2A 1E030-1E06D " + "1E08F 1E140-1E149 1E2F0-1E2F9 1E4F0-1E4F9 1E5F1-1E5FA 1E8C7-1E8CF 1E950-1E959 " + "1EC71-1ECAB 1ECAD-1ECAF 1ECB1-1ECB4 1ED01-1ED2D 1ED2F-1ED3D 1F100-1F10C 1FBF0-1FBF9" +) +_SEGMENTED_WORD_CHAR = "".join( + ( + f"\\U{int(span, 16):08X}" + if "-" not in span + else "\\U{:08X}-\\U{:08X}".format(*(int(end, 16) for end in span.split("-"))) + ) + for span in _SEGMENTED_WORD_RANGES.split() +) +_BOUNDARY_BEFORE = f"(? str: + """Port of the upstream ``escapeRegExp`` (escapes only regex metacharacters).""" + return _JS_REGEX_SPECIALS.sub(lambda match: f"\\{match.group(0)}", value) + + +def _is_word_char(char: str) -> bool: + """Port of ``/[\\p{L}\\p{N}]/u``.""" + return unicodedata.category(char)[0] in ("L", "N") + + +def _keyword_matcher(keyword: str) -> _KeywordMatcher: + cached = _matcher_cache.get(keyword) + if cached is not None: + return cached + + kw = keyword.lower() + if not kw or _UNSEGMENTED_SCRIPT.search(kw): + matcher = _KeywordMatcher(needle=kw, pattern=None) + else: + lead = _BOUNDARY_BEFORE if _is_word_char(kw[0]) else "" + escaped = _escape_reg_exp(kw) + body = escaped + trail = "" + needle = kw + last_word = kw[kw.rfind(" ") + 1 :] + if ( + _is_word_char(kw[-1]) + and _ASCII_PHRASE.fullmatch(kw) + and len(last_word) <= _BOUNDED_LAST_WORD_LENGTH + ): + # ASCII phrases contain no regex metacharacters, so `escaped` slices + # align with `kw` slices below. + forms: list[str] = [] + if kw[-1].isdigit(): + # "step 1" must not absorb "step 10". + forms.append(escaped) + elif len(last_word) <= 3: + forms.append(f"{escaped}{_SHORT_SUFFIXES}") + if kw.endswith(("x", "s", "z", "ch", "sh")): + forms.append(f"{escaped}{_SIBILANT_PLURAL}") + else: + forms.append(f"{escaped}{_MID_SUFFIXES}") + if kw.endswith("e"): + forms.append(f"{escaped[:-1]}{_DROPPED_E_SUFFIXES}") + needle = kw[:-1] + elif kw.endswith("y"): + forms.append(f"{escaped[:-1]}(?:ies|ied)") + needle = kw[:-1] + elif len(kw) >= 2 and kw[-2] in "aeiou" and kw[-1] in "bdgmnprt": + # debug → debugging, plan → planned + forms.append(f"{escaped}{kw[-1]}{_DOUBLED_CONSONANT_SUFFIXES}") + body = f"(?:{'|'.join(forms)})" + trail = _BOUNDARY_AFTER + matcher = _KeywordMatcher(needle=needle, pattern=re.compile(f"{lead}{body}{trail}")) + _matcher_cache[keyword] = matcher + return matcher + + +def _matches_keyword(text: str, keyword: str) -> bool: + """Whether lower-cased ``text`` contains ``keyword`` as a word (or as a substring for CJK).""" + matcher = _keyword_matcher(keyword) + if matcher.needle not in text: + return False + return True if matcher.pattern is None else matcher.pattern.search(text) is not None + + # ─── Dimension Scorers ─── # Each returns a score in [-1, 1] and an optional signal string. @@ -55,7 +214,7 @@ def _score_keyword_match( """``thresholds`` is ``(low, high)``; ``scores`` is ``(none, low, high)``.""" low_threshold, high_threshold = thresholds none_score, low_score, high_score = scores - matches = [keyword for keyword in keywords if keyword.lower() in text] + matches = [keyword for keyword in keywords if _matches_keyword(text, keyword)] if len(matches) >= high_threshold: return { "name": name, @@ -96,7 +255,7 @@ def _score_agentic_task(text: str, keywords: list[str]) -> tuple[DimensionScore, signals: list[str] = [] for keyword in keywords: - if keyword.lower() in text: + if _matches_keyword(text, keyword): match_count += 1 if len(signals) < 3: signals.append(keyword) @@ -252,7 +411,7 @@ def classify_by_rules( # Count reasoning markers for override — only the USER prompt, so a system # prompt saying "step by step" cannot force REASONING for simple queries. reasoning_matches = [ - keyword for keyword in config["reasoning_keywords"] if keyword.lower() in user_text + keyword for keyword in config["reasoning_keywords"] if _matches_keyword(user_text, keyword) ] # Direct reasoning override: 2+ reasoning markers = high confidence REASONING diff --git a/blockrun_llm/router_core/selector.py b/blockrun_llm/router_core/selector.py index 7546465..d63364d 100644 --- a/blockrun_llm/router_core/selector.py +++ b/blockrun_llm/router_core/selector.py @@ -105,7 +105,11 @@ def select_model( "reasoning": reasoning, "cost_estimate": cost_estimate, "baseline_cost": baseline_cost, - "savings": _savings(cost_estimate, baseline_cost, routing_profile), + # A model the host has no price for costs an unknown amount, not + # nothing; reporting its savings as 100% would be a fabricated figure. + "savings": ( + 0.0 if pricing is None else _savings(cost_estimate, baseline_cost, routing_profile) + ), } if agentic_score is not None: decision["agentic_score"] = agentic_score diff --git a/blockrun_llm/router_core/strategy.py b/blockrun_llm/router_core/strategy.py index 4cfd156..a4245b2 100644 --- a/blockrun_llm/router_core/strategy.py +++ b/blockrun_llm/router_core/strategy.py @@ -10,14 +10,14 @@ from __future__ import annotations import copy -import math from collections.abc import Sequence from datetime import datetime from typing import Protocol -from ._js import as_utc, js_regex, parse_date, to_fixed +from ._js import as_utc, parse_date, to_fixed from .rules import classify_by_rules from .selector import select_model +from .tokens import estimate_tokens, sample_for_scan, scan_limit from .types import ( TIER_RANK, Profile, @@ -28,8 +28,6 @@ TierConfig, ) -_STRUCTURED_OUTPUT = js_regex(r"json|structured|schema", ignorecase=True) - class RouterStrategy(Protocol): """Interface implemented by every routing strategy.""" @@ -45,20 +43,6 @@ def route( ) -> RoutingDecision: ... -def sample_prompt(value: str, scan_limit: int) -> str: - """Sample both ends of a long prompt, keeping instructions at either edge.""" - if len(value) <= scan_limit: - return value - prefix_length = math.ceil(scan_limit / 2) - suffix_length = scan_limit - prefix_length - suffix = value[-suffix_length:] if suffix_length else value - return f"{value[:prefix_length]}\n{suffix}" - - -def scan_limit_for(options: RouterOptions) -> int: - return max(1, min(8_000, options["config"]["classifier"]["prompt_truncation_chars"])) - - def apply_unavailable_models( tier_configs: dict[str, TierConfig], unavailable_models: Sequence[str] | None, @@ -149,12 +133,11 @@ def route( config = options["config"] model_pricing = options["model_pricing"] - # Estimate input tokens (~4 chars per token) full_text = f"{system_prompt or ''} {prompt}" - estimated_tokens = math.ceil(len(full_text) / 4) - scan_limit = scan_limit_for(options) - scanned_prompt = sample_prompt(prompt, scan_limit) - scanned_system_prompt = sample_prompt(system_prompt, scan_limit) if system_prompt else None + estimated_tokens = estimate_tokens(full_text) + limit = scan_limit(config["classifier"]["prompt_truncation_chars"]) + scanned_prompt = sample_for_scan(prompt, limit) + scanned_system_prompt = sample_for_scan(system_prompt, limit) if system_prompt else None # --- Rule-based classification (runs first to get agentic_score) --- rule_result = classify_by_rules( @@ -245,12 +228,11 @@ def route( decision["profile"] = profile return decision - # Structured output detection - has_structured_output = options.get("requires_structured_output") is True or ( - bool(_STRUCTURED_OUTPUT.search(scanned_system_prompt)) - if scanned_system_prompt - else False - ) + # Structured output is a protocol fact the host states (``response_format`` + # → requires_structured_output). It used to be inferred from the system + # prompt as well, but agent hosts describe their tool schemas there on + # every turn, which upgraded every one of their requests off SIMPLE. + has_structured_output = options.get("requires_structured_output") is True tier: Tier signals = ", ".join(rule_result.get("signals", [])) diff --git a/blockrun_llm/router_core/tokens.py b/blockrun_llm/router_core/tokens.py new file mode 100644 index 0000000..e2aadc8 --- /dev/null +++ b/blockrun_llm/router_core/tokens.py @@ -0,0 +1,54 @@ +""" +Request-size helpers shared by the rules and portfolio strategies. + +Python port of ``@blockrun/router-core`` ``tokens.ts``. + +The router never tokenizes — it is on the hot path and model-agnostic — so +sizes are estimates. The estimate only has to be honest enough for the +decisions that consume it: the long-context task type, the COMPLEX override +and the per-model capacity filter. +""" + +from __future__ import annotations + +import math +import re + +# Han, kana, Hangul, CJK punctuation and fullwidth forms. Every production +# tokenizer spends roughly one token per character here, so the Latin +# four-characters-per-token rule under-counts these scripts about fourfold: a +# 200k-character Chinese contract would read as 50k tokens, slip under the 80k +# long-context boundary and pass the capacity filter for models that cannot +# hold it. +_CJK_CHARS = re.compile( + "[\u3000-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uac00-\ud7af\uf900-\ufaff\uff00-\uffef]" +) + + +def estimate_tokens(text: str) -> int: + """Approximate token count: one per CJK character, one per four of anything else.""" + if not text: + return 0 + cjk = len(_CJK_CHARS.findall(text)) + return cjk + math.ceil((len(text) - cjk) / 4) + + +def scan_limit(prompt_truncation_chars: int) -> int: + """The scan window the feature regexes read, bounded so a pasted document cannot stall routing.""" + return max(1, min(8_000, prompt_truncation_chars)) + + +def sample_for_scan(value: str, limit: int) -> str: + """Sample both ends of a long value for the feature regexes. + + Feature regexes need request shape and intent, not the entire document. + Sampling both ends keeps the task instruction of a long pasted artifact at + either boundary, while the full length still drives capacity decisions. + """ + if len(value) <= limit: + return value + prefix_length = math.ceil(limit / 2) + suffix_length = limit - prefix_length + # JS ``slice(-0)`` is the whole string, not an empty one. + suffix = value[-suffix_length:] if suffix_length else value + return f"{value[:prefix_length]}\n{suffix}" diff --git a/tests/unit/router_core_decisions.snapshot.json b/tests/unit/router_core_decisions.snapshot.json index 04f259a..5e04602 100644 --- a/tests/unit/router_core_decisions.snapshot.json +++ b/tests/unit/router_core_decisions.snapshot.json @@ -25,10 +25,10 @@ "candidateScores": [ { "model": "google/gemini-2.5-flash", - "score": 0.7870260539502253, + "score": 0.8115260539502253, "quality": 0.86, "cost": 0.5, - "speed": 0.18322934214607486, + "speed": 0.5332293421460749, "reliability": 1 } ], @@ -42,7 +42,7 @@ "tier": "SIMPLE", "confidence": 0.7231218051243898, "method": "portfolio", - "reasoning": "score=-0.08 | short (1 tokens) | agentic (tools) | v3 task=tool_agent agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=13", + "reasoning": "score=-0.08 | short (1 tokens) | agentic (tools) | v3 task=tool_agent agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=24", "costEstimate": 0.0032001, "baselineCost": 0.032005, "savings": 0.9000124980471801, @@ -60,15 +60,26 @@ "google/gemini-3.5-flash", "openai/gpt-5.3-codex", "moonshot/kimi-k3", - "deepseek/deepseek-v4-pro" + "deepseek/deepseek-v4-pro", + "google/gemini-3.8-flash", + "openai/gpt-6-luna", + "openai/gpt-6-sol", + "openai/gpt-5.1", + "xai/grok-4.6", + "anthropic/claude-opus-5.5", + "xai/grok-4.7", + "anthropic/claude-sonnet-5.5", + "openai/gpt-6-astra", + "xiaomi/mimo-v2.5", + "anthropic/claude-fable-5.1" ], "candidateScores": [ { "model": "anthropic/claude-sonnet-5", - "score": 0.8469181450377915, + "score": 0.8714181450377915, "quality": 1, "cost": 0.5, - "speed": 0.09883064339702209, + "speed": 0.44883064339702206, "reliability": 1 } ], @@ -95,10 +106,10 @@ "candidateScores": [ { "model": "google/gemini-2.5-flash", - "score": 0.7024260539502254, + "score": 0.7269260539502254, "quality": 0.68, "cost": 0.5, - "speed": 0.18322934214607486, + "speed": 0.5332293421460749, "reliability": 1 } ], @@ -133,10 +144,10 @@ "candidateScores": [ { "model": "google/gemini-3.5-flash", - "score": 0.7030477432793295, + "score": 0.7275477432793295, "quality": 0.68, "cost": 0.5, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 } ], @@ -167,18 +178,18 @@ "candidateScores": [ { "model": "deepseek/deepseek-v4-pro", - "score": 0.9113686326916596, + "score": 0.9358686326916595, "quality": 0.95, "cost": 1, - "speed": 0.06955189559513505, + "speed": 0.41955189559513506, "reliability": 1 }, { "model": "google/gemini-3.5-flash", - "score": 0.6758477432793295, + "score": 0.7003477432793295, "quality": 0.92, "cost": 0, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 } ], @@ -213,10 +224,10 @@ "candidateScores": [ { "model": "google/gemini-3.5-flash", - "score": 0.7030477432793295, + "score": 0.7275477432793295, "quality": 0.68, "cost": 0.5, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 } ], @@ -230,7 +241,7 @@ "tier": "SIMPLE", "confidence": 0.7231218051243898, "method": "portfolio", - "reasoning": "score=-0.08 | short (20 tokens) | agentic (tools) | v3 task=tool_agent_parallel agentRisk=high deepWebResearch=false terminalCode=false terminalSafety=false candidates=13", + "reasoning": "score=-0.08 | short (20 tokens) | agentic (tools) | v3 task=tool_agent_parallel agentRisk=high deepWebResearch=false terminalCode=false terminalSafety=false candidates=24", "costEstimate": 0.020291200000000002, "baselineCost": 0.0577, "savings": 0.6483327556325823, @@ -248,15 +259,26 @@ "openai/gpt-4.1", "xai/grok-4.5", "google/gemini-3.5-flash", - "deepseek/deepseek-v4-pro" + "deepseek/deepseek-v4-pro", + "google/gemini-3.8-flash", + "openai/gpt-6-luna", + "openai/gpt-6-sol", + "openai/gpt-5.1", + "xai/grok-4.6", + "anthropic/claude-opus-5.5", + "xai/grok-4.7", + "anthropic/claude-sonnet-5.5", + "openai/gpt-6-astra", + "xiaomi/mimo-v2.5", + "anthropic/claude-fable-5.1" ], "candidateScores": [ { "model": "anthropic/claude-opus-4.8", - "score": 0.8468563440295361, + "score": 0.8713563440295362, "quality": 1, "cost": 0.5, - "speed": 0.09794777185051722, + "speed": 0.4479477718505172, "reliability": 1 } ], @@ -283,10 +305,10 @@ "candidateScores": [ { "model": "google/gemini-2.5-flash", - "score": 0.7024260539502254, + "score": 0.7269260539502254, "quality": 0.68, "cost": 0.5, - "speed": 0.18322934214607486, + "speed": 0.5332293421460749, "reliability": 1 } ], @@ -300,10 +322,10 @@ "tier": "MEDIUM", "confidence": 0.5, "method": "portfolio", - "reasoning": "score=-0.07 | short (5 tokens), constraints (不超过) | ambiguous -> default: MEDIUM | v3 task=extraction agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=13", - "costEstimate": 0.0017297000000000002, - "baselineCost": 0.006424999999999999, - "savings": 0.7307859922178989, + "reasoning": "score=-0.07 | short (20 tokens), constraints (不超过) | ambiguous -> default: MEDIUM | v3 task=extraction agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=13", + "costEstimate": 0.0017732000000000004, + "baselineCost": 0.0065, + "savings": 0.7272, "agenticScore": 0, "candidates": [ "moonshot/kimi-k3", @@ -323,10 +345,10 @@ "candidateScores": [ { "model": "moonshot/kimi-k3", - "score": 0.8419118013428357, + "score": 0.8664118013428358, "quality": 1, "cost": 0.5, - "speed": 0.02731144775479715, + "speed": 0.37731144775479714, "reliability": 1 } ], @@ -340,10 +362,10 @@ "tier": "MEDIUM", "confidence": 0.5, "method": "portfolio", - "reasoning": "score=-0.07 | short (16 tokens), imperative (设计), references (代码) | ambiguous -> default: MEDIUM | v3 task=chat agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=11", - "costEstimate": 0.0005584000000000001, - "baselineCost": 0.03208, - "savings": 0.9825935162094763, + "reasoning": "score=-0.07 | short (41 tokens), imperative (设计), references (代码) | ambiguous -> default: MEDIUM | v3 task=chat agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=11", + "costEstimate": 0.0006309000000000001, + "baselineCost": 0.032205, + "savings": 0.9804098742431301, "agenticScore": 0, "candidates": [ "google/gemini-2.5-flash", @@ -361,18 +383,18 @@ "candidateScores": [ { "model": "google/gemini-2.5-flash", - "score": 0.8270260539502253, + "score": 0.8515260539502253, "quality": 0.86, "cost": 1, - "speed": 0.18322934214607486, + "speed": 0.5332293421460749, "reliability": 1 }, { "model": "google/gemini-3.5-flash", - "score": 0.6976477432793294, + "score": 0.7221477432793294, "quality": 0.86, "cost": 0, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 } ], @@ -384,9 +406,9 @@ "profile": "auto", "model": "google/gemini-3.5-flash", "tier": "MEDIUM", - "confidence": 0.5, + "confidence": 0.7034956909518112, "method": "portfolio", - "reasoning": "score=-0.02 | short (23 tokens), code (def), simple (define), agentic-light (debug) | ambiguous -> default: MEDIUM | v3 task=debug agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=11", + "reasoning": "score=-0.07 | short (23 tokens), agentic-light (debug) | v3 task=debug agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=11", "costEstimate": 0.015519600000000001, "baselineCost": 0.057714999999999995, "savings": 0.7310993675820844, @@ -407,10 +429,10 @@ "candidateScores": [ { "model": "google/gemini-3.5-flash", - "score": 0.7030477432793295, + "score": 0.7275477432793295, "quality": 0.68, "cost": 0.5, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 } ], @@ -424,7 +446,7 @@ "tier": "MEDIUM", "confidence": 0.7373034537835593, "method": "portfolio", - "reasoning": "score=0.09 | long (1217 tokens), references (following) | agentic (tools) | v3 task=tool_agent agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=13", + "reasoning": "score=0.09 | long (1217 tokens), references (following) | agentic (tools) | v3 task=tool_agent agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=24", "costEstimate": 0.0084417, "baselineCost": 0.089285, "savings": 0.905452203617629, @@ -442,15 +464,26 @@ "anthropic/claude-opus-5", "openai/gpt-4.1", "openai/gpt-5.3-codex", - "deepseek/deepseek-v4-pro" + "deepseek/deepseek-v4-pro", + "google/gemini-3.8-flash", + "openai/gpt-6-luna", + "openai/gpt-6-sol", + "openai/gpt-5.1", + "xai/grok-4.6", + "anthropic/claude-opus-5.5", + "xai/grok-4.7", + "anthropic/claude-sonnet-5.5", + "openai/gpt-6-astra", + "xiaomi/mimo-v2.5", + "anthropic/claude-fable-5.1" ], "candidateScores": [ { "model": "anthropic/claude-sonnet-5", - "score": 0.8469181450377915, + "score": 0.8714181450377915, "quality": 1, "cost": 0.5, - "speed": 0.09883064339702209, + "speed": 0.44883064339702206, "reliability": 1 } ], @@ -464,7 +497,7 @@ "tier": "MEDIUM", "confidence": 0.5, "method": "portfolio", - "reasoning": "score=0.06 | short (32 tokens), code (let), multi-step, references (following) | ambiguous -> default: MEDIUM | v3 task=vision agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=7", + "reasoning": "score=-0.01 | short (32 tokens), multi-step, references (following) | ambiguous -> default: MEDIUM | v3 task=vision agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=7", "costEstimate": 0.0018304000000000003, "baselineCost": 0.00656, "savings": 0.7209756097560975, @@ -481,10 +514,10 @@ "candidateScores": [ { "model": "google/gemini-3.5-flash", - "score": 0.7030477432793295, + "score": 0.7275477432793295, "quality": 0.68, "cost": 0.5, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 } ], @@ -518,42 +551,42 @@ "candidateScores": [ { "model": "xai/grok-4.5", - "score": 0.8761979445576786, + "score": 0.9006979445576787, "quality": 0.93, "cost": 1, - "speed": 0.05854206510969567, + "speed": 0.40854206510969565, "reliability": 1 }, { "model": "anthropic/claude-sonnet-5", - "score": 0.7931612595363191, + "score": 0.8176612595363191, "quality": 0.9, "cost": 0.6707950805473757, - "speed": 0.09883064339702209, + "speed": 0.44883064339702206, "reliability": 1 }, { "model": "deepseek/deepseek-v4-pro", - "score": 0.7683415237361771, + "score": 0.7928415237361771, "quality": 0.9, "cost": 0.3359605058028754, - "speed": 0.06955189559513505, + "speed": 0.41955189559513506, "reliability": 1 }, { "model": "google/gemini-3.5-flash", - "score": 0.7509477432793293, + "score": 0.7754477432793293, "quality": 1, "cost": 0, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 }, { "model": "moonshot/kimi-k3", - "score": 0.6551144689333432, + "score": 0.6796144689333433, "quality": 0.9, "cost": 0.001125931058375107, - "speed": 0.02731144775479715, + "speed": 0.37731144775479714, "reliability": 1 } ], @@ -590,34 +623,34 @@ "candidateScores": [ { "model": "google/gemini-2.5-flash", - "score": 0.8791593872835586, + "score": 0.9036593872835585, "quality": 0.9, "cost": 1, - "speed": 0.18322934214607486, + "speed": 0.5332293421460749, "reliability": 1 }, { "model": "anthropic/claude-sonnet-5", - "score": 0.7808574061523814, + "score": 0.8053574061523814, "quality": 0.9, "cost": 0.6718847839699437, - "speed": 0.09883064339702209, + "speed": 0.44883064339702206, "reliability": 1 }, { "model": "google/gemini-3.5-flash", - "score": 0.7164477432793295, + "score": 0.7409477432793294, "quality": 0.9, "cost": 0, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 }, { "model": "moonshot/kimi-k3", - "score": 0.6717952205744078, + "score": 0.6962952205744078, "quality": 0.9, "cost": 0.001204180916140718, - "speed": 0.02731144775479715, + "speed": 0.37731144775479714, "reliability": 1 } ], @@ -628,42 +661,32 @@ "prompt": 15, "profile": "auto", "model": "google/gemini-2.5-flash", - "tier": "MEDIUM", - "confidence": 0.5, + "tier": "SIMPLE", + "confidence": 0.7009865197502964, "method": "portfolio", - "reasoning": "score=-0.06 | short (29 tokens), imperative (build), agentic-light (pip) | ambiguous -> default: MEDIUM | v3 task=chat agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=11", + "reasoning": "score=-0.07 | short (29 tokens), imperative (build) | v3 task=chat agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=9", "costEstimate": 0.0014153, "baselineCost": 0.083345, "savings": 0.983018777371168, - "agenticScore": 0.2, + "agenticScore": 0, "candidates": [ "google/gemini-2.5-flash", - "google/gemini-3.5-flash", - "google/gemini-3.6-flash", - "zai/glm-5.3-flash", - "openai/gpt-5.6-terra", "google/gemini-3-flash-preview", + "google/gemini-3.5-flash-lite", "deepseek/deepseek-chat", - "minimax/minimax-m3", "google/gemini-3.1-flash-lite", "openai/gpt-5.6-luna", - "google/gemini-2.5-flash-lite" + "openai/gpt-5.4-nano", + "google/gemini-2.5-flash-lite", + "nvidia/nemotron-3.5-lightning" ], "candidateScores": [ { "model": "google/gemini-2.5-flash", - "score": 0.8270260539502253, - "quality": 0.86, - "cost": 1, - "speed": 0.18322934214607486, - "reliability": 1 - }, - { - "model": "google/gemini-3.5-flash", - "score": 0.6976477432793294, + "score": 0.8115260539502253, "quality": 0.86, - "cost": 0, - "speed": 0.19211061827613424, + "cost": 0.5, + "speed": 0.5332293421460749, "reliability": 1 } ], @@ -677,7 +700,7 @@ "tier": "MEDIUM", "confidence": 0.5, "method": "portfolio", - "reasoning": "score=-0.06 | short (38 tokens), imperative (deploy), references (the api), agentic-light (check the, deploy) | ambiguous -> default: MEDIUM | agentic (tools) | v3 task=tool_agent agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=13", + "reasoning": "score=-0.06 | short (38 tokens), imperative (deploy), references (the api), agentic-light (check the, deploy) | ambiguous -> default: MEDIUM | agentic (tools) | v3 task=tool_agent agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=24", "costEstimate": 0.0006437999999999999, "baselineCost": 0.0065899999999999995, "savings": 0.9023065250379363, @@ -695,15 +718,26 @@ "anthropic/claude-opus-5", "openai/gpt-4.1", "openai/gpt-5.3-codex", - "deepseek/deepseek-v4-pro" + "deepseek/deepseek-v4-pro", + "google/gemini-3.8-flash", + "openai/gpt-6-luna", + "openai/gpt-6-sol", + "openai/gpt-5.1", + "xai/grok-4.6", + "anthropic/claude-opus-5.5", + "xai/grok-4.7", + "anthropic/claude-sonnet-5.5", + "openai/gpt-6-astra", + "xiaomi/mimo-v2.5", + "anthropic/claude-fable-5.1" ], "candidateScores": [ { "model": "anthropic/claude-sonnet-5", - "score": 0.8469181450377915, + "score": 0.8714181450377915, "quality": 1, "cost": 0.5, - "speed": 0.09883064339702209, + "speed": 0.44883064339702206, "reliability": 1 } ], @@ -734,10 +768,10 @@ "candidateScores": [ { "model": "google/gemini-3.5-flash", - "score": 0.7030477432793295, + "score": 0.7275477432793295, "quality": 0.68, "cost": 0.5, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 } ], @@ -774,34 +808,34 @@ "candidateScores": [ { "model": "google/gemini-2.5-flash", - "score": 0.8872869559818712, + "score": 0.9397869559818712, "quality": 0.9, "cost": 1, - "speed": 0.13969081765691935, + "speed": 0.48969081765691935, "reliability": 1 }, { "model": "anthropic/claude-sonnet-5", - "score": 0.7946345209396576, + "score": 0.8471345209396577, "quality": 0.9, "cost": 0.6729268997399596, - "speed": 0.1367178599097662, + "speed": 0.4867178599097662, "reliability": 1 }, { "model": "google/gemini-3.5-flash", - "score": 0.7278627942097315, + "score": 0.7803627942097315, "quality": 0.9, "cost": 0, - "speed": 0.16575196139820988, + "speed": 0.5157519613982099, "reliability": 1 }, { "model": "moonshot/kimi-k3", - "score": 0.6732711638661456, + "score": 0.7257711638661457, "quality": 0.9, "cost": 0.0014446691707599157, - "speed": 0.022296378324947415, + "speed": 0.3722963783249474, "reliability": 1 } ], @@ -835,42 +869,42 @@ "candidateScores": [ { "model": "xai/grok-4.5", - "score": 0.8761979445576786, + "score": 0.9006979445576787, "quality": 0.93, "cost": 1, - "speed": 0.05854206510969567, + "speed": 0.40854206510969565, "reliability": 1 }, { "model": "anthropic/claude-sonnet-5", - "score": 0.7931437096989948, + "score": 0.8176437096989948, "quality": 0.9, "cost": 0.6706975814511295, - "speed": 0.09883064339702209, + "speed": 0.44883064339702206, "reliability": 1 }, { "model": "deepseek/deepseek-v4-pro", - "score": 0.7683303556578799, + "score": 0.7928303556578798, "quality": 0.9, "cost": 0.335898460923446, - "speed": 0.06955189559513505, + "speed": 0.41955189559513506, "reliability": 1 }, { "model": "google/gemini-3.5-flash", - "score": 0.7509477432793293, + "score": 0.7754477432793293, "quality": 1, "cost": 0, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 }, { "model": "moonshot/kimi-k3", - "score": 0.6551096826140731, + "score": 0.6796096826140732, "quality": 0.9, "cost": 0.001099340395762649, - "speed": 0.02731144775479715, + "speed": 0.37731144775479714, "reliability": 1 } ], @@ -905,10 +939,10 @@ "candidateScores": [ { "model": "google/gemini-3.5-flash", - "score": 0.7030477432793295, + "score": 0.7275477432793295, "quality": 0.68, "cost": 0.5, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 } ], @@ -922,7 +956,7 @@ "tier": "MEDIUM", "confidence": 0.7231218051243898, "method": "portfolio", - "reasoning": "score=0.08 | long (14423 tokens) | agentic (tools) | v3 task=tool_agent agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=13", + "reasoning": "score=0.08 | long (14423 tokens) | agentic (tools) | v3 task=tool_agent agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=24", "costEstimate": 0.0046423, "baselineCost": 0.104115, "savings": 0.9554118042549105, @@ -940,15 +974,26 @@ "anthropic/claude-opus-5", "openai/gpt-4.1", "openai/gpt-5.3-codex", - "deepseek/deepseek-v4-pro" + "deepseek/deepseek-v4-pro", + "anthropic/claude-opus-5.5", + "openai/gpt-5.1", + "google/gemini-3.8-flash", + "openai/gpt-6-luna", + "xai/grok-4.6", + "anthropic/claude-sonnet-5.5", + "openai/gpt-6-sol", + "openai/gpt-6-astra", + "xiaomi/mimo-v2.5", + "xai/grok-4.7", + "anthropic/claude-fable-5.1" ], "candidateScores": [ { "model": "anthropic/claude-sonnet-5", - "score": 0.8469181450377915, + "score": 0.8714181450377915, "quality": 1, "cost": 0.5, - "speed": 0.09883064339702209, + "speed": 0.44883064339702206, "reliability": 1 } ], @@ -996,7 +1041,7 @@ "tier": "SIMPLE", "confidence": 0.7231218051243898, "method": "portfolio", - "reasoning": "score=-0.08 | short (1 tokens) | eco | v3 task=tool_agent agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=14", + "reasoning": "score=-0.08 | short (1 tokens) | eco | v3 task=tool_agent agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=25", "costEstimate": 0.0032001, "baselineCost": 0.032005, "savings": 0.9000124980471801, @@ -1015,55 +1060,66 @@ "google/gemini-3.1-flash-lite", "anthropic/claude-opus-5", "openai/gpt-4.1", - "openai/gpt-4o-mini" + "openai/gpt-4o-mini", + "google/gemini-3.8-flash", + "openai/gpt-6-luna", + "openai/gpt-6-sol", + "openai/gpt-5.1", + "xai/grok-4.6", + "anthropic/claude-opus-5.5", + "xai/grok-4.7", + "anthropic/claude-sonnet-5.5", + "openai/gpt-6-astra", + "xiaomi/mimo-v2.5", + "anthropic/claude-fable-5.1" ], "candidateScores": [ { "model": "anthropic/claude-sonnet-5", - "score": 0.9098830643397023, + "score": 0.9448830643397024, "quality": 1, "cost": 1, - "speed": 0.09883064339702209, + "speed": 0.44883064339702206, "reliability": 1 }, { "model": "openai/gpt-5-mini", - "score": 0.8516673859004991, + "score": 0.886667385900499, "quality": 0.84, "cost": 0.9996096291476904, - "speed": 0.13376689739145697, + "speed": 0.48376689739145695, "reliability": 1 }, { "model": "openai/gpt-5.3-codex", - "score": 0.8370981461564362, + "score": 0.8720981461564362, "quality": 0.87, "cost": 0.9997397527651268, - "speed": 0.039714153822005965, + "speed": 0.3897141538220059, "reliability": 0.79999 }, { "model": "deepseek/deepseek-v4-pro", - "score": 0.6821734068659546, + "score": 0.7171734068659547, "quality": 0.82, "cost": 0.5000650618087183, - "speed": 0.06955189559513505, + "speed": 0.41955189559513506, "reliability": 1 }, { "model": "google/gemini-3.5-flash", - "score": 0.5880110618276134, + "score": 0.6230110618276133, "quality": 0.88, "cost": 0, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 }, { "model": "moonshot/kimi-k3", - "score": 0.552767579388362, + "score": 0.5877675793883619, "quality": 0.85, "cost": 0.00013012361743647283, - "speed": 0.02731144775479715, + "speed": 0.37731144775479714, "reliability": 1 } ], @@ -1089,10 +1145,10 @@ "candidateScores": [ { "model": "zai/glm-5.3-flash", - "score": 0.6505854692782567, + "score": 0.6855854692782567, "quality": 0.68, "cost": 0.5, - "speed": 0.05785469278256664, + "speed": 0.40785469278256664, "reliability": 1 } ], @@ -1123,10 +1179,10 @@ "candidateScores": [ { "model": "zai/glm-5.3-flash", - "score": 0.6505854692782567, + "score": 0.6855854692782567, "quality": 0.68, "cost": 0.5, - "speed": 0.05785469278256664, + "speed": 0.40785469278256664, "reliability": 1 } ], @@ -1155,10 +1211,10 @@ "candidateScores": [ { "model": "deepseek/deepseek-v4-pro", - "score": 0.7489551895595136, + "score": 0.7839551895595136, "quality": 0.95, "cost": 0.5, - "speed": 0.06955189559513505, + "speed": 0.41955189559513506, "reliability": 1 } ], @@ -1189,10 +1245,10 @@ "candidateScores": [ { "model": "zai/glm-5.3-flash", - "score": 0.6505854692782567, + "score": 0.6855854692782567, "quality": 0.68, "cost": 0.5, - "speed": 0.05785469278256664, + "speed": 0.40785469278256664, "reliability": 1 } ], @@ -1206,7 +1262,7 @@ "tier": "SIMPLE", "confidence": 0.7231218051243898, "method": "portfolio", - "reasoning": "score=-0.08 | short (20 tokens) | eco | v3 task=tool_agent_parallel agentRisk=high deepWebResearch=false terminalCode=false terminalSafety=false candidates=14", + "reasoning": "score=-0.08 | short (20 tokens) | eco | v3 task=tool_agent_parallel agentRisk=high deepWebResearch=false terminalCode=false terminalSafety=false candidates=25", "costEstimate": 0.0009656, "baselineCost": 0.0577, "savings": 0.9832651646447141, @@ -1225,47 +1281,58 @@ "anthropic/claude-opus-5", "openai/gpt-5-mini", "openai/gpt-4.1", - "openai/gpt-4o-mini" + "openai/gpt-4o-mini", + "google/gemini-3.8-flash", + "openai/gpt-6-luna", + "openai/gpt-6-sol", + "openai/gpt-5.1", + "xai/grok-4.6", + "anthropic/claude-opus-5.5", + "xai/grok-4.7", + "anthropic/claude-sonnet-5.5", + "openai/gpt-6-astra", + "xiaomi/mimo-v2.5", + "anthropic/claude-fable-5.1" ], "candidateScores": [ { "model": "xai/grok-4.5", - "score": 0.8310542065109696, + "score": 0.8660542065109696, "quality": 0.82, "cost": 1, - "speed": 0.05854206510969567, + "speed": 0.40854206510969565, "reliability": 1 }, { "model": "anthropic/claude-sonnet-5", - "score": 0.777790163731183, + "score": 0.8127901637311831, "quality": 0.84, "cost": 0.7518110692552884, - "speed": 0.09883064339702209, + "speed": 0.44883064339702206, "reliability": 1 }, { "model": "deepseek/deepseek-v4-pro", - "score": 0.6677551895595135, + "score": 0.7027551895595135, "quality": 0.78, "cost": 0.5, - "speed": 0.06955189559513505, + "speed": 0.41955189559513506, "reliability": 1 }, { "model": "anthropic/claude-opus-4.8", - "score": 0.6297947771850518, + "score": 0.6647947771850519, "quality": 1, "cost": 0, - "speed": 0.09794777185051722, + "speed": 0.4479477718505172, "reliability": 1 }, { "model": "google/gemini-3.5-flash", - "score": 0.6215011226795404, + "score": 0.6565011226795403, "quality": 0.8, "cost": 0.24746450304259626, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 } ], @@ -1291,10 +1358,10 @@ "candidateScores": [ { "model": "zai/glm-5.3-flash", - "score": 0.6505854692782567, + "score": 0.6855854692782567, "quality": 0.68, "cost": 0.5, - "speed": 0.05785469278256664, + "speed": 0.40785469278256664, "reliability": 1 } ], @@ -1308,10 +1375,10 @@ "tier": "MEDIUM", "confidence": 0.5, "method": "portfolio", - "reasoning": "score=-0.07 | short (5 tokens), constraints (不超过) | ambiguous -> default: MEDIUM | eco | v3 task=extraction agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=10", - "costEstimate": 0.0001169, - "baselineCost": 0.006424999999999999, - "savings": 0.9818054474708172, + "reasoning": "score=-0.07 | short (20 tokens), constraints (不超过) | ambiguous -> default: MEDIUM | eco | v3 task=extraction agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=10", + "costEstimate": 0.0001604, + "baselineCost": 0.0065, + "savings": 0.975323076923077, "agenticScore": 0, "candidates": [ "google/gemini-2.5-flash", @@ -1328,34 +1395,34 @@ "candidateScores": [ { "model": "google/gemini-2.5-flash", - "score": 0.8751229342146075, + "score": 0.9101229342146075, "quality": 0.88, "cost": 1, - "speed": 0.18322934214607486, + "speed": 0.5332293421460749, "reliability": 1 }, { "model": "anthropic/claude-sonnet-5", - "score": 0.7559771310352413, + "score": 0.798791498074642, "quality": 0.88, - "cost": 0.6760502381983543, - "speed": 0.09883064339702209, + "cost": 0.7039586919104992, + "speed": 0.44883064339702206, "reliability": 1 }, { "model": "moonshot/kimi-k3", - "score": 0.6166708011923991, + "score": 0.65347411666303, "quality": 1, - "cost": 0.002165439584235651, - "speed": 0.02731144775479715, + "cost": 0.008605851979346046, + "speed": 0.37731144775479714, "reliability": 1 }, { "model": "google/gemini-3.5-flash", - "score": 0.5826777284942801, + "score": 0.61767772849428, "quality": 0.88, "cost": 0, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 } ], @@ -1369,10 +1436,10 @@ "tier": "MEDIUM", "confidence": 0.5, "method": "portfolio", - "reasoning": "score=-0.07 | short (16 tokens), imperative (设计), references (代码) | ambiguous -> default: MEDIUM | eco | v3 task=chat agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=7", - "costEstimate": 0.0005584000000000001, - "baselineCost": 0.03208, - "savings": 0.9825935162094763, + "reasoning": "score=-0.07 | short (41 tokens), imperative (设计), references (代码) | ambiguous -> default: MEDIUM | eco | v3 task=chat agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=7", + "costEstimate": 0.0006309000000000001, + "baselineCost": 0.032205, + "savings": 0.9804098742431301, "agenticScore": 0, "candidates": [ "google/gemini-2.5-flash", @@ -1386,10 +1453,10 @@ "candidateScores": [ { "model": "google/gemini-2.5-flash", - "score": 0.7279229342146076, + "score": 0.7629229342146076, "quality": 0.86, "cost": 0.5, - "speed": 0.18322934214607486, + "speed": 0.5332293421460749, "reliability": 1 } ], @@ -1401,9 +1468,9 @@ "profile": "eco", "model": "zai/glm-5.3-flash", "tier": "MEDIUM", - "confidence": 0.5, + "confidence": 0.7034956909518112, "method": "portfolio", - "reasoning": "score=-0.02 | short (23 tokens), code (def), simple (define), agentic-light (debug) | ambiguous -> default: MEDIUM | eco | v3 task=debug agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=7", + "reasoning": "score=-0.07 | short (23 tokens), agentic-light (debug) | eco | v3 task=debug agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=7", "costEstimate": 0.020293600000000002, "baselineCost": 0.057714999999999995, "savings": 0.6483825695226544, @@ -1420,10 +1487,10 @@ "candidateScores": [ { "model": "zai/glm-5.3-flash", - "score": 0.6505854692782567, + "score": 0.6855854692782567, "quality": 0.68, "cost": 0.5, - "speed": 0.05785469278256664, + "speed": 0.40785469278256664, "reliability": 1 } ], @@ -1437,7 +1504,7 @@ "tier": "MEDIUM", "confidence": 0.7373034537835593, "method": "portfolio", - "reasoning": "score=0.09 | long (1217 tokens), references (following) | eco | v3 task=tool_agent agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=16", + "reasoning": "score=0.09 | long (1217 tokens), references (following) | eco | v3 task=tool_agent agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=27", "costEstimate": 0.0084417, "baselineCost": 0.089285, "savings": 0.905452203617629, @@ -1458,55 +1525,66 @@ "google/gemini-2.5-flash", "anthropic/claude-opus-5", "openai/gpt-4.1", - "openai/gpt-4o-mini" + "openai/gpt-4o-mini", + "google/gemini-3.8-flash", + "openai/gpt-6-luna", + "openai/gpt-6-sol", + "openai/gpt-5.1", + "xai/grok-4.6", + "anthropic/claude-opus-5.5", + "xai/grok-4.7", + "anthropic/claude-sonnet-5.5", + "openai/gpt-6-astra", + "xiaomi/mimo-v2.5", + "anthropic/claude-fable-5.1" ], "candidateScores": [ { "model": "anthropic/claude-sonnet-5", - "score": 0.9098830643397023, + "score": 0.9448830643397024, "quality": 1, "cost": 1, - "speed": 0.09883064339702209, + "speed": 0.44883064339702206, "reliability": 1 }, { "model": "openai/gpt-5-mini", - "score": 0.8125382143236466, + "score": 0.8475382143236465, "quality": 0.84, "cost": 0.8598625878017887, - "speed": 0.13376689739145697, + "speed": 0.48376689739145695, "reliability": 1 }, { "model": "openai/gpt-5.3-codex", - "score": 0.8110120317718679, + "score": 0.8460120317718679, "quality": 0.87, "cost": 0.9065750585345258, - "speed": 0.039714153822005965, + "speed": 0.3897141538220059, "reliability": 0.79999 }, { "model": "deepseek/deepseek-v4-pro", - "score": 0.6886949354620967, + "score": 0.7236949354620967, "quality": 0.82, "cost": 0.5233562353663685, - "speed": 0.06955189559513505, + "speed": 0.41955189559513506, "reliability": 1 }, { "model": "google/gemini-3.5-flash", - "score": 0.5880110618276134, + "score": 0.6230110618276133, "quality": 0.88, "cost": 0, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 }, { "model": "moonshot/kimi-k3", - "score": 0.5658106365806462, + "score": 0.6008106365806462, "quality": 0.85, "cost": 0.04671247073273721, - "speed": 0.02731144775479715, + "speed": 0.37731144775479714, "reliability": 1 } ], @@ -1520,7 +1598,7 @@ "tier": "MEDIUM", "confidence": 0.5, "method": "portfolio", - "reasoning": "score=0.06 | short (32 tokens), code (let), multi-step, references (following) | ambiguous -> default: MEDIUM | eco | v3 task=vision agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=3", + "reasoning": "score=-0.01 | short (32 tokens), multi-step, references (following) | ambiguous -> default: MEDIUM | eco | v3 task=vision agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=3", "costEstimate": 0.0022784000000000003, "baselineCost": 0.00656, "savings": 0.6526829268292683, @@ -1533,10 +1611,10 @@ "candidateScores": [ { "model": "zai/glm-5.3-flash", - "score": 0.6505854692782567, + "score": 0.6855854692782567, "quality": 0.68, "cost": 0.5, - "speed": 0.05785469278256664, + "speed": 0.40785469278256664, "reliability": 1 } ], @@ -1569,42 +1647,42 @@ "candidateScores": [ { "model": "xai/grok-4.5", - "score": 0.8706542065109696, + "score": 0.9056542065109696, "quality": 0.93, "cost": 1, - "speed": 0.05854206510969567, + "speed": 0.40854206510969565, "reliability": 1 }, { "model": "anthropic/claude-sonnet-5", - "score": 0.7667056868929675, + "score": 0.8017056868929675, "quality": 0.9, "cost": 0.6707950805473757, - "speed": 0.09883064339702209, + "speed": 0.44883064339702206, "reliability": 1 }, { "model": "deepseek/deepseek-v4-pro", - "score": 0.6850241311843186, + "score": 0.7200241311843186, "quality": 0.9, "cost": 0.3359605058028754, - "speed": 0.06955189559513505, + "speed": 0.41955189559513506, "reliability": 1 }, { "model": "google/gemini-3.5-flash", - "score": 0.6342110618276136, + "score": 0.6692110618276135, "quality": 1, "cost": 0, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 }, { "model": "moonshot/kimi-k3", - "score": 0.5670464054718248, + "score": 0.6020464054718248, "quality": 0.9, "cost": 0.001125931058375107, - "speed": 0.02731144775479715, + "speed": 0.37731144775479714, "reliability": 1 } ], @@ -1638,34 +1716,34 @@ "candidateScores": [ { "model": "google/gemini-2.5-flash", - "score": 0.8823229342146076, + "score": 0.9173229342146076, "quality": 0.9, "cost": 1, - "speed": 0.18322934214607486, + "speed": 0.5332293421460749, "reliability": 1 }, { "model": "anthropic/claude-sonnet-5", - "score": 0.7620108038512865, + "score": 0.7970108038512865, "quality": 0.9, "cost": 0.6718847839699437, - "speed": 0.09883064339702209, + "speed": 0.44883064339702206, "reliability": 1 }, { "model": "google/gemini-3.5-flash", - "score": 0.5898777284942802, + "score": 0.6248777284942801, "quality": 0.9, "cost": 0, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 }, { "model": "moonshot/kimi-k3", - "score": 0.5804016487653325, + "score": 0.6154016487653324, "quality": 0.9, "cost": 0.001204180916140718, - "speed": 0.02731144775479715, + "speed": 0.37731144775479714, "reliability": 1 } ], @@ -1675,31 +1753,31 @@ { "prompt": 15, "profile": "eco", - "model": "google/gemini-2.5-flash", - "tier": "MEDIUM", - "confidence": 0.5, + "model": "nvidia/nemotron-3.5-lightning", + "tier": "SIMPLE", + "confidence": 0.7009865197502964, "method": "portfolio", - "reasoning": "score=-0.06 | short (29 tokens), imperative (build), agentic-light (pip) | ambiguous -> default: MEDIUM | eco | v3 task=chat agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=7", + "reasoning": "score=-0.07 | short (29 tokens), imperative (build) | eco | v3 task=chat agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=7", "costEstimate": 0.0014153, "baselineCost": 0.083345, "savings": 0.983018777371168, - "agenticScore": 0.2, + "agenticScore": 0, "candidates": [ - "google/gemini-2.5-flash", + "nvidia/nemotron-3.5-lightning", + "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning", + "google/gemini-2.5-flash-lite", "zai/glm-5.3-flash", - "deepseek/deepseek-chat", - "google/gemini-3.1-flash-lite", "openai/gpt-5.6-luna", "openai/gpt-5.4-nano", - "google/gemini-2.5-flash-lite" + "google/gemini-3.1-flash-lite" ], "candidateScores": [ { - "model": "google/gemini-2.5-flash", - "score": 0.7279229342146076, - "quality": 0.86, + "model": "nvidia/nemotron-3.5-lightning", + "score": 0.6948000000000001, + "quality": 0.68, "cost": 0.5, - "speed": 0.18322934214607486, + "speed": 0.5, "reliability": 1 } ], @@ -1713,7 +1791,7 @@ "tier": "MEDIUM", "confidence": 0.5, "method": "portfolio", - "reasoning": "score=-0.06 | short (38 tokens), imperative (deploy), references (the api), agentic-light (check the, deploy) | ambiguous -> default: MEDIUM | eco | v3 task=tool_agent agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=16", + "reasoning": "score=-0.06 | short (38 tokens), imperative (deploy), references (the api), agentic-light (check the, deploy) | ambiguous -> default: MEDIUM | eco | v3 task=tool_agent agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=27", "costEstimate": 0.0006437999999999999, "baselineCost": 0.0065899999999999995, "savings": 0.9023065250379363, @@ -1734,55 +1812,66 @@ "google/gemini-2.5-flash", "anthropic/claude-opus-5", "openai/gpt-4.1", - "openai/gpt-4o-mini" + "openai/gpt-4o-mini", + "google/gemini-3.8-flash", + "openai/gpt-6-luna", + "openai/gpt-6-sol", + "openai/gpt-5.1", + "xai/grok-4.6", + "anthropic/claude-opus-5.5", + "xai/grok-4.7", + "anthropic/claude-sonnet-5.5", + "openai/gpt-6-astra", + "xiaomi/mimo-v2.5", + "anthropic/claude-fable-5.1" ], "candidateScores": [ { "model": "anthropic/claude-sonnet-5", - "score": 0.9098830643397023, + "score": 0.9448830643397024, "quality": 1, "cost": 1, - "speed": 0.09883064339702209, + "speed": 0.44883064339702206, "reliability": 1 }, { "model": "openai/gpt-5-mini", - "score": 0.8332830628561794, + "score": 0.8682830628561793, "quality": 0.84, "cost": 0.9339513325608342, - "speed": 0.13376689739145697, + "speed": 0.48376689739145695, "reliability": 1 }, { "model": "openai/gpt-5.3-codex", - "score": 0.8248419307935564, + "score": 0.8598419307935564, "quality": 0.87, "cost": 0.9559675550405561, - "speed": 0.039714153822005965, + "speed": 0.3897141538220059, "reliability": 0.79999 }, { "model": "deepseek/deepseek-v4-pro", - "score": 0.6852374607066745, + "score": 0.7202374607066745, "quality": 0.82, "cost": 0.5110081112398608, - "speed": 0.06955189559513505, + "speed": 0.41955189559513506, "reliability": 1 }, { "model": "google/gemini-3.5-flash", - "score": 0.5880110618276134, + "score": 0.6230110618276133, "quality": 0.88, "cost": 0, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 }, { "model": "moonshot/kimi-k3", - "score": 0.5588956870698019, + "score": 0.5938956870698018, "quality": 0.85, "cost": 0.022016222479721792, - "speed": 0.02731144775479715, + "speed": 0.37731144775479714, "reliability": 1 } ], @@ -1809,10 +1898,10 @@ "candidateScores": [ { "model": "zai/glm-5.3-flash", - "score": 0.6505854692782567, + "score": 0.6855854692782567, "quality": 0.68, "cost": 0.5, - "speed": 0.05785469278256664, + "speed": 0.40785469278256664, "reliability": 1 } ], @@ -1846,34 +1935,34 @@ "candidateScores": [ { "model": "google/gemini-2.5-flash", - "score": 0.8891443471782455, + "score": 0.9521443471782456, "quality": 0.9, "cost": 1, - "speed": 0.13969081765691935, + "speed": 0.48969081765691935, "reliability": 1 }, { "model": "anthropic/claude-sonnet-5", - "score": 0.7770287467109466, + "score": 0.8400287467109466, "quality": 0.9, "cost": 0.6729268997399596, - "speed": 0.1367178599097662, + "speed": 0.4867178599097662, "reliability": 1 }, { "model": "google/gemini-3.5-flash", - "score": 0.6005020197183445, + "score": 0.6635020197183445, "quality": 0.9, "cost": 0, - "speed": 0.16575196139820988, + "speed": 0.5157519613982099, "reliability": 1 }, { "model": "moonshot/kimi-k3", - "score": 0.5817511887996366, + "score": 0.6447511887996366, "quality": 0.9, "cost": 0.0014446691707599157, - "speed": 0.022296378324947415, + "speed": 0.3722963783249474, "reliability": 1 } ], @@ -1906,42 +1995,42 @@ "candidateScores": [ { "model": "xai/grok-4.5", - "score": 0.8706542065109696, + "score": 0.9056542065109696, "quality": 0.93, "cost": 1, - "speed": 0.05854206510969567, + "speed": 0.40854206510969565, "reliability": 1 }, { "model": "anthropic/claude-sonnet-5", - "score": 0.7666783871460185, + "score": 0.8016783871460186, "quality": 0.9, "cost": 0.6706975814511295, - "speed": 0.09883064339702209, + "speed": 0.44883064339702206, "reliability": 1 }, { "model": "deepseek/deepseek-v4-pro", - "score": 0.6850067586180785, + "score": 0.7200067586180785, "quality": 0.9, "cost": 0.335898460923446, - "speed": 0.06955189559513505, + "speed": 0.41955189559513506, "reliability": 1 }, { "model": "google/gemini-3.5-flash", - "score": 0.6342110618276136, + "score": 0.6692110618276135, "quality": 1, "cost": 0, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 }, { "model": "moonshot/kimi-k3", - "score": 0.5670389600862934, + "score": 0.6020389600862933, "quality": 0.9, "cost": 0.001099340395762649, - "speed": 0.02731144775479715, + "speed": 0.37731144775479714, "reliability": 1 } ], @@ -1972,10 +2061,10 @@ "candidateScores": [ { "model": "zai/glm-5.3-flash", - "score": 0.6505854692782567, + "score": 0.6855854692782567, "quality": 0.68, "cost": 0.5, - "speed": 0.05785469278256664, + "speed": 0.40785469278256664, "reliability": 1 } ], @@ -1989,7 +2078,7 @@ "tier": "MEDIUM", "confidence": 0.7231218051243898, "method": "portfolio", - "reasoning": "score=0.08 | long (14423 tokens) | eco | v3 task=tool_agent agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=16", + "reasoning": "score=0.08 | long (14423 tokens) | eco | v3 task=tool_agent agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=27", "costEstimate": 0.0046423, "baselineCost": 0.104115, "savings": 0.9554118042549105, @@ -2010,55 +2099,66 @@ "google/gemini-2.5-flash", "anthropic/claude-opus-5", "openai/gpt-4.1", - "openai/gpt-4o-mini" + "openai/gpt-4o-mini", + "anthropic/claude-opus-5.5", + "openai/gpt-5.1", + "google/gemini-3.8-flash", + "openai/gpt-6-luna", + "xai/grok-4.6", + "anthropic/claude-sonnet-5.5", + "openai/gpt-6-sol", + "openai/gpt-6-astra", + "xiaomi/mimo-v2.5", + "xai/grok-4.7", + "anthropic/claude-fable-5.1" ], "candidateScores": [ { "model": "anthropic/claude-sonnet-5", - "score": 0.9098830643397023, + "score": 0.9448830643397024, "quality": 1, "cost": 1, - "speed": 0.09883064339702209, + "speed": 0.44883064339702206, "reliability": 1 }, { "model": "openai/gpt-5.3-codex", - "score": 0.735950638165583, + "score": 0.7709506381655831, "quality": 0.87, "cost": 0.6384986527977944, - "speed": 0.039714153822005965, + "speed": 0.3897141538220059, "reliability": 0.79999 }, { "model": "deepseek/deepseek-v4-pro", - "score": 0.7074602838636679, + "score": 0.7424602838636679, "quality": 0.82, "cost": 0.5903753368005515, - "speed": 0.06955189559513505, + "speed": 0.41955189559513506, "reliability": 1 }, { "model": "openai/gpt-5-mini", - "score": 0.6999461239142194, + "score": 0.7349461239142193, "quality": 0.84, "cost": 0.45774797919669175, - "speed": 0.13376689739145697, + "speed": 0.48376689739145695, "reliability": 1 }, { "model": "moonshot/kimi-k3", - "score": 0.6033413333837886, + "score": 0.6383413333837885, "quality": 0.85, "cost": 0.18075067360110297, - "speed": 0.02731144775479715, + "speed": 0.37731144775479714, "reliability": 1 }, { "model": "google/gemini-3.5-flash", - "score": 0.5880110618276134, + "score": 0.6230110618276133, "quality": 0.88, "cost": 0, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 } ], @@ -2091,10 +2191,10 @@ "candidateScores": [ { "model": "google/gemini-2.5-flash", - "score": 0.7870260539502253, + "score": 0.8115260539502253, "quality": 0.86, "cost": 0.5, - "speed": 0.18322934214607486, + "speed": 0.5332293421460749, "reliability": 1 } ], @@ -2108,7 +2208,7 @@ "tier": "SIMPLE", "confidence": 0.7231218051243898, "method": "portfolio", - "reasoning": "score=-0.08 | short (1 tokens) | agentic (tools) | v3 task=tool_agent agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=13", + "reasoning": "score=-0.08 | short (1 tokens) | agentic (tools) | v3 task=tool_agent agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=24", "costEstimate": 0.0032001, "baselineCost": 0.032005, "savings": 0.9000124980471801, @@ -2126,15 +2226,26 @@ "google/gemini-3.5-flash", "openai/gpt-5.3-codex", "moonshot/kimi-k3", - "deepseek/deepseek-v4-pro" + "deepseek/deepseek-v4-pro", + "google/gemini-3.8-flash", + "openai/gpt-6-luna", + "openai/gpt-6-sol", + "openai/gpt-5.1", + "xai/grok-4.6", + "anthropic/claude-opus-5.5", + "xai/grok-4.7", + "anthropic/claude-sonnet-5.5", + "openai/gpt-6-astra", + "xiaomi/mimo-v2.5", + "anthropic/claude-fable-5.1" ], "candidateScores": [ { "model": "anthropic/claude-sonnet-5", - "score": 0.8469181450377915, + "score": 0.8714181450377915, "quality": 1, "cost": 0.5, - "speed": 0.09883064339702209, + "speed": 0.44883064339702206, "reliability": 1 } ], @@ -2161,10 +2272,10 @@ "candidateScores": [ { "model": "google/gemini-2.5-flash", - "score": 0.7024260539502254, + "score": 0.7269260539502254, "quality": 0.68, "cost": 0.5, - "speed": 0.18322934214607486, + "speed": 0.5332293421460749, "reliability": 1 } ], @@ -2199,10 +2310,10 @@ "candidateScores": [ { "model": "google/gemini-3.5-flash", - "score": 0.7030477432793295, + "score": 0.7275477432793295, "quality": 0.68, "cost": 0.5, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 } ], @@ -2233,18 +2344,18 @@ "candidateScores": [ { "model": "deepseek/deepseek-v4-pro", - "score": 0.9113686326916596, + "score": 0.9358686326916595, "quality": 0.95, "cost": 1, - "speed": 0.06955189559513505, + "speed": 0.41955189559513506, "reliability": 1 }, { "model": "google/gemini-3.5-flash", - "score": 0.6758477432793295, + "score": 0.7003477432793295, "quality": 0.92, "cost": 0, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 } ], @@ -2279,10 +2390,10 @@ "candidateScores": [ { "model": "google/gemini-3.5-flash", - "score": 0.7030477432793295, + "score": 0.7275477432793295, "quality": 0.68, "cost": 0.5, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 } ], @@ -2296,7 +2407,7 @@ "tier": "SIMPLE", "confidence": 0.7231218051243898, "method": "portfolio", - "reasoning": "score=-0.08 | short (20 tokens) | agentic (tools) | v3 task=tool_agent_parallel agentRisk=high deepWebResearch=false terminalCode=false terminalSafety=false candidates=13", + "reasoning": "score=-0.08 | short (20 tokens) | agentic (tools) | v3 task=tool_agent_parallel agentRisk=high deepWebResearch=false terminalCode=false terminalSafety=false candidates=24", "costEstimate": 0.020291200000000002, "baselineCost": 0.0577, "savings": 0.6483327556325823, @@ -2314,15 +2425,26 @@ "openai/gpt-4.1", "xai/grok-4.5", "google/gemini-3.5-flash", - "deepseek/deepseek-v4-pro" + "deepseek/deepseek-v4-pro", + "google/gemini-3.8-flash", + "openai/gpt-6-luna", + "openai/gpt-6-sol", + "openai/gpt-5.1", + "xai/grok-4.6", + "anthropic/claude-opus-5.5", + "xai/grok-4.7", + "anthropic/claude-sonnet-5.5", + "openai/gpt-6-astra", + "xiaomi/mimo-v2.5", + "anthropic/claude-fable-5.1" ], "candidateScores": [ { "model": "anthropic/claude-opus-4.8", - "score": 0.8468563440295361, + "score": 0.8713563440295362, "quality": 1, "cost": 0.5, - "speed": 0.09794777185051722, + "speed": 0.4479477718505172, "reliability": 1 } ], @@ -2349,10 +2471,10 @@ "candidateScores": [ { "model": "google/gemini-2.5-flash", - "score": 0.7024260539502254, + "score": 0.7269260539502254, "quality": 0.68, "cost": 0.5, - "speed": 0.18322934214607486, + "speed": 0.5332293421460749, "reliability": 1 } ], @@ -2366,10 +2488,10 @@ "tier": "MEDIUM", "confidence": 0.5, "method": "portfolio", - "reasoning": "score=-0.07 | short (5 tokens), constraints (不超过) | ambiguous -> default: MEDIUM | v3 task=extraction agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=13", - "costEstimate": 0.0017297000000000002, - "baselineCost": 0.006424999999999999, - "savings": 0.7307859922178989, + "reasoning": "score=-0.07 | short (20 tokens), constraints (不超过) | ambiguous -> default: MEDIUM | v3 task=extraction agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=13", + "costEstimate": 0.0017732000000000004, + "baselineCost": 0.0065, + "savings": 0.7272, "agenticScore": 0, "candidates": [ "moonshot/kimi-k3", @@ -2389,10 +2511,10 @@ "candidateScores": [ { "model": "moonshot/kimi-k3", - "score": 0.8419118013428357, + "score": 0.8664118013428358, "quality": 1, "cost": 0.5, - "speed": 0.02731144775479715, + "speed": 0.37731144775479714, "reliability": 1 } ], @@ -2406,10 +2528,10 @@ "tier": "MEDIUM", "confidence": 0.5, "method": "portfolio", - "reasoning": "score=-0.07 | short (16 tokens), imperative (设计), references (代码) | ambiguous -> default: MEDIUM | v3 task=chat agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=11", - "costEstimate": 0.0005584000000000001, - "baselineCost": 0.03208, - "savings": 0.9825935162094763, + "reasoning": "score=-0.07 | short (41 tokens), imperative (设计), references (代码) | ambiguous -> default: MEDIUM | v3 task=chat agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=11", + "costEstimate": 0.0006309000000000001, + "baselineCost": 0.032205, + "savings": 0.9804098742431301, "agenticScore": 0, "candidates": [ "google/gemini-2.5-flash", @@ -2427,18 +2549,18 @@ "candidateScores": [ { "model": "google/gemini-2.5-flash", - "score": 0.8270260539502253, + "score": 0.8515260539502253, "quality": 0.86, "cost": 1, - "speed": 0.18322934214607486, + "speed": 0.5332293421460749, "reliability": 1 }, { "model": "google/gemini-3.5-flash", - "score": 0.6976477432793294, + "score": 0.7221477432793294, "quality": 0.86, "cost": 0, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 } ], @@ -2450,9 +2572,9 @@ "profile": "auto", "model": "google/gemini-3.5-flash", "tier": "MEDIUM", - "confidence": 0.5, + "confidence": 0.7034956909518112, "method": "portfolio", - "reasoning": "score=-0.02 | short (23 tokens), code (def), simple (define), agentic-light (debug) | ambiguous -> default: MEDIUM | v3 task=debug agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=11", + "reasoning": "score=-0.07 | short (23 tokens), agentic-light (debug) | v3 task=debug agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=11", "costEstimate": 0.015519600000000001, "baselineCost": 0.057714999999999995, "savings": 0.7310993675820844, @@ -2473,10 +2595,10 @@ "candidateScores": [ { "model": "google/gemini-3.5-flash", - "score": 0.7030477432793295, + "score": 0.7275477432793295, "quality": 0.68, "cost": 0.5, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 } ], @@ -2490,7 +2612,7 @@ "tier": "MEDIUM", "confidence": 0.7373034537835593, "method": "portfolio", - "reasoning": "score=0.09 | long (1217 tokens), references (following) | agentic (tools) | v3 task=tool_agent agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=13", + "reasoning": "score=0.09 | long (1217 tokens), references (following) | agentic (tools) | v3 task=tool_agent agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=24", "costEstimate": 0.0084417, "baselineCost": 0.089285, "savings": 0.905452203617629, @@ -2508,15 +2630,26 @@ "anthropic/claude-opus-5", "openai/gpt-4.1", "openai/gpt-5.3-codex", - "deepseek/deepseek-v4-pro" + "deepseek/deepseek-v4-pro", + "google/gemini-3.8-flash", + "openai/gpt-6-luna", + "openai/gpt-6-sol", + "openai/gpt-5.1", + "xai/grok-4.6", + "anthropic/claude-opus-5.5", + "xai/grok-4.7", + "anthropic/claude-sonnet-5.5", + "openai/gpt-6-astra", + "xiaomi/mimo-v2.5", + "anthropic/claude-fable-5.1" ], "candidateScores": [ { "model": "anthropic/claude-sonnet-5", - "score": 0.8469181450377915, + "score": 0.8714181450377915, "quality": 1, "cost": 0.5, - "speed": 0.09883064339702209, + "speed": 0.44883064339702206, "reliability": 1 } ], @@ -2530,7 +2663,7 @@ "tier": "MEDIUM", "confidence": 0.5, "method": "portfolio", - "reasoning": "score=0.06 | short (32 tokens), code (let), multi-step, references (following) | ambiguous -> default: MEDIUM | v3 task=vision agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=7", + "reasoning": "score=-0.01 | short (32 tokens), multi-step, references (following) | ambiguous -> default: MEDIUM | v3 task=vision agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=7", "costEstimate": 0.0018304000000000003, "baselineCost": 0.00656, "savings": 0.7209756097560975, @@ -2547,10 +2680,10 @@ "candidateScores": [ { "model": "google/gemini-3.5-flash", - "score": 0.7030477432793295, + "score": 0.7275477432793295, "quality": 0.68, "cost": 0.5, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 } ], @@ -2584,42 +2717,42 @@ "candidateScores": [ { "model": "xai/grok-4.5", - "score": 0.8761979445576786, + "score": 0.9006979445576787, "quality": 0.93, "cost": 1, - "speed": 0.05854206510969567, + "speed": 0.40854206510969565, "reliability": 1 }, { "model": "anthropic/claude-sonnet-5", - "score": 0.7931612595363191, + "score": 0.8176612595363191, "quality": 0.9, "cost": 0.6707950805473757, - "speed": 0.09883064339702209, + "speed": 0.44883064339702206, "reliability": 1 }, { "model": "deepseek/deepseek-v4-pro", - "score": 0.7683415237361771, + "score": 0.7928415237361771, "quality": 0.9, "cost": 0.3359605058028754, - "speed": 0.06955189559513505, + "speed": 0.41955189559513506, "reliability": 1 }, { "model": "google/gemini-3.5-flash", - "score": 0.7509477432793293, + "score": 0.7754477432793293, "quality": 1, "cost": 0, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 }, { "model": "moonshot/kimi-k3", - "score": 0.6551144689333432, + "score": 0.6796144689333433, "quality": 0.9, "cost": 0.001125931058375107, - "speed": 0.02731144775479715, + "speed": 0.37731144775479714, "reliability": 1 } ], @@ -2656,34 +2789,34 @@ "candidateScores": [ { "model": "google/gemini-2.5-flash", - "score": 0.8791593872835586, + "score": 0.9036593872835585, "quality": 0.9, "cost": 1, - "speed": 0.18322934214607486, + "speed": 0.5332293421460749, "reliability": 1 }, { "model": "anthropic/claude-sonnet-5", - "score": 0.7808574061523814, + "score": 0.8053574061523814, "quality": 0.9, "cost": 0.6718847839699437, - "speed": 0.09883064339702209, + "speed": 0.44883064339702206, "reliability": 1 }, { "model": "google/gemini-3.5-flash", - "score": 0.7164477432793295, + "score": 0.7409477432793294, "quality": 0.9, "cost": 0, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 }, { "model": "moonshot/kimi-k3", - "score": 0.6717952205744078, + "score": 0.6962952205744078, "quality": 0.9, "cost": 0.001204180916140718, - "speed": 0.02731144775479715, + "speed": 0.37731144775479714, "reliability": 1 } ], @@ -2694,42 +2827,32 @@ "prompt": 15, "profile": "auto", "model": "google/gemini-2.5-flash", - "tier": "MEDIUM", - "confidence": 0.5, + "tier": "SIMPLE", + "confidence": 0.7009865197502964, "method": "portfolio", - "reasoning": "score=-0.06 | short (29 tokens), imperative (build), agentic-light (pip) | ambiguous -> default: MEDIUM | v3 task=chat agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=11", + "reasoning": "score=-0.07 | short (29 tokens), imperative (build) | v3 task=chat agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=9", "costEstimate": 0.0014153, "baselineCost": 0.083345, "savings": 0.983018777371168, - "agenticScore": 0.2, + "agenticScore": 0, "candidates": [ "google/gemini-2.5-flash", - "google/gemini-3.5-flash", - "google/gemini-3.6-flash", - "zai/glm-5.3-flash", - "openai/gpt-5.6-terra", "google/gemini-3-flash-preview", + "google/gemini-3.5-flash-lite", "deepseek/deepseek-chat", - "minimax/minimax-m3", "google/gemini-3.1-flash-lite", "openai/gpt-5.6-luna", - "google/gemini-2.5-flash-lite" + "openai/gpt-5.4-nano", + "google/gemini-2.5-flash-lite", + "nvidia/nemotron-3.5-lightning" ], "candidateScores": [ { "model": "google/gemini-2.5-flash", - "score": 0.8270260539502253, - "quality": 0.86, - "cost": 1, - "speed": 0.18322934214607486, - "reliability": 1 - }, - { - "model": "google/gemini-3.5-flash", - "score": 0.6976477432793294, + "score": 0.8115260539502253, "quality": 0.86, - "cost": 0, - "speed": 0.19211061827613424, + "cost": 0.5, + "speed": 0.5332293421460749, "reliability": 1 } ], @@ -2743,7 +2866,7 @@ "tier": "MEDIUM", "confidence": 0.5, "method": "portfolio", - "reasoning": "score=-0.06 | short (38 tokens), imperative (deploy), references (the api), agentic-light (check the, deploy) | ambiguous -> default: MEDIUM | agentic (tools) | v3 task=tool_agent agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=13", + "reasoning": "score=-0.06 | short (38 tokens), imperative (deploy), references (the api), agentic-light (check the, deploy) | ambiguous -> default: MEDIUM | agentic (tools) | v3 task=tool_agent agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=24", "costEstimate": 0.0006437999999999999, "baselineCost": 0.0065899999999999995, "savings": 0.9023065250379363, @@ -2761,15 +2884,26 @@ "anthropic/claude-opus-5", "openai/gpt-4.1", "openai/gpt-5.3-codex", - "deepseek/deepseek-v4-pro" + "deepseek/deepseek-v4-pro", + "google/gemini-3.8-flash", + "openai/gpt-6-luna", + "openai/gpt-6-sol", + "openai/gpt-5.1", + "xai/grok-4.6", + "anthropic/claude-opus-5.5", + "xai/grok-4.7", + "anthropic/claude-sonnet-5.5", + "openai/gpt-6-astra", + "xiaomi/mimo-v2.5", + "anthropic/claude-fable-5.1" ], "candidateScores": [ { "model": "anthropic/claude-sonnet-5", - "score": 0.8469181450377915, + "score": 0.8714181450377915, "quality": 1, "cost": 0.5, - "speed": 0.09883064339702209, + "speed": 0.44883064339702206, "reliability": 1 } ], @@ -2800,10 +2934,10 @@ "candidateScores": [ { "model": "google/gemini-3.5-flash", - "score": 0.7030477432793295, + "score": 0.7275477432793295, "quality": 0.68, "cost": 0.5, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 } ], @@ -2840,34 +2974,34 @@ "candidateScores": [ { "model": "google/gemini-2.5-flash", - "score": 0.8872869559818712, + "score": 0.9397869559818712, "quality": 0.9, "cost": 1, - "speed": 0.13969081765691935, + "speed": 0.48969081765691935, "reliability": 1 }, { "model": "anthropic/claude-sonnet-5", - "score": 0.7946345209396576, + "score": 0.8471345209396577, "quality": 0.9, "cost": 0.6729268997399596, - "speed": 0.1367178599097662, + "speed": 0.4867178599097662, "reliability": 1 }, { "model": "google/gemini-3.5-flash", - "score": 0.7278627942097315, + "score": 0.7803627942097315, "quality": 0.9, "cost": 0, - "speed": 0.16575196139820988, + "speed": 0.5157519613982099, "reliability": 1 }, { "model": "moonshot/kimi-k3", - "score": 0.6732711638661456, + "score": 0.7257711638661457, "quality": 0.9, "cost": 0.0014446691707599157, - "speed": 0.022296378324947415, + "speed": 0.3722963783249474, "reliability": 1 } ], @@ -2901,42 +3035,42 @@ "candidateScores": [ { "model": "xai/grok-4.5", - "score": 0.8761979445576786, + "score": 0.9006979445576787, "quality": 0.93, "cost": 1, - "speed": 0.05854206510969567, + "speed": 0.40854206510969565, "reliability": 1 }, { "model": "anthropic/claude-sonnet-5", - "score": 0.7931437096989948, + "score": 0.8176437096989948, "quality": 0.9, "cost": 0.6706975814511295, - "speed": 0.09883064339702209, + "speed": 0.44883064339702206, "reliability": 1 }, { "model": "deepseek/deepseek-v4-pro", - "score": 0.7683303556578799, + "score": 0.7928303556578798, "quality": 0.9, "cost": 0.335898460923446, - "speed": 0.06955189559513505, + "speed": 0.41955189559513506, "reliability": 1 }, { "model": "google/gemini-3.5-flash", - "score": 0.7509477432793293, + "score": 0.7754477432793293, "quality": 1, "cost": 0, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 }, { "model": "moonshot/kimi-k3", - "score": 0.6551096826140731, + "score": 0.6796096826140732, "quality": 0.9, "cost": 0.001099340395762649, - "speed": 0.02731144775479715, + "speed": 0.37731144775479714, "reliability": 1 } ], @@ -2971,10 +3105,10 @@ "candidateScores": [ { "model": "google/gemini-3.5-flash", - "score": 0.7030477432793295, + "score": 0.7275477432793295, "quality": 0.68, "cost": 0.5, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 } ], @@ -2988,7 +3122,7 @@ "tier": "MEDIUM", "confidence": 0.7231218051243898, "method": "portfolio", - "reasoning": "score=0.08 | long (14423 tokens) | agentic (tools) | v3 task=tool_agent agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=13", + "reasoning": "score=0.08 | long (14423 tokens) | agentic (tools) | v3 task=tool_agent agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=24", "costEstimate": 0.0046423, "baselineCost": 0.104115, "savings": 0.9554118042549105, @@ -3006,15 +3140,26 @@ "anthropic/claude-opus-5", "openai/gpt-4.1", "openai/gpt-5.3-codex", - "deepseek/deepseek-v4-pro" + "deepseek/deepseek-v4-pro", + "anthropic/claude-opus-5.5", + "openai/gpt-5.1", + "google/gemini-3.8-flash", + "openai/gpt-6-luna", + "xai/grok-4.6", + "anthropic/claude-sonnet-5.5", + "openai/gpt-6-sol", + "openai/gpt-6-astra", + "xiaomi/mimo-v2.5", + "xai/grok-4.7", + "anthropic/claude-fable-5.1" ], "candidateScores": [ { "model": "anthropic/claude-sonnet-5", - "score": 0.8469181450377915, + "score": 0.8714181450377915, "quality": 1, "cost": 0.5, - "speed": 0.09883064339702209, + "speed": 0.44883064339702206, "reliability": 1 } ], @@ -3045,18 +3190,18 @@ "candidateScores": [ { "model": "google/gemini-2.5-flash", - "score": 0.8497937605287644, + "score": 0.8707937605287643, "quality": 0.86, "cost": 1, - "speed": 0.18322934214607486, + "speed": 0.5332293421460749, "reliability": 1 }, { "model": "google/gemini-3.5-flash", - "score": 0.790326637096568, + "score": 0.8113266370965679, "quality": 0.86, "cost": 0, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 } ], @@ -3070,7 +3215,7 @@ "tier": "SIMPLE", "confidence": 0.7231218051243898, "method": "portfolio", - "reasoning": "score=-0.08 | short (1 tokens) | premium | v3 task=tool_agent agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=15", + "reasoning": "score=-0.08 | short (1 tokens) | premium | v3 task=tool_agent agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=26", "costEstimate": 0.0032001, "baselineCost": 0.032005, "savings": 0, @@ -3090,15 +3235,26 @@ "openai/gpt-4o-mini", "openai/gpt-5.3-codex", "moonshot/kimi-k3", - "deepseek/deepseek-v4-pro" + "deepseek/deepseek-v4-pro", + "anthropic/claude-fable-5.1", + "anthropic/claude-opus-5.5", + "anthropic/claude-sonnet-5.5", + "google/gemini-3.8-flash", + "openai/gpt-5.1", + "openai/gpt-6-astra", + "openai/gpt-6-luna", + "openai/gpt-6-sol", + "xai/grok-4.6", + "xai/grok-4.7", + "xiaomi/mimo-v2.5" ], "candidateScores": [ { "model": "anthropic/claude-sonnet-5", - "score": 0.9059298386038215, + "score": 0.9269298386038214, "quality": 1, "cost": 0.5, - "speed": 0.09883064339702209, + "speed": 0.44883064339702206, "reliability": 1 } ], @@ -3126,10 +3282,10 @@ "candidateScores": [ { "model": "google/gemini-3.5-flash", - "score": 0.7259266370965682, + "score": 0.7469266370965681, "quality": 0.68, "cost": 0.5, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 } ], @@ -3163,10 +3319,10 @@ "candidateScores": [ { "model": "openai/gpt-5.3-codex", - "score": 0.8903822492293204, + "score": 0.9113822492293204, "quality": 1, "cost": 0.5, - "speed": 0.039714153822005965, + "speed": 0.3897141538220059, "reliability": 0.79999 } ], @@ -3200,34 +3356,34 @@ "candidateScores": [ { "model": "anthropic/claude-sonnet-5", - "score": 0.9093298386038213, + "score": 0.9303298386038215, "quality": 0.98, "cost": 0.6875, - "speed": 0.09883064339702209, + "speed": 0.44883064339702206, "reliability": 1 }, { "model": "xai/grok-4.5", - "score": 0.8953791905732483, + "score": 0.9163791905732482, "quality": 0.94, "cost": 1, - "speed": 0.05854206510969567, + "speed": 0.40854206510969565, "reliability": 1 }, { "model": "anthropic/claude-sonnet-4.6", - "score": 0.847328760008195, + "score": 0.8683287600081949, "quality": 0.98, "cost": 0, - "speed": 0.09325711124769513, + "speed": 0.4432571112476951, "reliability": 1 }, { "model": "deepseek/deepseek-v4-pro", - "score": 0.8423953359579301, + "score": 0.8633953359579303, "quality": 0.95, "cost": 0.3402777777777777, - "speed": 0.06955189559513505, + "speed": 0.41955189559513506, "reliability": 1 } ], @@ -3261,10 +3417,10 @@ "candidateScores": [ { "model": "openai/gpt-5.3-codex", - "score": 0.8903822492293204, + "score": 0.9113822492293204, "quality": 1, "cost": 0.5, - "speed": 0.039714153822005965, + "speed": 0.3897141538220059, "reliability": 0.79999 } ], @@ -3278,7 +3434,7 @@ "tier": "SIMPLE", "confidence": 0.7231218051243898, "method": "portfolio", - "reasoning": "score=-0.08 | short (20 tokens) | premium | v3 task=tool_agent_parallel agentRisk=high deepWebResearch=false terminalCode=false terminalSafety=false candidates=15", + "reasoning": "score=-0.08 | short (20 tokens) | premium | v3 task=tool_agent_parallel agentRisk=high deepWebResearch=false terminalCode=false terminalSafety=false candidates=26", "costEstimate": 0.020291200000000002, "baselineCost": 0.0577, "savings": 0, @@ -3298,15 +3454,26 @@ "openai/gpt-4.1", "openai/gpt-4o-mini", "xai/grok-4.5", - "deepseek/deepseek-v4-pro" + "deepseek/deepseek-v4-pro", + "anthropic/claude-fable-5.1", + "anthropic/claude-opus-5.5", + "anthropic/claude-sonnet-5.5", + "google/gemini-3.8-flash", + "openai/gpt-5.1", + "openai/gpt-6-astra", + "openai/gpt-6-luna", + "openai/gpt-6-sol", + "xai/grok-4.6", + "xai/grok-4.7", + "xiaomi/mimo-v2.5" ], "candidateScores": [ { "model": "anthropic/claude-opus-4.8", - "score": 0.905876866311031, + "score": 0.9268768663110312, "quality": 1, "cost": 0.5, - "speed": 0.09794777185051722, + "speed": 0.4479477718505172, "reliability": 1 } ], @@ -3334,10 +3501,10 @@ "candidateScores": [ { "model": "google/gemini-3.5-flash", - "score": 0.7259266370965682, + "score": 0.7469266370965681, "quality": 0.68, "cost": 0.5, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 } ], @@ -3351,9 +3518,9 @@ "tier": "MEDIUM", "confidence": 0.5, "method": "portfolio", - "reasoning": "score=-0.07 | short (5 tokens), constraints (不超过) | ambiguous -> default: MEDIUM | premium | v3 task=extraction agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=10", - "costEstimate": 0.0017297000000000002, - "baselineCost": 0.006424999999999999, + "reasoning": "score=-0.07 | short (20 tokens), constraints (不超过) | ambiguous -> default: MEDIUM | premium | v3 task=extraction agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=10", + "costEstimate": 0.0017732000000000004, + "baselineCost": 0.0065, "savings": 0, "agenticScore": 0, "candidates": [ @@ -3371,10 +3538,10 @@ "candidateScores": [ { "model": "moonshot/kimi-k3", - "score": 0.9016386868652879, + "score": 0.9226386868652878, "quality": 1, "cost": 0.5, - "speed": 0.02731144775479715, + "speed": 0.37731144775479714, "reliability": 1 } ], @@ -3388,9 +3555,9 @@ "tier": "MEDIUM", "confidence": 0.5, "method": "portfolio", - "reasoning": "score=-0.07 | short (16 tokens), imperative (设计), references (代码) | ambiguous -> default: MEDIUM | premium | v3 task=chat agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=10", - "costEstimate": 0.008622400000000002, - "baselineCost": 0.03208, + "reasoning": "score=-0.07 | short (41 tokens), imperative (设计), references (代码) | ambiguous -> default: MEDIUM | premium | v3 task=chat agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=10", + "costEstimate": 0.008694900000000002, + "baselineCost": 0.032205, "savings": 0, "agenticScore": 0, "candidates": [ @@ -3408,18 +3575,18 @@ "candidateScores": [ { "model": "moonshot/kimi-k3", - "score": 0.8604386868652878, + "score": 0.8814386868652877, "quality": 0.86, "cost": 1, - "speed": 0.02731144775479715, + "speed": 0.37731144775479714, "reliability": 1 }, { "model": "google/gemini-3.5-flash", - "score": 0.770326637096568, + "score": 0.7913266370965679, "quality": 0.86, "cost": 0, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 } ], @@ -3431,9 +3598,9 @@ "profile": "premium", "model": "openai/gpt-5.3-codex", "tier": "MEDIUM", - "confidence": 0.5, + "confidence": 0.7034956909518112, "method": "portfolio", - "reasoning": "score=-0.02 | short (23 tokens), code (def), simple (define), agentic-light (debug) | ambiguous -> default: MEDIUM | premium | v3 task=debug agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=10", + "reasoning": "score=-0.07 | short (23 tokens), agentic-light (debug) | premium | v3 task=debug agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=10", "costEstimate": 0.0057945, "baselineCost": 0.057714999999999995, "savings": 0, @@ -3453,10 +3620,10 @@ "candidateScores": [ { "model": "openai/gpt-5.3-codex", - "score": 0.8903822492293204, + "score": 0.9113822492293204, "quality": 1, "cost": 0.5, - "speed": 0.039714153822005965, + "speed": 0.3897141538220059, "reliability": 0.79999 } ], @@ -3470,7 +3637,7 @@ "tier": "MEDIUM", "confidence": 0.7373034537835593, "method": "portfolio", - "reasoning": "score=0.09 | long (1217 tokens), references (following) | premium | v3 task=tool_agent agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=15", + "reasoning": "score=0.09 | long (1217 tokens), references (following) | premium | v3 task=tool_agent agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=26", "costEstimate": 0.0084417, "baselineCost": 0.089285, "savings": 0, @@ -3490,15 +3657,26 @@ "openai/gpt-5-mini", "openai/gpt-4.1", "openai/gpt-4o-mini", - "deepseek/deepseek-v4-pro" + "deepseek/deepseek-v4-pro", + "anthropic/claude-fable-5.1", + "anthropic/claude-opus-5.5", + "anthropic/claude-sonnet-5.5", + "google/gemini-3.8-flash", + "openai/gpt-5.1", + "openai/gpt-6-astra", + "openai/gpt-6-luna", + "openai/gpt-6-sol", + "xai/grok-4.6", + "xai/grok-4.7", + "xiaomi/mimo-v2.5" ], "candidateScores": [ { "model": "anthropic/claude-sonnet-5", - "score": 0.9059298386038215, + "score": 0.9269298386038214, "quality": 1, "cost": 0.5, - "speed": 0.09883064339702209, + "speed": 0.44883064339702206, "reliability": 1 } ], @@ -3512,7 +3690,7 @@ "tier": "MEDIUM", "confidence": 0.5, "method": "portfolio", - "reasoning": "score=0.06 | short (32 tokens), code (let), multi-step, references (following) | ambiguous -> default: MEDIUM | premium | v3 task=vision agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=8", + "reasoning": "score=-0.01 | short (32 tokens), multi-step, references (following) | ambiguous -> default: MEDIUM | premium | v3 task=vision agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=8", "costEstimate": 0.0017856000000000003, "baselineCost": 0.00656, "savings": 0, @@ -3530,18 +3708,18 @@ "candidateScores": [ { "model": "anthropic/claude-sonnet-4.6", - "score": 0.8675954266748616, + "score": 0.8885954266748617, "quality": 0.9, "cost": 1, - "speed": 0.09325711124769513, + "speed": 0.4432571112476951, "reliability": 1 }, { "model": "moonshot/kimi-k3", - "score": 0.8036386868652878, + "score": 0.8246386868652877, "quality": 0.9, "cost": 0, - "speed": 0.02731144775479715, + "speed": 0.37731144775479714, "reliability": 1 } ], @@ -3577,10 +3755,10 @@ "candidateScores": [ { "model": "google/gemini-3.5-flash", - "score": 0.9115266370965682, + "score": 0.9325266370965681, "quality": 1, "cost": 0.5, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 } ], @@ -3614,34 +3792,34 @@ "candidateScores": [ { "model": "anthropic/claude-sonnet-5", - "score": 0.8879298386038212, + "score": 0.9089298386038214, "quality": 0.9, "cost": 1, - "speed": 0.09883064339702209, + "speed": 0.44883064339702206, "reliability": 1 }, { "model": "google/gemini-3.5-flash", - "score": 0.8001933037632348, + "score": 0.8211933037632347, "quality": 0.9, "cost": 0, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 }, { "model": "moonshot/kimi-k3", - "score": 0.7971153996523453, + "score": 0.8181153996523453, "quality": 0.9, "cost": 0.0017922431715533538, - "speed": 0.02731144775479715, + "speed": 0.37731144775479714, "reliability": 1 }, { "model": "anthropic/claude-sonnet-4.6", - "score": 0.7878821855823102, + "score": 0.8088821855823103, "quality": 0.9, "cost": 0.0035844863431069296, - "speed": 0.09325711124769513, + "speed": 0.4432571112476951, "reliability": 1 } ], @@ -3651,42 +3829,39 @@ { "prompt": 15, "profile": "premium", - "model": "moonshot/kimi-k3", - "tier": "MEDIUM", - "confidence": 0.5, + "model": "google/gemini-2.5-flash", + "tier": "SIMPLE", + "confidence": 0.7009865197502964, "method": "portfolio", - "reasoning": "score=-0.06 | short (29 tokens), imperative (build), agentic-light (pip) | ambiguous -> default: MEDIUM | premium | v3 task=chat agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=10", - "costEstimate": 0.0223817, + "reasoning": "score=-0.07 | short (29 tokens), imperative (build) | premium | v3 task=chat agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=7", + "costEstimate": 0.0014153, "baselineCost": 0.083345, "savings": 0, - "agenticScore": 0.2, + "agenticScore": 0, "candidates": [ - "moonshot/kimi-k3", + "google/gemini-2.5-flash", "google/gemini-3.5-flash", - "openai/gpt-5.3-codex", - "anthropic/claude-sonnet-5", - "zai/glm-5.3", "google/gemini-3.6-flash", - "google/gemini-2.5-pro", - "xai/grok-4.5", - "anthropic/claude-sonnet-4.6", - "openai/gpt-5.6-terra" + "anthropic/claude-haiku-4.5", + "zai/glm-5.3", + "google/gemini-3.5-flash-lite", + "deepseek/deepseek-chat" ], "candidateScores": [ { - "model": "moonshot/kimi-k3", - "score": 0.8604386868652878, + "model": "google/gemini-2.5-flash", + "score": 0.8707937605287643, "quality": 0.86, "cost": 1, - "speed": 0.02731144775479715, + "speed": 0.5332293421460749, "reliability": 1 }, { "model": "google/gemini-3.5-flash", - "score": 0.770326637096568, + "score": 0.8113266370965679, "quality": 0.86, "cost": 0, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 } ], @@ -3700,7 +3875,7 @@ "tier": "MEDIUM", "confidence": 0.5, "method": "portfolio", - "reasoning": "score=-0.06 | short (38 tokens), imperative (deploy), references (the api), agentic-light (check the, deploy) | ambiguous -> default: MEDIUM | premium | v3 task=tool_agent agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=15", + "reasoning": "score=-0.06 | short (38 tokens), imperative (deploy), references (the api), agentic-light (check the, deploy) | ambiguous -> default: MEDIUM | premium | v3 task=tool_agent agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=26", "costEstimate": 0.0006437999999999999, "baselineCost": 0.0065899999999999995, "savings": 0, @@ -3720,15 +3895,26 @@ "openai/gpt-5-mini", "openai/gpt-4.1", "openai/gpt-4o-mini", - "deepseek/deepseek-v4-pro" + "deepseek/deepseek-v4-pro", + "anthropic/claude-fable-5.1", + "anthropic/claude-opus-5.5", + "anthropic/claude-sonnet-5.5", + "google/gemini-3.8-flash", + "openai/gpt-5.1", + "openai/gpt-6-astra", + "openai/gpt-6-luna", + "openai/gpt-6-sol", + "xai/grok-4.6", + "xai/grok-4.7", + "xiaomi/mimo-v2.5" ], "candidateScores": [ { "model": "anthropic/claude-sonnet-5", - "score": 0.9059298386038215, + "score": 0.9269298386038214, "quality": 1, "cost": 0.5, - "speed": 0.09883064339702209, + "speed": 0.44883064339702206, "reliability": 1 } ], @@ -3760,18 +3946,18 @@ "candidateScores": [ { "model": "anthropic/claude-sonnet-4.6", - "score": 0.8675954266748616, + "score": 0.8885954266748617, "quality": 0.9, "cost": 1, - "speed": 0.09325711124769513, + "speed": 0.4432571112476951, "reliability": 1 }, { "model": "moonshot/kimi-k3", - "score": 0.8036386868652878, + "score": 0.8246386868652877, "quality": 0.9, "cost": 0, - "speed": 0.02731144775479715, + "speed": 0.37731144775479714, "reliability": 1 } ], @@ -3805,34 +3991,34 @@ "candidateScores": [ { "model": "anthropic/claude-sonnet-5", - "score": 0.9011405003873671, + "score": 0.9501405003873673, "quality": 0.9, "cost": 1, - "speed": 0.1367178599097662, + "speed": 0.4867178599097662, "reliability": 1 }, { "model": "google/gemini-3.5-flash", - "score": 0.8118719412624161, + "score": 0.8608719412624161, "quality": 0.9, "cost": 0, - "speed": 0.16575196139820988, + "speed": 0.5157519613982099, "reliability": 1 }, { "model": "anthropic/claude-sonnet-4.6", - "score": 0.8011542380866079, + "score": 0.8501542380866078, "quality": 0.9, "cost": 0.004293688278231067, - "speed": 0.13436245017392473, + "speed": 0.4843624501739247, "reliability": 1 }, { "model": "moonshot/kimi-k3", - "score": 0.7986265738299553, + "score": 0.8476265738299552, "quality": 0.9, "cost": 0.002146844139115478, - "speed": 0.022296378324947415, + "speed": 0.3722963783249474, "reliability": 1 } ], @@ -3868,10 +4054,10 @@ "candidateScores": [ { "model": "google/gemini-3.5-flash", - "score": 0.9115266370965682, + "score": 0.9325266370965681, "quality": 1, "cost": 0.5, - "speed": 0.19211061827613424, + "speed": 0.5421106182761342, "reliability": 1 } ], @@ -3905,10 +4091,10 @@ "candidateScores": [ { "model": "openai/gpt-5.3-codex", - "score": 0.8903822492293204, + "score": 0.9113822492293204, "quality": 1, "cost": 0.5, - "speed": 0.039714153822005965, + "speed": 0.3897141538220059, "reliability": 0.79999 } ], @@ -3922,7 +4108,7 @@ "tier": "MEDIUM", "confidence": 0.7231218051243898, "method": "portfolio", - "reasoning": "score=0.08 | long (14423 tokens) | premium | v3 task=tool_agent agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=15", + "reasoning": "score=0.08 | long (14423 tokens) | premium | v3 task=tool_agent agentRisk=standard deepWebResearch=false terminalCode=false terminalSafety=false candidates=26", "costEstimate": 0.0046423, "baselineCost": 0.104115, "savings": 0, @@ -3942,15 +4128,26 @@ "openai/gpt-5-mini", "openai/gpt-4.1", "openai/gpt-4o-mini", - "deepseek/deepseek-v4-pro" + "deepseek/deepseek-v4-pro", + "anthropic/claude-fable-5.1", + "anthropic/claude-opus-5.5", + "anthropic/claude-sonnet-5.5", + "google/gemini-3.8-flash", + "openai/gpt-5.1", + "openai/gpt-6-astra", + "openai/gpt-6-luna", + "openai/gpt-6-sol", + "xai/grok-4.6", + "xai/grok-4.7", + "xiaomi/mimo-v2.5" ], "candidateScores": [ { "model": "anthropic/claude-sonnet-5", - "score": 0.9059298386038215, + "score": 0.9269298386038214, "quality": 1, "cost": 0.5, - "speed": 0.09883064339702209, + "speed": 0.44883064339702206, "reliability": 1 } ], diff --git a/tests/unit/test_router_core.py b/tests/unit/test_router_core.py index 8145d7d..6590bff 100644 --- a/tests/unit/test_router_core.py +++ b/tests/unit/test_router_core.py @@ -2,9 +2,10 @@ Parity tests for the Router Core port. Every case here is a 1:1 port of an upstream ``@blockrun/router-core`` vitest -case (``portfolio.test.ts``, ``selector.test.ts``, ``strategy.test.ts``, +case (``model-capabilities.test.ts``, ``portfolio.test.ts``, ``rules.test.ts``, +``selector.test.ts``, ``strategy.test.ts``, ``tokens.test.ts``, ``tool-intent.test.ts``, ``unavailable-models.test.ts`` at commit -``5ee7c23``). They are the regression guard +``2cd5a10``). They are the regression guard that the Python port keeps choosing the same models as the TypeScript SDK — when upstream is re-synced, re-port these alongside the source. """ @@ -17,10 +18,13 @@ import pytest from blockrun_llm.router_core import ( + DEFAULT_MODEL_CAPABILITIES, DEFAULT_ROUTING_CONFIG, RulesStrategy, apply_unavailable_models, calculate_model_cost, + classify_by_rules, + estimate_tokens, filter_by_exclude_list, filter_by_tool_calling, filter_candidates_by_capacity, @@ -30,6 +34,7 @@ route, ) from blockrun_llm.router_core.selector import select_model +from blockrun_llm.router_core.tokens import sample_for_scan def _price(input_price: float, output_price: float) -> dict[str, float]: @@ -59,6 +64,17 @@ def _price(input_price: float, output_price: float) -> dict[str, float]: "openai/gpt-4o-mini": _price(0.15, 0.6), "deepseek/deepseek-chat": _price(0.2, 0.4), "free/seed-oss-36b": _price(0, 0), + "anthropic/claude-fable-5.1": _price(10, 50), + "anthropic/claude-opus-5.5": _price(5, 25), + "anthropic/claude-sonnet-5.5": _price(3, 15), + "google/gemini-3.8-flash": _price(1, 5), + "openai/gpt-5.1": _price(1, 8), + "openai/gpt-6-astra": _price(5, 25), + "openai/gpt-6-luna": _price(0.2, 1.2), + "openai/gpt-6-sol": _price(1.75, 14), + "xai/grok-4.6": _price(2, 10), + "xai/grok-4.7": _price(2.5, 12), + "xiaomi/mimo-v2.5": _price(0.5, 2), } STRATEGY_PRICING = { @@ -222,6 +238,72 @@ def test_keeps_an_ordinary_single_lookup_on_the_standard_tool_agent_path(self): assert decision["model"] == "anthropic/claude-sonnet-5" assert "google/gemini-3.5-flash" in decision["candidates"] + def test_keeps_newly_tool_verified_catalog_models_as_fallbacks_without_promoting_them(self): + decision = _portfolio( + "Use lookup_order for order B-42.", + 256, + routing_profile="auto", + has_tools=True, + requires_tools=True, + tool_count=1, + unavailable_models=["openai/gpt-6-astra"], + ) + + assert decision["task_type"] == "tool_agent" + assert decision["model"] == "anthropic/claude-sonnet-5" + for model in ( + "anthropic/claude-fable-5.1", + "anthropic/claude-opus-5.5", + "anthropic/claude-sonnet-5.5", + "google/gemini-3.8-flash", + "openai/gpt-5.1", + "openai/gpt-6-luna", + "openai/gpt-6-sol", + "xai/grok-4.6", + "xai/grok-4.7", + "xiaomi/mimo-v2.5", + ): + assert model in decision["candidates"] + assert "openai/gpt-6-astra" not in decision["candidates"] + + def test_does_not_add_tool_only_availability_fallbacks_to_an_ordinary_chat_request(self): + decision = _portfolio("What is the capital of France?", 256) + + assert decision["task_type"] == "chat" + assert "openai/gpt-6-luna" not in decision["candidates"] + assert "anthropic/claude-sonnet-5.5" not in decision["candidates"] + + @pytest.mark.parametrize( + ("routing_profile", "first_new_model"), + [ + ("eco", "openai/gpt-6-luna"), + ("auto", "openai/gpt-6-luna"), + ("premium", "anthropic/claude-fable-5.1"), + ], + ) + def test_recovers_a_tool_turn_when_the_established_tool_chain_is_unavailable( + self, routing_profile, first_new_model + ): + options = { + "routing_profile": routing_profile, + "has_tools": True, + "requires_tools": True, + "tool_count": 1, + } + initial = _portfolio("Use lookup_order for order B-42.", 256, **options) + candidates = initial.get("candidates") or [] + first_new_index = candidates.index(first_new_model) if first_new_model in candidates else -1 + + assert first_new_index > 0 + decision = _portfolio( + "Use lookup_order for order B-42.", + 256, + **options, + unavailable_models=candidates[:first_new_index], + ) + + assert decision["model"] == first_new_model + def test_keeps_deep_multi_clue_web_research_on_sonnet_5(self): decision = _portfolio( "Research the following clues across multiple public sources and identify the country.", @@ -881,6 +963,139 @@ def test_lets_a_host_capability_snapshot_override_the_built_in_catalog(self): assert "anthropic/claude-sonnet-5" not in decision["candidates"] +class TestPortfolioSignalHygiene: + def test_estimates_cjk_input_by_character_so_a_long_chinese_document_is_long_context(self): + # 200k Chinese characters is far more than 80k tokens on every + # production tokenizer; a chars/4 estimate reports 50k and misses both + # the long-context task type and the capacity filter. + document = "这是一段很长的中文合同文本，需要总结其中的关键条款。" * 8_000 + decision = _portfolio(f"总结下面的合同：\n{document}", 1024) + + assert decision["task_type"] == "long_context" + assert decision["tier"] == "COMPLEX" + + def test_ignores_stakes_and_latency_cues_that_only_appear_in_the_system_prompt(self): + options = { + "config": DEFAULT_ROUTING_CONFIG, + "model_pricing": PORTFOLIO_PRICING, + "has_tools": True, + "requires_tools": True, + "tool_count": 1, + "tool_names": ["web_search"], + "now": datetime(2026, 7, 21, tzinfo=timezone.utc), + } + prompt = "search the web for the latest news on solar panels" + bare = route(prompt, None, 1024, options) + framed = route( + prompt, + "Follow security best practices for this production deployment. Be quick.", + 1024, + options, + ) + + assert framed["candidate_scores"] == bare["candidate_scores"] + + def test_detects_the_request_language_from_the_user_prompt_not_the_system_prompt(self): + prompt = "Extract orderId, quantity and status from: order A-17, qty 3, shipped. JSON only." + options = {"config": DEFAULT_ROUTING_CONFIG, "model_pricing": PORTFOLIO_PRICING} + bare = route(prompt, None, 256, options) + zh_system = route(prompt, "你是一个乐于助人的助手。", 256, options) + + assert bare["task_type"] == "extraction" + assert zh_system["model"] == bare["model"] + assert zh_system["candidates"] == bare["candidates"] + + def test_decays_a_stale_speed_observation_toward_neutral_not_toward_zero(self): + # A model with an old probe must not score below a model nobody has + # measured (which gets the neutral 0.5). Both models share the + # extraction affinity band, so both are scored. + equal_cost_pricing = { + "google/gemini-2.5-flash": _price(1, 1), + "openai/gpt-4o-mini": _price(1, 1), + } + two_candidate_config = { + **DEFAULT_ROUTING_CONFIG, + "tiers": { + tier: {"primary": "google/gemini-2.5-flash", "fallback": ["openai/gpt-4o-mini"]} + for tier in DEFAULT_ROUTING_CONFIG["tiers"] + }, + } + decision = route( + "Extract the fields as JSON", + None, + 512, + { + "config": two_candidate_config, + "model_pricing": equal_cost_pricing, + "now": datetime(2026, 7, 21, tzinfo=timezone.utc), + "model_performance": { + "openai/gpt-4o-mini": { + "measured_at": "2025-07-21T00:00:00Z", # a year old: freshness ~0 + "latency_ms": 600, + "output_tokens_per_second": 250, + }, + "google/gemini-2.5-flash": { + "measured_at": "not-measured", # no usable observation: neutral + "latency_ms": 0, + "output_tokens_per_second": 0, + }, + }, + }, + ) + + rows = {row["model"]: row for row in decision.get("candidate_scores") or []} + assert decision["task_type"] == "extraction" + assert rows["openai/gpt-4o-mini"]["speed"] == pytest.approx(0.5, abs=0.005) + assert rows["google/gemini-2.5-flash"]["speed"] == pytest.approx(0.5, abs=0.005) + + +class TestPortfolioCodeTaskTierFloor: + def test_keeps_a_stack_trace_off_the_simple_chain_even_when_the_rules_score_is_low(self): + # Word-boundary keyword matching removed the accidental `def`/`define` + # hits inside "undefined" that used to lift this prompt to MEDIUM. The + # portfolio classifier still knows it is a debug task, and the SIMPLE + # chain is curated for trivia and translation, not for reading a trace. + decision = _portfolio( + "debug: TypeError: Cannot read properties of undefined (reading 'map') " + "at UserList.render", + 1024, + ) + + assert decision["task_type"] == "debug" + assert decision["tier"] == "MEDIUM" + + +class TestPortfolioCodeTaskFloorGating: + @pytest.mark.parametrize( + "prompt", + [ + "Rewrite this sentence: The weather is nice today.", + "What does margin of error mean?", + ], + ) + def test_leaves_prose_that_merely_says_rewrite_or_error_on_the_simple_chain(self, prompt): + assert _portfolio(prompt, 256)["tier"] == "SIMPLE" + + @pytest.mark.parametrize( + "prompt", + [ + "What is Python?", + "Translate this sentence into French: I like Python.", + "Translate into French: Please make an exception.", + ], + ) + def test_does_not_treat_a_bare_programming_language_mention_as_code_evidence(self, prompt): + assert _portfolio(prompt, 256)["tier"] == "SIMPLE" + + def test_treats_a_camelcase_error_class_as_code_evidence(self): + decision = _portfolio( + "NullPointerException at OrderService.java:88 when the cart is empty, why?", 256 + ) + + assert decision["task_type"] == "debug" + assert decision["tier"] == "MEDIUM" + + # ─── selector.test.ts ─── SELECTOR_TIER_CONFIGS = { @@ -914,6 +1129,18 @@ def test_select_model_uses_opus_4_7_as_the_savings_baseline(self): assert decision["baseline_cost"] > 0 assert decision["savings"] > 0 + def test_select_model_does_not_claim_savings_for_a_model_whose_price_is_unknown(self): + unpriced = { + **SELECTOR_TIER_CONFIGS, + "SIMPLE": {"primary": "vendor/unlisted-model", "fallback": []}, + } + decision = select_model( + "SIMPLE", 0.95, "rules", "test", unpriced, SELECTOR_PRICING, 1000, 1000 + ) + + assert decision["baseline_cost"] > 0 + assert decision["savings"] == 0 + def test_calculate_model_cost_uses_opus_4_7_as_the_baseline(self): costs = calculate_model_cost("moonshot/kimi-k2.5", SELECTOR_PRICING, 1000, 1000) @@ -1199,6 +1426,24 @@ def test_recognizes_math_word_problems_without_question_marks(self): assert decision["task_type"] == "reasoning_math" +class TestRulesStrategySignalHygiene: + def test_does_not_read_a_structured_output_requirement_out_of_the_system_prompt(self): + # Agent hosts describe their tool schemas in the system prompt on every + # turn; that text is not evidence that this request needs structured + # output. Hosts state the requirement through `requires_structured_output`. + strategy = RulesStrategy() + with_schema_prompt = strategy.route( + "hello", + "You are a helpful assistant. Tools are described below as a JSON schema.", + 100, + BASE_OPTIONS, + ) + bare = strategy.route("hello", None, 100, BASE_OPTIONS) + + assert with_schema_prompt["tier"] == bare["tier"] + assert "structured output" not in with_schema_prompt["reasoning"] + + # ─── tool-intent.test.ts ─── @@ -1377,3 +1622,141 @@ def test_survives_killing_an_entire_tier_chain(self): self._options(unavailable_models=chain), ) assert len(decision["model"]) > 0 + + +# ─── model-capabilities.test.ts ─── + + +class TestGeneratedModelCapabilitySnapshot: + @pytest.mark.parametrize( + "model", + [ + "anthropic/claude-fable-5.1", + "anthropic/claude-opus-5.5", + "anthropic/claude-sonnet-5.5", + "google/gemini-3.8-flash", + "openai/gpt-5.1", + "openai/gpt-6-astra", + "openai/gpt-6-luna", + "openai/gpt-6-sol", + "xai/grok-4.6", + "xai/grok-4.7", + "xiaomi/mimo-v2.5", + ], + ) + def test_accepts_a_live_probed_tool_call(self, model): + assert DEFAULT_MODEL_CAPABILITIES[model]["supports_tools"] is True + + @pytest.mark.parametrize( + "model", ["deepseek/deepseek-v4-flash-vision-exp", "qwen/qwen3.8-flash"] + ) + def test_keeps_a_model_out_of_tool_required_routes_after_an_invalid_tool_response(self, model): + assert DEFAULT_MODEL_CAPABILITIES[model]["supports_tools"] is False + + +# ─── rules.test.ts ─── + +SCORING = DEFAULT_ROUTING_CONFIG["scoring"] + + +def _signals_for(prompt: str) -> list[str]: + return classify_by_rules(prompt, None, math.ceil(len(prompt) / 4), SCORING)["signals"] + + +def _has_signal(signals: list[str], fragment: str) -> bool: + return any(fragment in signal for signal in signals) + + +class TestClassifyByRulesKeywordMatching: + def test_does_not_count_a_keyword_that_only_appears_inside_a_longer_word(self): + # "classic" is not `class`, "history" is not `story`, "credit" is not + # `edit`, "pipeline" is not `pip`, "prefix" is not `fix`. + assert not _has_signal(_signals_for("I need a recipe for a classic dinner party"), "code") + assert not _has_signal(_signals_for("Tell me the history of the Roman Empire"), "creative") + assert not _has_signal( + _signals_for("Why does my credit card pipeline report a prefix mismatch?"), "agentic" + ) + + def test_still_matches_ordinary_english_inflections_of_a_keyword(self): + assert _has_signal(_signals_for("I imported the classes and defined two functions"), "code") + assert _has_signal( + _signals_for("Keep editing until the tests pass, then deploying"), "agentic" + ) + + def test_keeps_substring_matching_for_cjk_keywords_which_have_no_word_boundaries(self): + assert _has_signal(_signals_for("请帮我定义一个函数来处理输入"), "code") + + def test_keeps_inflections_of_the_ascii_keywords_of_other_european_languages(self): + # `beweis` and `mathematisch` are German reasoning markers spelled in + # ASCII; the end boundary must not require English morphology of them. + result = classify_by_rules("Beweise den mathematischen Satz.", None, 8, SCORING) + + assert result["tier"] == "REASONING" + + def test_matches_doubled_consonant_english_inflections(self): + result = classify_by_rules("Keep debugging and editing until it works.", None, 11, SCORING) + + assert result["agentic_score"] == 0.6 + + def test_does_not_let_a_short_keyword_plus_a_suffix_spell_a_different_word(self): + # "pipes" is not `pip`, "defer" is not `def`, "important" is not + # `import`, "basic" is not `asic`, "semantic" is not `sem`. + prompt = "It is important to defer the basic semantic check for the pipes" + result = classify_by_rules(prompt, None, 16, SCORING) + + assert [ + signal + for signal in result["signals"] + if any(label in signal for label in ("code", "agentic", "domain", "negation")) + ] == [] + assert _has_signal(_signals_for("Apply the fixes and rerun"), "agentic") + + def test_treats_a_cjk_character_next_to_a_latin_keyword_as_a_word_boundary(self): + # Mixed-script prompts carry no spaces between scripts. + result = classify_by_rules("请step by step证明这个结论。", None, 8, SCORING) + + assert _has_signal(result["signals"], "step by step") + assert result["tier"] == "REASONING" + + def test_keeps_stem_matching_for_cyrillic_keywords(self): + # "оптимизаци" is a deliberate stem that must still hit "оптимизацию". + assert _has_signal( + _signals_for("Проведи оптимизацию алгоритма для базы данных"), "technical" + ) + + +# ─── tokens.test.ts ─── + + +class TestEstimateTokens: + def test_keeps_the_four_characters_per_token_estimate_for_latin_text(self): + text = "The quick brown fox jumps over the lazy dog." * 10 # 440 chars + + assert estimate_tokens(text) == 110 + + def test_counts_a_cjk_character_as_roughly_one_token_not_a_quarter_of_one(self): + text = "这是一段需要总结的中文合同文本。" * 100 # 1600 chars + + assert 1400 <= estimate_tokens(text) <= 1700 + + def test_adds_the_two_estimates_for_mixed_text(self): + latin = "a" * 400 # 100 tokens + cjk = "中" * 100 # ~100 tokens + + assert estimate_tokens(latin + cjk) == 200 + + def test_returns_zero_for_empty_text(self): + assert estimate_tokens("") == 0 + + +class TestSampleForScan: + def test_returns_short_text_unchanged(self): + assert sample_for_scan("hello", 10) == "hello" + + def test_keeps_both_ends_of_a_long_text_so_the_instruction_survives(self): + text = f"HEAD {'x' * 100} TAIL" + sampled = sample_for_scan(text, 20) + + assert sampled.startswith("HEAD") + assert sampled.endswith("TAIL") + assert len(sampled) <= 21 diff --git a/tests/unit/test_router_core_snapshot.py b/tests/unit/test_router_core_snapshot.py index e447e1a..7fd79c2 100644 --- a/tests/unit/test_router_core_snapshot.py +++ b/tests/unit/test_router_core_snapshot.py @@ -2,7 +2,7 @@ Cross-language decision-snapshot parity. ``router_core_decisions.snapshot.json`` is a verbatim copy of upstream -``decisions.snapshot.json`` at commit ``e38958b`` — 88 complete decisions the +``decisions.snapshot.json`` at commit ``2cd5a10`` — 88 complete decisions the TypeScript engine produced for a frozen corpus (22 prompts x 4 profiles with rotating tool/vision/structured-output shapes, frozen pricing, frozen clock). This test recomputes every decision with the Python port and compares field From 893a94b3ee194ff767760a5b48cb9c86b8f36740 Mon Sep 17 00:00:00 2001 From: 1bcMax Date: Tue, 20 Oct 2026 10:56:17 -0500 Subject: [PATCH 2/2] ci: pin the test job to ubuntu-24.04 so Python 3.9 still installs --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f0ca04f..55981d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,8 @@ on: jobs: test: - runs-on: ubuntu-latest + # ubuntu-latest moved to 26.04 (2026-10-20), which has no Python 3.9 build. + runs-on: ubuntu-24.04 strategy: matrix: python-version: ['3.9', '3.11', '3.12']