feat(auto-router): add JEV classifier alongside LLM classifier

litellm/proxy/health_check.py

@@ -377,6 +377,7 @@ def _strategy_router_dependency_error(
         (
             failure
             for dependency in strategy_router_dependencies(params)
+            if dependency.role != "evaluation"
             if (failure := _dependency_failure(dependency, router, unhealthy_ids))
         ),
         None,
@@ -419,6 +420,7 @@ def _dependency_deployments_to_probe(
             for deployment in frontier
             if isinstance(params := deployment.get("litellm_params"), Mapping)
             for dependency in strategy_router_dependencies(params)
+            if dependency.role != "evaluation"
         )
         fresh_ids = (
             frozenset(ident for name in names for ident in (_resolved_deployment_ids(router, name) or ())) - reached

litellm/proxy/management_endpoints/auto_router_endpoints.py

@@ -294,14 +294,16 @@ def _models_this_test_can_call(config: RequestComplexityRouterConfig) -> tuple[s
     Excludes every tier's models: the prompt is never sent to the model it routed to.
     """
     return tuple(
-        model
-        for model in (
-            config.classifier_llm_config.model
-            if config.uses_llm_classifier and config.classifier_llm_config is not None
-            else None,
-            config.embedding_model if config.semantic_keyword_matching else None,
+        dependency.model_name
+        for dependency in strategy_router_dependencies(
+            MappingProxyType(
+                {
+                    "model": "auto_router/complexity_router",
+                    "complexity_router_config": config.model_dump(exclude_none=True),
+                }
+            )
         )
-        if model is not None
+        if dependency.role in ("classifier", "embedding", "evaluation")
     )
 
 
@@ -390,6 +392,40 @@ async def validate_complexity_router_config(
     return ComplexityRouterConfigValidationResponse(valid=error is None, error=error)
 
 
+async def _resolve_saved_routing_test(
+    data: AutoRouterRoutingTestRequest,
+    user_api_key_dict: UserAPIKeyAuth,
+    llm_router: "Router",
+) -> AutoRouterRoutingTestRequest:
+    if data.saved_model_id is None:
+        return data
+    deployment: Final = llm_router.get_deployment(data.saved_model_id)
+    if deployment is None or deployment.model_info.blocked:
+        raise HTTPException(status_code=404, detail="Saved auto router is unavailable")
+    if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN and deployment.model_info.team_id != data.team_id:
+        raise HTTPException(status_code=403, detail="Saved auto router belongs to a different team")
+    await can_key_call_resolved_model(
+        model=deployment.model_info.team_public_model_name or deployment.model_name,
+        llm_model_list=llm_router.model_list,
+        valid_token=user_api_key_dict,
+        llm_router=llm_router,
+    )
+    params: Final = deployment.litellm_params
+    if classify_strategy_router_model(params.model or "") != "complexity" or params.complexity_router_config is None:
+        raise HTTPException(status_code=400, detail="Saved deployment is not a complexity auto router")
+    return data.model_copy(
+        update=MappingProxyType(
+            {
+                "complexity_router_config": RequestComplexityRouterConfig.model_validate(
+                    params.complexity_router_config
+                ),
+                "default_model": params.complexity_router_default_model,
+                "router_name": deployment.model_name,
+            }
+        )
+    )
+
+
 @router.post(
     "/auto_router/test_routing",
     tags=["model management"],  # mutable-ok: fastapi's decorator signature types tags as a list
@@ -445,23 +481,31 @@ async def preview_auto_router_routing(
     from litellm.proxy.utils import get_available_models_for_user
 
     member_team: Final = await _authorize_router_dry_run(user_api_key_dict=user_api_key_dict, team_id=data.team_id)
+    if llm_router is None:
+        raise HTTPException(
+            status_code=500,
+            detail={  # mutable-ok: HTTPException detail must be a plain mapping
+                "error": CommonProxyErrors.no_llm_router.value
+            },
+        )
+    resolved: Final = await _resolve_saved_routing_test(data, user_api_key_dict, llm_router)
     actor: Final = (
         await _authorize_member_dry_run_config(
-            config=data.complexity_router_config.model_dump(exclude_none=True),
-            default_model=data.default_model,
+            config=resolved.complexity_router_config.model_dump(exclude_none=True),
+            default_model=resolved.default_model,
             user_api_key_dict=user_api_key_dict,
             team=member_team,
         )
         if member_team is not None
         else user_api_key_dict
     )
     request_data: Final[dict[str, object]] = {  # mutable-ok: auth and routing enrich this request in place
-        **data.wire_body(),
+        **resolved.wire_body(),
         "metadata": {},  # mutable-ok: centralized auth and identity stamping share this metadata bucket
         "proxy_server_request": {"body": None},  # mutable-ok: the snapshot owner fills this body in place
     }
 
-    if member_team is not None and _models_this_test_can_call(data.complexity_router_config):
+    if member_team is not None and _models_this_test_can_call(resolved.complexity_router_config):
         from litellm.proxy.auth.user_api_key_auth import (
             _run_centralized_common_checks,  # pyright: ignore[reportPrivateUsage]  # reuse the serving admission policy
         )
@@ -473,25 +517,17 @@ async def preview_auto_router_routing(
             route="/auto_router/test_routing",
         )
 
-    if llm_router is None:
-        raise HTTPException(
-            status_code=500,
-            detail={  # mutable-ok: HTTPException detail must be a plain mapping
-                "error": CommonProxyErrors.no_llm_router.value
-            },
-        )
-
     await _authorize_models_this_test_can_call(
-        config=data.complexity_router_config,
+        config=resolved.complexity_router_config,
         user_api_key_dict=actor,
         llm_router=llm_router,
     )
 
     complexity_router: Final = ComplexityRouter(
-        model_name=data.router_name,
+        model_name=resolved.router_name,
         litellm_router_instance=llm_router,
-        complexity_router_config=data.complexity_router_config.model_dump(exclude_none=True),
-        default_model=data.default_model,
+        complexity_router_config=resolved.complexity_router_config.model_dump(exclude_none=True),
+        default_model=resolved.default_model,
         derive_savings_baseline=False,
     )
 
@@ -504,7 +540,7 @@ async def preview_auto_router_routing(
 
     try:
         hook_response: Final = await complexity_router.async_pre_routing_hook(
-            model=data.router_name,
+            model=resolved.router_name,
             request_kwargs=request_kwargs,
             messages=request_kwargs["messages"],
         )

litellm/proxy/management_endpoints/model_management_endpoints.py

@@ -22,7 +22,7 @@
 from typing import TYPE_CHECKING, Annotated, Final, Literal, Protocol, TypeVar, cast, runtime_checkable
 
 from fastapi import APIRouter, Depends, Header, HTTPException, Request, status
-from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator
+from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError, field_validator
 
 import litellm
 from litellm._logging import verbose_proxy_logger
@@ -289,7 +289,11 @@ def _strategy_router_write_violation(
     if incoming_params is None:
         return None
     config_violation: Final = validate_complexity_router_config_write(
-        complexity_router_config=incoming_params.complexity_router_config
+        complexity_router_config=(
+            _effective_complexity_router_config(incoming_params, existing_params)
+            if incoming_params.complexity_router_config is not None
+            else None
+        )
     )
     if config_violation is not None:
         return config_violation
@@ -350,11 +354,33 @@ def _raise_on_strategy_router_write_violation(
 def _effective_complexity_router_config(
     incoming_params: GenericLiteLLMParams | None, existing_params: GenericLiteLLMParams | None
 ) -> object:
-    """The complexity config a write leaves on the row: the incoming one when the write carries it, else the stored one."""
     incoming: Final = None if incoming_params is None else incoming_params.complexity_router_config
-    if incoming is not None or existing_params is None:
+    existing: Final = None if existing_params is None else existing_params.complexity_router_config
+    if incoming is None:
+        return existing
+    if existing is None or incoming.get("classifier_type") != "jev" or existing.get("classifier_type") != "jev":
+        return incoming
+    incoming_jev: Final[object] = incoming.get("jev_classifier_config")
+    existing_jev: Final[object] = existing.get("jev_classifier_config")
+    if not isinstance(incoming_jev, Mapping) or not isinstance(existing_jev, Mapping):
         return incoming
-    return existing_params.complexity_router_config
+    supplied: Final = TypeAdapter(dict[str, object]).validate_python(incoming_jev)
+    stored: Final = TypeAdapter(dict[str, object]).validate_python(existing_jev)
+    same_base: Final = "api_base" not in supplied or supplied["api_base"] == stored.get("api_base")
+    transport: Final = MappingProxyType(
+        {
+            key: value
+            for key, value in stored.items()
+            if key in ("api_key", "api_base") and (key != "api_key" or same_base)
+        }
+    )
+    return {  # mutable-ok: persisted JSON requires concrete nested dicts
+        **incoming,
+        "jev_classifier_config": {  # mutable-ok: json.dumps cannot serialize MappingProxyType
+            **transport,
+            **supplied,
+        },
+    }
 
 
 def _effective_model(
@@ -886,7 +912,12 @@ def update_db_model(db_model: Deployment, updated_patch: updateDeployment) -> Pr
     if updated_patch.litellm_params:
         # Encrypt any sensitive values
         encrypted_params: Final = {
-            k: encrypt_value_helper(v) for k, v in updated_patch.litellm_params.model_dump(exclude_none=True).items()
+            k: (
+                _effective_complexity_router_config(updated_patch.litellm_params, db_model.litellm_params)
+                if k == "complexity_router_config"
+                else encrypt_value_helper(v)
+            )
+            for k, v in updated_patch.litellm_params.model_dump(exclude_none=True).items()
         }
 
         merged_litellm_params.update(encrypted_params)
@@ -2528,14 +2559,21 @@ async def update_model(
             _new_litellm_params_dict: Final = model_params.litellm_params.dict(exclude_none=True)
 
             ### ENCRYPT PARAMS ###
-            for k, v in _new_litellm_params_dict.items():
-                encrypted_value = encrypt_value_helper(value=v)
-                model_params.litellm_params[k] = encrypted_value
+            encrypted_params: Final = MappingProxyType(
+                {
+                    k: (
+                        _effective_complexity_router_config(model_params.litellm_params, deployment.litellm_params)
+                        if k == "complexity_router_config"
+                        else encrypt_value_helper(value=v)
+                    )
+                    for k, v in _new_litellm_params_dict.items()
+                }
+            )
 
             ### MERGE WITH EXISTING DATA ###
             _mp: Final[dict[str, object]] = model_params.litellm_params.dict()
             merged_dictionary: Final = {
-                key: _existing_litellm_params_dict[key] if value is None else value
+                key: _existing_litellm_params_dict[key] if value is None else encrypted_params[key]
                 for key, value in _mp.items()
                 if value is not None or _existing_litellm_params_dict.get(key) is not None
             }

litellm/proxy/management_helpers/auto_router_permissions.py

@@ -179,14 +179,23 @@ async def authorize_member_auto_router_dependencies(
             }
         )
     )
-    for model, deployments in (
-        (dependency.model_name, llm_router.get_model_list(model_name=dependency.model_name, team_id=team.team_id))
+    for dependency, model, deployments in (
+        (
+            dependency,
+            dependency.model_name,
+            llm_router.get_model_list(model_name=dependency.model_name, team_id=team.team_id),
+        )
         for dependency in dependencies
     ):
-        if not deployments or any(
-            classify_strategy_router_model(_RouterConfigSource.model_validate(deployment["litellm_params"]).model or "")
-            is not None
-            for deployment in deployments
+        if dependency.role != "evaluation" and (
+            not deployments
+            or any(
+                classify_strategy_router_model(
+                    _RouterConfigSource.model_validate(deployment["litellm_params"]).model or ""
+                )
+                is not None
+                for deployment in deployments
+            )
         ):
             raise HTTPException(status_code=400, detail=f"Auto-router target {model!r} must be a configured model.")
         await can_team_access_model(