@@ -354,58 +354,6 @@ func TestHandleHTTP_AllNodesDownReturnsError(t *testing.T) {
354354 })
355355}
356356
357- // TestHandleHTTP_InferenceRouting proves every path classified as inference
358- // applies model-based candidate filtering and is forwarded unchanged — not
359- // only the single engineCase.inferencePath most bodies use. That includes
360- // shared paths such as /v1/messages.
361- func TestHandleHTTP_InferenceRouting (t * testing.T ) {
362- forEachEngine (t , func (t * testing.T , tc engineCase ) {
363- for _ , r := range tc .profile .Routes {
364- if r .Role != roleInferencePOST {
365- continue
366- }
367- path := r .Path
368- t .Run (path , func (t * testing.T ) {
369- var gotBody string
370- var gotPath string
371- good := httptest .NewServer (http .HandlerFunc (func (w http.ResponseWriter , r * http.Request ) {
372- b , _ := io .ReadAll (r .Body )
373- gotBody = string (b )
374- gotPath = r .URL .Path
375- w .WriteHeader (http .StatusOK )
376- }))
377- defer good .Close ()
378-
379- wrongModel := httptest .NewServer (http .HandlerFunc (func (w http.ResponseWriter , _ * http.Request ) {
380- t .Error ("wrong-model node should not receive request" )
381- w .WriteHeader (http .StatusOK )
382- }))
383- defer wrongModel .Close ()
384-
385- disc := NewDiscovery ()
386- disc .AddManual (nodeForModel (t , "wrong" , wrongModel .URL , "different-model" ))
387- disc .AddManual (nodeForModel (t , "good" , good .URL , tc .advertisedModel ))
388- p := testProxy (tc .profile , disc , tc .profile .FacadePort )
389- p .soleFacade ().SetSelected ("wrong" )
390-
391- body := tc .inferenceBody ()
392- rec := httptest .NewRecorder ()
393- p .soleFacade ().handleHTTP (rec , httptest .NewRequest (http .MethodPost , path , strings .NewReader (body )))
394-
395- if rec .Code != http .StatusOK {
396- t .Fatalf ("status = %d, want 200" , rec .Code )
397- }
398- if gotBody != body {
399- t .Errorf ("node got body %q, want %q" , gotBody , body )
400- }
401- if gotPath != path {
402- t .Errorf ("path = %q, want %q" , gotPath , path )
403- }
404- })
405- }
406- })
407- }
408-
409357// TestHandleHTTP_404FailoverInferenceOnly: a 404 (model-not-found) on an
410358// inference call fails over to the next advertised owner, but a 404 on a
411359// non-inference path is returned as-is.
@@ -431,24 +379,15 @@ func TestHandleHTTP_404FailoverInferenceOnly(t *testing.T) {
431379 return p
432380 }
433381
434- // Inference POST: 404 on first → fail over → 200, on every classified
435- // inference path (including shared routes such as /v1/messages).
436- for _ , r := range tc .profile .Routes {
437- if r .Role != roleInferencePOST {
438- continue
439- }
440- path := r .Path
441- t .Run (path , func (t * testing.T ) {
442- rec := httptest .NewRecorder ()
443- newProxy ().soleFacade ().handleHTTP (rec , httptest .NewRequest (http .MethodPost , path , strings .NewReader (tc .inferenceBody ())))
444- if rec .Code != http .StatusOK {
445- t .Fatalf ("inference 404: status = %d, want 200 (should fail over)" , rec .Code )
446- }
447- })
382+ // Inference POST: 404 on first → fail over → 200.
383+ rec := httptest .NewRecorder ()
384+ newProxy ().soleFacade ().handleHTTP (rec , tc .inferenceRequest ())
385+ if rec .Code != http .StatusOK {
386+ t .Fatalf ("inference 404: status = %d, want 200 (should fail over)" , rec .Code )
448387 }
449388
450389 // An ordinary non-inference GET still returns the first node's 404.
451- rec : =httptest .NewRecorder ()
390+ rec = httptest .NewRecorder ()
452391 newProxy ().soleFacade ().handleHTTP (rec , httptest .NewRequest (http .MethodGet , tc .nonInferencePath , nil ))
453392 if rec .Code != http .StatusNotFound {
454393 t .Fatalf ("non-inference 404: status = %d, want 404 (must NOT fail over)" , rec .Code )
@@ -702,6 +641,87 @@ func TestHandleHTTP_StrictModelRouting(t *testing.T) {
702641 })
703642}
704643
644+ // TestHandleHTTP_InferenceRouting proves each engine inference route is
645+ // model-routed, retries a model-not-found response, and
646+ // forwards the request path and body unchanged.
647+ func TestHandleHTTP_InferenceRouting (t * testing.T ) {
648+ test := func (name , path string , profiles ... engineProfile ) {
649+ t .Run (name , func (t * testing.T ) {
650+ for _ , profile := range profiles {
651+ t .Run (profile .Name , func (t * testing.T ) {
652+ requestedModel := "requested-model"
653+ advertisedModel := profile .normalizeModel (requestedModel )
654+
655+ wrongModel := httptest .NewServer (http .HandlerFunc (func (w http.ResponseWriter , _ * http.Request ) {
656+ t .Error ("wrong-model node should not receive request" )
657+ w .WriteHeader (http .StatusOK )
658+ }))
659+ defer wrongModel .Close ()
660+
661+ missing := httptest .NewServer (http .HandlerFunc (func (w http.ResponseWriter , _ * http.Request ) {
662+ w .WriteHeader (http .StatusNotFound )
663+ if _ , err := io .WriteString (w , `{"error":"model not found"}` ); err != nil {
664+ t .Errorf ("write missing-model response: %v" , err )
665+ }
666+ }))
667+ defer missing .Close ()
668+
669+ var gotBody string
670+ var gotPath string
671+ good := httptest .NewServer (http .HandlerFunc (func (w http.ResponseWriter , r * http.Request ) {
672+ body , err := io .ReadAll (r .Body )
673+ if err != nil {
674+ t .Errorf ("read forwarded request body: %v" , err )
675+ w .WriteHeader (http .StatusInternalServerError )
676+ return
677+ }
678+ gotBody = string (body )
679+ gotPath = r .URL .Path
680+ w .WriteHeader (http .StatusOK )
681+ }))
682+ defer good .Close ()
683+
684+ disc := NewDiscovery ()
685+ disc .AddManual (nodeForModel (t , "wrong" , wrongModel .URL , "different-model" ))
686+ disc .AddManual (nodeForModel (t , "missing" , missing .URL , advertisedModel ))
687+ disc .AddManual (nodeForModel (t , "good" , good .URL , advertisedModel ))
688+ p := testProxy (profile , disc , profile .FacadePort )
689+ p .soleFacade ().SetSelected ("wrong" )
690+ // Stable node ordering would send this request to "good" first and
691+ // never exercise 404 failover. Prioritize "missing" so the test
692+ // independently proves both model filtering and retry behavior.
693+ p .SetPriority ([]string {"missing" , "good" })
694+
695+ body := `{"model":"requested-model","max_tokens":1024,"messages":[{"role":"user","content":"Hello"}]}`
696+ rec := httptest .NewRecorder ()
697+ p .soleFacade ().handleHTTP (rec , httptest .NewRequest (http .MethodPost , path , strings .NewReader (body )))
698+
699+ if rec .Code != http .StatusOK {
700+ t .Fatalf ("status = %d, want 200 after 404 failover" , rec .Code )
701+ }
702+ if gotBody != body {
703+ t .Errorf ("node got body %q, want %q" , gotBody , body )
704+ }
705+ if gotPath != path {
706+ t .Errorf ("path = %q, want %q" , gotPath , path )
707+ }
708+ })
709+ }
710+ })
711+ }
712+
713+ ollama := ollamaCase (t ).profile
714+ lmstudio := lmstudioCase (t ).profile
715+ test ("native Ollama generate" , "/api/generate" , ollama )
716+ test ("native Ollama chat" , "/api/chat" , ollama )
717+ test ("native Ollama embeddings" , "/api/embeddings" , ollama )
718+ test ("native Ollama embed" , "/api/embed" , ollama )
719+ test ("OpenAI chat completions" , "/v1/chat/completions" , ollama , lmstudio )
720+ test ("OpenAI completions" , "/v1/completions" , ollama , lmstudio )
721+ test ("OpenAI embeddings" , "/v1/embeddings" , ollama , lmstudio )
722+ test ("Anthropic messages" , "/v1/messages" , ollama , lmstudio )
723+ }
724+
705725func TestHandleHTTP_NoAdvertisedModelRejectsLocally (t * testing.T ) {
706726 forEachEngine (t , func (t * testing.T , tc engineCase ) {
707727 hits := 0
0 commit comments