Skip to content

Commit 5840b2b

Browse files
RatherrayeesRatherrayees
committed
Add Anthropic Messages API Routing
Signed-off-by: Rayees <rayiesamin@gmail.com>
1 parent 13b6811 commit 5840b2b

7 files changed

Lines changed: 171 additions & 9 deletions

File tree

‎docs/getting-started.mdx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -406,8 +406,8 @@ What that base URL serves depends on the engine behind it:
406406

407407
| Endpoint | Default base URL | Paths it serves |
408408
| --- | --- | --- |
409-
| Ollama | `http://127.0.0.1:11434` | Ollama's own API — `/api/chat`, `/api/generate`, `/api/embed`, `/api/tags` — and the OpenAI-compatible `/v1/chat/completions`, `/v1/completions`, `/v1/embeddings`, `/v1/models` |
410-
| LM Studio | `http://127.0.0.1:1234` | The OpenAI-compatible paths only |
409+
| Ollama | `http://127.0.0.1:11434` | Ollama's own API — `/api/chat`, `/api/generate`, `/api/embed`, `/api/tags` — and the OpenAI-compatible `/v1/chat/completions`, `/v1/completions`, `/v1/embeddings`, `/v1/models`, plus the Anthropic Messages API `/v1/messages` |
410+
| LM Studio | `http://127.0.0.1:1234` | The OpenAI-compatible paths (`/v1/chat/completions`, `/v1/completions`, `/v1/embeddings`, `/v1/models`) and the Anthropic Messages API `/v1/messages` |
411411

412412
The distinction matters when you fill in an application's settings. A client
413413
written against the OpenAI API usually wants the `/v1` included, as in

‎services/lmstudio-proxy/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ SPDX-License-Identifier: Apache-2.0
77

88
A discovery-aware HTTP reverse proxy for LM Studio nodes on the local network. It runs no mDNS browse of its own: its routing targets come from the broker's discovery relay (it sends `discovery:subscribe {services:[lm]}` and replaces its routing overlay from each pushed `discovery:nodes` snapshot) plus user-added manual nodes. It forwards HTTP requests to the selected node, aggregates the model-list route across candidate nodes, and exposes a bidirectional JSON-RPC 2.0 control channel over stdio (or an IPC socket).
99

10-
> **Clone of `ollama-proxy`.** This proxy is a deliberate clone of [`ollama-proxy`](../ollama-proxy/README.md) so the two share identical routing, failover, CORS, and node-selection behavior — the CORS policy is literally the same code, `nvpair-shared/cors`, and is documented [there](../ollama-proxy/README.md#http-reverse-proxy). The differences are engine-specific: it subscribes to the discovery relay for `lm` nodes, forwards the OpenAI-compatible inference routes (`/v1/chat/completions`, `/v1/completions`, `/v1/embeddings`), tags workloads `lmstudio`, and persists its port to its own file. It has no `--alias-address`, so its self-forward guard covers only its own listener.
10+
> **Clone of `ollama-proxy`.** This proxy is a deliberate clone of [`ollama-proxy`](../ollama-proxy/README.md) so the two share identical routing, failover, CORS, and node-selection behavior — the CORS policy is literally the same code, `nvpair-shared/cors`, and is documented [there](../ollama-proxy/README.md#http-reverse-proxy). The differences are engine-specific: it subscribes to the discovery relay for `lm` nodes, forwards the OpenAI-compatible inference routes (`/v1/chat/completions`, `/v1/completions`, `/v1/embeddings`) and the Anthropic Messages API (`/v1/messages`), tags workloads `lmstudio`, and persists its port to its own file. It has no `--alias-address`, so its self-forward guard covers only its own listener.
1111
1212
## Build
1313

‎services/lmstudio-proxy/failover_test.go‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -598,3 +598,83 @@ func TestResolveCandidates_SelfGuard(t *testing.T) {
598598
t.Errorf("expected the real node to survive the self-guard, candidates = %+v", cands)
599599
}
600600
}
601+
602+
// TestHandleHTTP_AnthropicMessages_InferenceRouting proves /v1/messages is
603+
// treated as an inference endpoint: model-based candidate filtering is applied
604+
// and the request body is forwarded unchanged to the matching node.
605+
func TestHandleHTTP_AnthropicMessages_InferenceRouting(t *testing.T) {
606+
var gotBody string
607+
var gotPath string
608+
good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
609+
b, _ := io.ReadAll(r.Body)
610+
gotBody = string(b)
611+
gotPath = r.URL.Path
612+
w.WriteHeader(http.StatusOK)
613+
_, _ = io.WriteString(w, `{"id":"msg_test","type":"message","role":"assistant","content":[{"type":"text","text":"Hello!"}]}`)
614+
}))
615+
defer good.Close()
616+
617+
wrongModel := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
618+
t.Error("wrong-model node should not receive request")
619+
w.WriteHeader(http.StatusOK)
620+
}))
621+
defer wrongModel.Close()
622+
623+
disc := NewDiscovery()
624+
disc.AddManual(nodeForModel(t, "wrong", wrongModel.URL, "mistral"))
625+
disc.AddManual(nodeForModel(t, "good", good.URL, "claude-3-5-sonnet-20241022"))
626+
p := testProxy(disc, 11434)
627+
p.SetSelected("wrong")
628+
629+
body := `{"model":"claude-3-5-sonnet-20241022","max_tokens":1024,"messages":[{"role":"user","content":"Hello"}]}`
630+
rec := httptest.NewRecorder()
631+
p.handleHTTP(rec, httptest.NewRequest(http.MethodPost, "/v1/messages", strings.NewReader(body)))
632+
633+
if rec.Code != http.StatusOK {
634+
t.Fatalf("status = %d, want 200", rec.Code)
635+
}
636+
if gotBody != body {
637+
t.Errorf("node got body %q, want the original request body", gotBody)
638+
}
639+
if gotPath != "/v1/messages" {
640+
t.Errorf("path = %q, want /v1/messages", gotPath)
641+
}
642+
}
643+
644+
// TestHandleHTTP_AnthropicMessages_404Failover proves /v1/messages is treated
645+
// as an inference endpoint for 404 failover: when the first eligible candidate
646+
// returns 404, the request retries to the second candidate and the body is
647+
// preserved.
648+
func TestHandleHTTP_AnthropicMessages_404Failover(t *testing.T) {
649+
missing := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
650+
w.WriteHeader(http.StatusNotFound)
651+
_, _ = io.WriteString(w, `{"error":{"type":"not_found_error","message":"model not found"}}`)
652+
}))
653+
defer missing.Close()
654+
655+
var gotBody string
656+
has := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
657+
b, _ := io.ReadAll(r.Body)
658+
gotBody = string(b)
659+
w.WriteHeader(http.StatusOK)
660+
_, _ = io.WriteString(w, `{"id":"msg_test","type":"message","role":"assistant","content":[{"type":"text","text":"Hello!"}]}`)
661+
}))
662+
defer has.Close()
663+
664+
disc := NewDiscovery()
665+
disc.AddManual(nodeForModel(t, "missing", missing.URL, "claude-3-5-sonnet-20241022"))
666+
disc.AddManual(nodeForModel(t, "has", has.URL, "claude-3-5-sonnet-20241022"))
667+
p := testProxy(disc, 11434)
668+
p.SetSelected("missing")
669+
670+
body := `{"model":"claude-3-5-sonnet-20241022","max_tokens":1024,"messages":[{"role":"user","content":"Hello"}]}`
671+
rec := httptest.NewRecorder()
672+
p.handleHTTP(rec, httptest.NewRequest(http.MethodPost, "/v1/messages", strings.NewReader(body)))
673+
674+
if rec.Code != http.StatusOK {
675+
t.Fatalf("status = %d, want 200 (should fail over past the 404)", rec.Code)
676+
}
677+
if gotBody != body {
678+
t.Errorf("failover node got body %q, want the original request body", gotBody)
679+
}
680+
}

‎services/lmstudio-proxy/proxy.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -146,12 +146,13 @@ const (
146146
// inferenceEndpoints is the set of request paths that count as cluster
147147
// workloads. Health checks, model listings (/v1/models), and other control
148148
// traffic are deliberately excluded so we don't flood the cluster with
149-
// non-inference noise. LM Studio serves the OpenAI-compatible API, so these
150-
// are the OpenAI inference routes.
149+
// non-inference noise. LM Studio serves the OpenAI-compatible API and the
150+
// Anthropic Messages API, so both inference route families are included.
151151
var inferenceEndpoints = map[string]bool{
152152
"/v1/chat/completions": true,
153153
"/v1/completions": true,
154154
"/v1/embeddings": true,
155+
"/v1/messages": true,
155156
}
156157

157158
// isInferenceRequest reports whether a request should be tracked as a

‎services/ollama-proxy/failover_test.go‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -642,3 +642,83 @@ func TestResolveCandidates_SelfGuard(t *testing.T) {
642642
t.Errorf("expected the real node to survive the self-guard, candidates = %+v", cands)
643643
}
644644
}
645+
646+
// TestHandleHTTP_AnthropicMessages_InferenceRouting proves /v1/messages is
647+
// treated as an inference endpoint: model-based candidate filtering is applied
648+
// and the request body is forwarded unchanged to the matching node.
649+
func TestHandleHTTP_AnthropicMessages_InferenceRouting(t *testing.T) {
650+
var gotBody string
651+
var gotPath string
652+
good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
653+
b, _ := io.ReadAll(r.Body)
654+
gotBody = string(b)
655+
gotPath = r.URL.Path
656+
w.WriteHeader(http.StatusOK)
657+
_, _ = io.WriteString(w, `{"id":"msg_test","type":"message","role":"assistant","content":[{"type":"text","text":"Hello!"}]}`)
658+
}))
659+
defer good.Close()
660+
661+
wrongModel := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
662+
t.Error("wrong-model node should not receive request")
663+
w.WriteHeader(http.StatusOK)
664+
}))
665+
defer wrongModel.Close()
666+
667+
disc := NewDiscovery()
668+
disc.AddManual(nodeForModel(t, "wrong", wrongModel.URL, "mistral"))
669+
disc.AddManual(nodeForModel(t, "good", good.URL, "claude-3-5-sonnet-20241022"))
670+
p := testProxy(disc, 11434)
671+
p.SetSelected("wrong")
672+
673+
body := `{"model":"claude-3-5-sonnet-20241022","max_tokens":1024,"messages":[{"role":"user","content":"Hello"}]}`
674+
rec := httptest.NewRecorder()
675+
p.handleHTTP(rec, httptest.NewRequest(http.MethodPost, "/v1/messages", strings.NewReader(body)))
676+
677+
if rec.Code != http.StatusOK {
678+
t.Fatalf("status = %d, want 200", rec.Code)
679+
}
680+
if gotBody != body {
681+
t.Errorf("node got body %q, want the original request body", gotBody)
682+
}
683+
if gotPath != "/v1/messages" {
684+
t.Errorf("path = %q, want /v1/messages", gotPath)
685+
}
686+
}
687+
688+
// TestHandleHTTP_AnthropicMessages_404Failover proves /v1/messages is treated
689+
// as an inference endpoint for 404 failover: when the first eligible candidate
690+
// returns 404, the request retries to the second candidate and the body is
691+
// preserved.
692+
func TestHandleHTTP_AnthropicMessages_404Failover(t *testing.T) {
693+
missing := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
694+
w.WriteHeader(http.StatusNotFound)
695+
_, _ = io.WriteString(w, `{"error":{"type":"not_found_error","message":"model not found"}}`)
696+
}))
697+
defer missing.Close()
698+
699+
var gotBody string
700+
has := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
701+
b, _ := io.ReadAll(r.Body)
702+
gotBody = string(b)
703+
w.WriteHeader(http.StatusOK)
704+
_, _ = io.WriteString(w, `{"id":"msg_test","type":"message","role":"assistant","content":[{"type":"text","text":"Hello!"}]}`)
705+
}))
706+
defer has.Close()
707+
708+
disc := NewDiscovery()
709+
disc.AddManual(nodeForModel(t, "missing", missing.URL, "claude-3-5-sonnet-20241022"))
710+
disc.AddManual(nodeForModel(t, "has", has.URL, "claude-3-5-sonnet-20241022"))
711+
p := testProxy(disc, 11434)
712+
p.SetSelected("missing")
713+
714+
body := `{"model":"claude-3-5-sonnet-20241022","max_tokens":1024,"messages":[{"role":"user","content":"Hello"}]}`
715+
rec := httptest.NewRecorder()
716+
p.handleHTTP(rec, httptest.NewRequest(http.MethodPost, "/v1/messages", strings.NewReader(body)))
717+
718+
if rec.Code != http.StatusOK {
719+
t.Fatalf("status = %d, want 200 (should fail over past the 404)", rec.Code)
720+
}
721+
if gotBody != body {
722+
t.Errorf("failover node got body %q, want the original request body", gotBody)
723+
}
724+
}

‎services/ollama-proxy/proxy.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -147,8 +147,8 @@ const (
147147
// inferenceEndpoints is the set of request paths that count as cluster
148148
// workloads. Health checks, model listings (/api/tags), and other control
149149
// traffic are deliberately excluded so we don't flood the cluster with
150-
// non-inference noise. Both the native Ollama and OpenAI-compatible
151-
// inference routes are included.
150+
// non-inference noise. Both the native Ollama, OpenAI-compatible, and
151+
// Anthropic Messages API inference routes are included.
152152
var inferenceEndpoints = map[string]bool{
153153
"/api/generate": true,
154154
"/api/chat": true,
@@ -157,6 +157,7 @@ var inferenceEndpoints = map[string]bool{
157157
"/v1/chat/completions": true,
158158
"/v1/completions": true,
159159
"/v1/embeddings": true,
160+
"/v1/messages": true,
160161
}
161162

162163
// isInferenceRequest reports whether a request should be tracked as a

‎services/versions.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@
33
"product": "0.91.7",
44
"installer": "0.91.7",
55
"components": {
6-
"ollama-proxy": "0.26.2",
7-
"lmstudio-proxy": "0.16.2",
6+
"ollama-proxy": "0.27.0",
7+
"lmstudio-proxy": "0.17.0",
88
"nvpair-node-info": "0.13.3",
99
"nvpair-node-scanner": "0.20.3",
1010
"nvpair-manual-nodes": "0.11.1",

0 commit comments

Comments
 (0)