diff --git a/README.md b/README.md index 757c3f39..7ffc8a2d 100644 --- a/README.md +++ b/README.md @@ -10,10 +10,10 @@ SPDX-License-Identifier: Apache-2.0 NVIDIA Personal AI Router (PAIR) is a local inference router for a group of compatible computers on the same network. It discovers participating nodes, -manages supported inference engines, and presents Ollama-compatible and -OpenAI-compatible proxy endpoints to applications and agents. Independent -requests can be routed to eligible nodes according to engine availability, -model availability, and current workload. +manages supported inference engines, and presents local proxy endpoints for +Ollama-compatible, OpenAI-compatible, and Anthropic Messages API requests. +Independent requests can be routed to eligible nodes according to engine +availability, model availability, and current workload. PAIR is useful for concurrent local workloads such as multi-agent applications. Prompts and responses are intended to remain on the local network when every diff --git a/docs/getting-started.mdx b/docs/getting-started.mdx index cb7e7e34..b17af1bb 100644 --- a/docs/getting-started.mdx +++ b/docs/getting-started.mdx @@ -406,8 +406,8 @@ What that base URL serves depends on the engine behind it: | Endpoint | Default base URL | Paths it serves | | --- | --- | --- | -| Ollama | `http://127.0.0.1:11434` | Ollama's own API — `/api/chat`, `/api/generate`, `/api/embed`, `/api/tags` — and the OpenAI-compatible `/v1/chat/completions`, `/v1/completions`, `/v1/embeddings`, `/v1/models` | -| LM Studio | `http://127.0.0.1:1234` | The OpenAI-compatible paths only | +| Ollama | `http://127.0.0.1:11434` | Ollama's own API — `/api/chat`, `/api/generate`, `/api/embed`, `/api/tags` — and the OpenAI-compatible `/v1/chat/completions`, `/v1/completions`, `/v1/embeddings`, `/v1/models`, plus the Anthropic Messages API `/v1/messages` | +| LM Studio | `http://127.0.0.1:1234` | The OpenAI-compatible paths (`/v1/chat/completions`, `/v1/completions`, `/v1/embeddings`, `/v1/models`) and the Anthropic Messages API `/v1/messages` | The distinction matters when you fill in an application's settings. A client written against the OpenAI API usually wants the `/v1` included, as in diff --git a/docs/overview.mdx b/docs/overview.mdx index fb519d50..0efbb5ab 100644 --- a/docs/overview.mdx +++ b/docs/overview.mdx @@ -76,14 +76,14 @@ you form a cluster, only the last step repeats. ## Request Model -An application sends an ordinary Ollama-compatible or OpenAI-compatible HTTP -request to a local proxy. The proxy selects one eligible node and forwards the -whole request. That node's engine performs the inference, and the response -streams back through the proxy. +An application sends an ordinary Ollama-compatible or OpenAI-compatible +request, or an Anthropic Messages API request, to a local proxy. The proxy +selects one eligible node and forwards the whole request. That node's engine +performs the inference, and the response streams back through the proxy. ```mermaid flowchart LR - Client["Your AI app or agent"] -->|"Ollama- or OpenAI-compatible request"| Endpoint["PAIR endpoint
on your machine"] + Client["Your AI app or agent"] -->|"Ollama, OpenAI, or Anthropic Messages request"| Endpoint["PAIR endpoint
on your machine"] Endpoint <-->|"encrypted both ways"| Node["A paired node
with the model"] Node --> Engine["Inference engine"] ``` @@ -136,7 +136,8 @@ PAIR provides these capabilities: - A local endpoint for compatible AI applications and development tools. - LAN discovery plus manually configured nodes. -- Ollama-compatible and LM Studio/OpenAI-compatible routing proxies. +- Routing proxies for Ollama-compatible, OpenAI-compatible, and Anthropic + Messages API requests. - Pairing and cluster membership managed by the background services. - Model-aware, workload-informed routing of independent requests. - Encrypted routing between machines: a request sent to another node travels over diff --git a/services/nvpair-proxy/README.md b/services/nvpair-proxy/README.md index c36fb0c2..20c753e1 100644 --- a/services/nvpair-proxy/README.md +++ b/services/nvpair-proxy/README.md @@ -102,7 +102,7 @@ and the health crash key are matched against each other, so they move together | Standalone port, used when `port` is omitted | 11435 | 1234 | | Persisted-port file (declared, not derived) | `proxy-port.json` | `lmstudio-proxy-port.json` | | Model-list routes | `GET /api/tags` (native), `GET /v1/models` (OpenAI) | `GET /v1/models` (OpenAI) | -| Inference routes | `/api/generate`, `/api/chat`, `/api/embeddings`, `/api/embed`, plus the OpenAI set | `/v1/chat/completions`, `/v1/completions`, `/v1/embeddings` | +| Inference routes | `/api/generate`, `/api/chat`, `/api/embeddings`, `/api/embed`, plus the OpenAI and Anthropic Messages sets | `/v1/chat/completions`, `/v1/completions`, `/v1/embeddings`, `/v1/messages` | | Model naming | untagged means `:latest`, so `llama3` and `llama3:latest` are one model | identifiers compared byte for byte | The route table is a **classifier, not an allowlist**. An unlisted path is diff --git a/services/nvpair-proxy/engines.go b/services/nvpair-proxy/engines.go index 06d1a54f..71201a30 100644 --- a/services/nvpair-proxy/engines.go +++ b/services/nvpair-proxy/engines.go @@ -20,6 +20,7 @@ package main // nvpair-shared/engines. import ( + "slices" "strings" "nvpair-shared/engines" @@ -30,8 +31,8 @@ import ( // POST to /v1/models is not a model list, and a GET to /api/chat is not // inference. Folding them into one constant keeps the two from disagreeing. // -// The dialect distinction is meaningful only for model-list roles. All seven -// of Ollama's inference paths are handled identically — no envelope, identity +// The dialect distinction is meaningful only for model-list roles. All of +// Ollama's inference paths are handled identically — no envelope, identity // field or response shape is selected — so there is deliberately no // per-dialect inference role. type routeRole int @@ -116,32 +117,48 @@ type engineProfile struct { SupportsHostAlias bool } -// openAIInferenceRoutes is the inference surface every OpenAI-compatible -// engine exposes. Ollama serves these alongside its native routes. +// ollamaBaseRoutes is the engine-specific surface that Ollama exposes before +// the shared compatibility routes are added. +var ollamaBaseRoutes = []route{ + {Path: "/api/generate", Role: roleInferencePOST}, + {Path: "/api/chat", Role: roleInferencePOST}, + {Path: "/api/embeddings", Role: roleInferencePOST}, + {Path: "/api/embed", Role: roleInferencePOST}, + {Path: "/api/tags", Role: roleModelListNativeGET}, + {Path: "/v1/models", Role: roleModelListOpenAIGET}, +} + +// lmStudioBaseRoutes is the engine-specific surface that LM Studio exposes +// before the shared compatibility routes are added. +var lmStudioBaseRoutes = []route{ + {Path: "/v1/models", Role: roleModelListOpenAIGET}, +} + +// openAIInferenceRoutes is the OpenAI-compatible inference surface. var openAIInferenceRoutes = []route{ {Path: "/v1/chat/completions", Role: roleInferencePOST}, {Path: "/v1/completions", Role: roleInferencePOST}, {Path: "/v1/embeddings", Role: roleInferencePOST}, } +// anthropicInferenceRoutes is the Anthropic-compatible inference surface. +var anthropicInferenceRoutes = []route{ + {Path: "/v1/messages", Role: roleInferencePOST}, +} + var profiles = buildProfiles() func buildProfiles() []engineProfile { ollama, _ := engines.ByName("ollama") lmstudio, _ := engines.ByName("lmstudio") + ollamaRoutes := slices.Concat(ollamaBaseRoutes, openAIInferenceRoutes, anthropicInferenceRoutes) + lmStudioRoutes := slices.Concat(lmStudioBaseRoutes, openAIInferenceRoutes, anthropicInferenceRoutes) return []engineProfile{ { - Engine: ollama, - StandalonePort: 11435, - Routes: append([]route{ - {Path: "/api/generate", Role: roleInferencePOST}, - {Path: "/api/chat", Role: roleInferencePOST}, - {Path: "/api/embeddings", Role: roleInferencePOST}, - {Path: "/api/embed", Role: roleInferencePOST}, - {Path: "/api/tags", Role: roleModelListNativeGET}, - {Path: "/v1/models", Role: roleModelListOpenAIGET}, - }, openAIInferenceRoutes...), + Engine: ollama, + StandalonePort: 11435, + Routes: ollamaRoutes, ModelNaming: impliedLatestTag, ReservedPersistedPort: 0, SupportsHostAlias: true, @@ -149,10 +166,8 @@ func buildProfiles() []engineProfile { { Engine: lmstudio, StandalonePort: 1234, - Routes: append([]route{ - {Path: "/v1/models", Role: roleModelListOpenAIGET}, - }, openAIInferenceRoutes...), - ModelNaming: exactID, + Routes: lmStudioRoutes, + ModelNaming: exactID, // 1235 is where engine-manager runs a managed LM Studio, so a // proxy that restored it would sit on the engine's own port. The // stored value predates the current default of 1234. diff --git a/services/nvpair-proxy/engines_test.go b/services/nvpair-proxy/engines_test.go index 4b149e3f..422159c7 100644 --- a/services/nvpair-proxy/engines_test.go +++ b/services/nvpair-proxy/engines_test.go @@ -29,12 +29,14 @@ func TestRoleForClassifiesOnlyDeclaredRoutes(t *testing.T) { }{ {"ollama native chat", ollama, "POST", "/api/chat", roleInferencePOST, true}, {"ollama openai chat", ollama, "POST", "/v1/chat/completions", roleInferencePOST, true}, + {"ollama anthropic messages", ollama, "POST", "/v1/messages", roleInferencePOST, true}, {"ollama native list", ollama, "GET", "/api/tags", roleModelListNativeGET, true}, {"ollama openai list", ollama, "GET", "/v1/models", roleModelListOpenAIGET, true}, {"ollama passthrough", ollama, "POST", "/api/pull", 0, false}, {"ollama version passthrough", ollama, "GET", "/api/version", 0, false}, {"lmstudio chat", lmstudio, "POST", "/v1/chat/completions", roleInferencePOST, true}, + {"lmstudio anthropic messages", lmstudio, "POST", "/v1/messages", roleInferencePOST, true}, {"lmstudio list", lmstudio, "GET", "/v1/models", roleModelListOpenAIGET, true}, // LM Studio serves no native Ollama routes, so /api/chat is not // inference for it — it is forwarded verbatim like any other path. diff --git a/services/nvpair-proxy/failover_test.go b/services/nvpair-proxy/failover_test.go index e84429ba..0d54b6fe 100644 --- a/services/nvpair-proxy/failover_test.go +++ b/services/nvpair-proxy/failover_test.go @@ -641,6 +641,87 @@ func TestHandleHTTP_StrictModelRouting(t *testing.T) { }) } +// TestHandleHTTP_InferenceRouting proves each engine inference route is +// model-routed, retries a model-not-found response, and +// forwards the request path and body unchanged. +func TestHandleHTTP_InferenceRouting(t *testing.T) { + test := func(name, path string, profiles ...engineProfile) { + t.Run(name, func(t *testing.T) { + for _, profile := range profiles { + t.Run(profile.Name, func(t *testing.T) { + requestedModel := "requested-model" + advertisedModel := profile.normalizeModel(requestedModel) + + wrongModel := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + t.Error("wrong-model node should not receive request") + w.WriteHeader(http.StatusOK) + })) + defer wrongModel.Close() + + missing := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + if _, err := io.WriteString(w, `{"error":"model not found"}`); err != nil { + t.Errorf("write missing-model response: %v", err) + } + })) + defer missing.Close() + + var gotBody string + var gotPath string + good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + t.Errorf("read forwarded request body: %v", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + gotBody = string(body) + gotPath = r.URL.Path + w.WriteHeader(http.StatusOK) + })) + defer good.Close() + + disc := NewDiscovery() + disc.AddManual(nodeForModel(t, "wrong", wrongModel.URL, "different-model")) + disc.AddManual(nodeForModel(t, "missing", missing.URL, advertisedModel)) + disc.AddManual(nodeForModel(t, "good", good.URL, advertisedModel)) + p := testProxy(profile, disc, profile.FacadePort) + p.soleFacade().SetSelected("wrong") + // Stable node ordering would send this request to "good" first and + // never exercise 404 failover. Prioritize "missing" so the test + // independently proves both model filtering and retry behavior. + p.SetPriority([]string{"missing", "good"}) + + body := `{"model":"requested-model","max_tokens":1024,"messages":[{"role":"user","content":"Hello"}]}` + rec := httptest.NewRecorder() + p.soleFacade().handleHTTP(rec, httptest.NewRequest(http.MethodPost, path, strings.NewReader(body))) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200 after 404 failover", rec.Code) + } + if gotBody != body { + t.Errorf("node got body %q, want %q", gotBody, body) + } + if gotPath != path { + t.Errorf("path = %q, want %q", gotPath, path) + } + }) + } + }) + } + + ollama := ollamaCase(t).profile + lmstudio := lmstudioCase(t).profile + test("native Ollama generate", "/api/generate", ollama) + test("native Ollama chat", "/api/chat", ollama) + test("native Ollama embeddings", "/api/embeddings", ollama) + test("native Ollama embed", "/api/embed", ollama) + test("OpenAI chat completions", "/v1/chat/completions", ollama, lmstudio) + test("OpenAI completions", "/v1/completions", ollama, lmstudio) + test("OpenAI embeddings", "/v1/embeddings", ollama, lmstudio) + test("Anthropic messages", "/v1/messages", ollama, lmstudio) +} + func TestHandleHTTP_NoAdvertisedModelRejectsLocally(t *testing.T) { forEachEngine(t, func(t *testing.T, tc engineCase) { hits := 0 diff --git a/services/tests/model_routing_interop_test.go b/services/tests/model_routing_interop_test.go index 8c81a276..58ef22ea 100644 --- a/services/tests/model_routing_interop_test.go +++ b/services/tests/model_routing_interop_test.go @@ -70,7 +70,9 @@ func TestStrictModelRoutingAcrossProcesses(t *testing.T) { } cases := []proxyCase{ {name: "ollama", rpcPrefix: "ollama-proxy", path: "/api/chat", port: ollamaPort}, + {name: "ollama-anthropic", rpcPrefix: "ollama-proxy", path: "/v1/messages", port: ollamaPort}, {name: "lmstudio", rpcPrefix: "lmstudio-proxy", path: "/v1/chat/completions", port: lmstudioPort}, + {name: "lmstudio-anthropic", rpcPrefix: "lmstudio-proxy", path: "/v1/messages", port: lmstudioPort}, } client := &http.Client{Timeout: 5 * time.Second} t.Cleanup(client.CloseIdleConnections) @@ -111,7 +113,9 @@ func TestStrictModelRoutingAcrossProcesses(t *testing.T) { requestID++ } callBrokerRPC(t, stdin, msgs, requestID, tc.rpcPrefix+":node/set-priority", map[string]any{ - "generation": 1, + // Both facades share one proxy process, so every snapshot must + // advance the process-wide generation. + "generation": caseIndex + 1, "nodes": []string{missingID, unknownID, owner404ID, ownerOKID}, })