diff --git a/.github/workflows/buf.yml b/.github/workflows/buf.yml index e2651a9..08e9c52 100644 --- a/.github/workflows/buf.yml +++ b/.github/workflows/buf.yml @@ -18,21 +18,19 @@ permissions: contents: read jobs: - lint-and-breaking: + lint: runs-on: ubuntu-latest steps: - name: Check out repository uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Lint schema and check compatibility + - name: Lint schema uses: bufbuild/buf-action@v1 with: version: "1.71.0" lint: true format: false - breaking: ${{ github.event_name == 'pull_request' }} + breaking: false push: false archive: false pr_comment: false diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 703e857..811dd88 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -20,11 +20,11 @@ limitations under the License. Thank you for your interest in OpenEngine. We welcome bug reports, feedback on the API design, and pull requests. -OpenEngine is a pre-adoption API draft. `proto/openengine.proto` is the canonical -wire contract and the single source of truth. Until an external consumer adopts -it, the schema may remove or renumber fields to stay minimal. After external -adoption, changes within `openengine.v1` will be additive. Please open an issue -to discuss protocol changes before sending a PR. +OpenEngine is a pre-adoption API draft. `proto/openengine/v1/openengine.proto` is +the canonical wire contract and the single source of truth. Until an external +consumer adopts it, the schema may remove or renumber fields to stay minimal. +After external adoption, changes within `openengine.v1` will be additive. Please +open an issue to discuss protocol changes before sending a PR. - **Bugs / feedback / design questions**: open a [GitHub issue](https://github.com/ai-dynamo/openengine/issues). - **Pull requests**: open against `main`. Keep changes focused (one logical change per PR). diff --git a/README.md b/README.md index aeb79ca..e8d7ea6 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ SPDX-License-Identifier: Apache-2.0 Buf CI Apache 2.0 License Status: Experimental - API: openengine.v1 + API: openengine.v1 Transport: gRPC Schema: Protocol Buffers

@@ -29,7 +29,7 @@ SPDX-License-Identifier: Apache-2.0

Why OpenEngine? · API reference - · Canonical proto + · Canonical proto · Contributing

@@ -101,7 +101,7 @@ exist alongside OpenEngine. ## Capabilities The complete service is defined in -[`proto/openengine.proto`](proto/openengine.proto). +[`proto/openengine/v1/openengine.proto`](proto/openengine/v1/openengine.proto). | Area | What the contract provides | | --------------------- | ----------------------------------------------------------------------------------------------------------------- | @@ -130,11 +130,9 @@ cd openengine buf build buf lint -buf breaking --against '.git#branch=main,subdir=proto' ``` -The same lint and breaking checks run in GitHub Actions for schema pull -requests. +Buf lint runs in GitHub Actions for schema pull requests. ### Generate Python bindings @@ -156,7 +154,7 @@ python -m grpc_tools.protoc \ -I "$PROTO_INCLUDE" \ --python_out="$OUT_DIR" \ --grpc_python_out="$OUT_DIR" \ - proto/openengine.proto + proto/openengine/v1/openengine.proto ``` Other protobuf-supported languages can generate clients and servers from the @@ -190,7 +188,7 @@ git commit --signoff -m "docs: describe the change" ``` Please validate protobuf changes with Buf and keep -[`proto/openengine.proto`](proto/openengine.proto) and +[`proto/openengine/v1/openengine.proto`](proto/openengine/v1/openengine.proto) and [`docs/api.md`](docs/api.md) synchronized. ## Security diff --git a/buf.yaml b/buf.yaml index ac68faf..592b8f1 100644 --- a/buf.yaml +++ b/buf.yaml @@ -7,14 +7,8 @@ lint: use: - STANDARD except: - # The canonical v1 contract intentionally remains at proto/openengine.proto. - - PACKAGE_DIRECTORY_MATCH # OpenEngine is the public protocol name; appending Service would obscure it. - SERVICE_SUFFIX # Discovery and event RPCs intentionally return their domain message type # directly instead of adding one-field response wrappers. - RPC_RESPONSE_STANDARD_NAME - -breaking: - use: - - FILE diff --git a/docs/api.md b/docs/api.md index ffc6577..2d4a20b 100644 --- a/docs/api.md +++ b/docs/api.md @@ -5,7 +5,8 @@ SPDX-License-Identifier: Apache-2.0 # OpenEngine API v1 -This is the human-readable reference for [`openengine.v1`](../proto/openengine.proto). +This is the human-readable reference for +[`openengine.v1`](../proto/openengine/v1/openengine.proto). The proto is the source of truth. --- @@ -103,7 +104,7 @@ name remains `openengine.v1`. Discovery response scalars use proto3 `optional` presence. An absent value means the engine cannot report the value; an explicitly present zero or `false` is a -reported value and must not be replaced with an orchestrator default. +reported value and must not be replaced with a client default. Role semantics: @@ -332,9 +333,8 @@ enum Modality { } // A single multimodal input. Exactly one `source` should be set. The engine -// owns fetch/decode/preprocess -- in the sidecar deployment the orchestrator -// has no GPU/NIXL agent, so a pre-decoded/RDMA media descriptor is NOT -// representable here by design. +// owns fetch, decode, and preprocessing, so pre-decoded or RDMA media +// descriptors are not represented here. message MediaItem { Modality modality = 1; oneof source { @@ -522,7 +522,7 @@ dedicated field. Prefill flow: -1. Orchestrator sends `GenerateRequest` to a `PREFILL` engine. +1. Client sends `GenerateRequest` to a `PREFILL` engine. 2. Engine returns a `KvSessionRef` in the terminal `PrefillReady` response when decode may attach. 3. Engine owns KV session lifetime and cleanup, including finish, abort, drain, timeout, and transfer failure paths. @@ -530,7 +530,7 @@ Prefill flow: Decode flow: -1. Orchestrator sends `GenerateRequest` to a `DECODE` engine with `kv.session` set. +1. Client sends `GenerateRequest` to a `DECODE` engine with `kv.session` set. 2. Decode engine validates the session and transfer backend. 3. Decode engine generates tokens. @@ -659,7 +659,8 @@ Compatibility notes: - SGLang/vLLM-style `BlockStored`, `BlockRemoved`, and `AllBlocksCleared` are first-class OpenEngine events. - OpenEngine preserves batch timestamp, DP-rank attribution, monotonic sequence numbers, replay start sequence, topic, endpoint, replay endpoint, buffer size, HWM, and queue-size metadata. - Native OpenEngine streams should use protobuf. Existing ZMQ/msgpack publishers can be exposed through `GetKvEventSources` during migration. -- Orchestrators should prefer `SubscribeKvEvents` when available and fall back to engine-native sources when advertised. +- Clients should prefer `SubscribeKvEvents` when available and fall back to + engine-native sources when advertised. - `endpoint_addr` MUST carry a routable `host:port`, never a bind wildcard such as `*` or `0.0.0.0`. diff --git a/docs/motivation.md b/docs/motivation.md index 53c1152..4e59d2f 100644 --- a/docs/motivation.md +++ b/docs/motivation.md @@ -5,9 +5,10 @@ SPDX-License-Identifier: Apache-2.0 # Why OpenEngine -OpenEngine defines the runtime boundary between an inference engine and an -orchestrator. It lets each side change without sharing a process, dependency -tree, or private control API. +OpenEngine defines a runtime boundary around an inference engine. Applications +can call it directly, and distributed frameworks can use the same contract to +coordinate engine workers. The client and engine can change without sharing a +process, dependency tree, or private control API. ## The integration problem @@ -20,7 +21,8 @@ Inference engines already own request execution: - guided decoding, LoRA, and logprobs; - engine-specific performance work. -Orchestrators own a different set of concerns: +Distributed frameworks add a different set of concerns when coordinating +workers: - discovery and routing; - admission and load balancing; @@ -28,22 +30,27 @@ Orchestrators own a different set of concerns: - health, drain, and cancellation policy; - KV-aware scheduling across workers. -Without a shared boundary, every engine-orchestrator pair needs a separate -adapter. These adapters often import the engine, copy its launch flags, and -depend on scheduler details. Engine upgrades then become orchestrator upgrades. +Without a shared boundary, direct users need engine-specific clients and every +engine-framework pair needs a separate adapter. These integrations often import +the engine, copy its launch flags, or depend on scheduler details. Engine +upgrades then force changes in client or framework code. ## The boundary ```mermaid flowchart LR - C["Client API"] --> O["Orchestrator
routing and lifecycle"] - O --> S["OpenEngine client
endpoint-only sidecar"] + D["Direct OpenEngine client
application · SDK · tooling"] + F["Distributed framework
routing · admission · placement"] + D -->|"openengine.v1 · gRPC"| S["OpenEngine server
engine adapter"] + F -->|"openengine.v1 · gRPC"| S S --> E["Native engine
scheduler and GPU execution"] ``` -The engine implements the OpenEngine server. The orchestrator implements the -client. The sidecar discovers the model, role, topology, limits, and supported -features from the engine instead of duplicating engine configuration. +The engine implements the OpenEngine server. Direct applications and +distributed frameworks use generated clients. A direct client can use +generation and control APIs without a framework; a distributed framework can +also discover model, role, topology, limits, and supported features instead of +duplicating engine configuration. The engine remains the authority for request execution. OpenEngine carries the request and control data needed across the process boundary. @@ -60,7 +67,7 @@ request and control data needed across the process boundary. | KV routing | Event streams and native event-source discovery | The [API reference](api.md) defines the fields. The -[proto](../proto/openengine.proto) is the source of truth. +[proto](../proto/openengine/v1/openengine.proto) is the source of truth. ## What stays engine-specific @@ -74,7 +81,7 @@ OpenEngine does not define: - native HTTP or gRPC APIs. An engine maps OpenEngine messages to its existing request path. It can keep -native APIs for direct clients and expose OpenEngine for orchestrators. +native APIs and expose OpenEngine to direct clients and distributed frameworks. ## OpenEngine and OpenAI-compatible APIs @@ -83,18 +90,20 @@ The two APIs serve different callers. - An OpenAI-compatible API is a client-facing product API. It accepts chat or completion requests and hides deployment topology. - OpenEngine is a runtime protocol. It exposes engine role, load, lifecycle, - KV handoff, rank affinity, and event sources to an orchestrator. + KV handoff, rank affinity, and event sources to its clients. -OpenEngine does not ask users to replace their client API. An orchestrator may -accept OpenAI-compatible traffic, normalize it, and use OpenEngine between its -router and engine workers. +OpenEngine can be called directly by engine users, applications, and tooling. A +distributed framework may instead accept OpenAI-compatible traffic, normalize +it, and use OpenEngine between its router and engine workers. Engines can +continue exposing their existing client APIs alongside OpenEngine. ## Why an engine would implement it -- One server contract can support multiple orchestrators. +- One server contract can support direct clients and multiple distributed + frameworks. - The engine keeps its launch path, dependencies, and scheduler. -- The orchestrator can run in a small CPU-side process. -- Engine and orchestrator releases can move independently within the protocol's +- Framework-side clients can run in small CPU-only processes. +- Engine and client releases can move independently within the protocol's compatibility rules. - Disaggregated serving uses a common handoff shape without requiring a common transfer backend. @@ -114,20 +123,18 @@ integrations. An implementation can add support in stages: 1. Aggregated text generation, discovery, health, abort, and drain. -2. Logprobs, guided decoding, LoRA, and multimodal input as needed. -3. Prefill/decode roles, KV handoff, rank affinity, and KV events. +2. Prefill/decode roles, KV handoff, rank affinity, and KV events. +3. Logprobs, guided decoding, LoRA, and multimodal input as needed. -Capability fields let the orchestrator reject unsupported requests or choose a -compatible worker. +Capability fields let clients reject unsupported requests or select compatible +workers. -## Existing implementations +## Implementation status -The current vLLM and SGLang integrations use the same OpenEngine contract and -different engine adapters: +OpenEngine is pre-adoption and has no production engine implementations today. +The contract is designed for engines such as vLLM, SGLang, and TensorRT-LLM and +for both direct clients and distributed frameworks such as Dynamo. -- vLLM exposes a Rust gRPC service backed by its text-generation path. -- SGLang exposes an opt-in OpenEngine serve mode bridged to its scheduler. -- Dynamo uses endpoint-configured sidecars and discovers engine state over RPC. - -The shared contract is the common part. Scheduling, request conversion, and KV -transfer remain engine-owned. +These are intended integration targets, not claims that those projects +currently implement OpenEngine. Scheduling, request conversion, and KV transfer +remain engine-owned in any future adapter. diff --git a/proto/openengine.proto b/proto/openengine/v1/openengine.proto similarity index 95% rename from proto/openengine.proto rename to proto/openengine/v1/openengine.proto index 468de4e..3ccb5d0 100644 --- a/proto/openengine.proto +++ b/proto/openengine/v1/openengine.proto @@ -1,9 +1,9 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 // -// OpenEngine API v1 -- vendor-neutral runtime API between inference -// orchestration systems (e.g. Dynamo) and model engines (e.g. vLLM, SGLang, -// TensorRT-LLM). +// OpenEngine API v1 -- vendor-neutral runtime API between model engines (e.g. +// vLLM, SGLang, TensorRT-LLM) and clients, including direct applications and +// distributed frameworks (e.g. Dynamo). // // This file is the CANONICAL contract and single source of truth for the wire // shape. Implementations should generate client and server bindings from this @@ -106,11 +106,9 @@ message ModelInfo { optional bool supports_lora = 23; optional bool supports_multimodal = 24; - // Engine-advertised response parser names. The orchestrator's frontend applies - // these to a sidecar-served model's output stream (tool-call extraction, - // reasoning/thinking separation) — the engine streams tokens, the frontend - // parses. Empty = no parser configured for this model. Discovered from the - // engine so the sidecar stays endpoint-only (no parser flags on the worker). + // Engine-advertised response parser names. Clients can apply these to a + // model's output stream for tool-call extraction or reasoning separation. + // Empty means no parser is configured for this model. string reasoning_parser = 25; string tool_call_parser = 26; } @@ -234,9 +232,8 @@ enum Modality { } // A single multimodal input. Exactly one `source` should be set. The engine -// owns fetch/decode/preprocess -- the orchestrator never derefs media (it has -// no GPU/NIXL agent in the sidecar deployment), so a pre-decoded/RDMA media -// descriptor is NOT representable here by design. +// owns fetch, decode, and preprocessing, so pre-decoded or RDMA media +// descriptors are not represented here. message MediaItem { Modality modality = 1; oneof source { @@ -306,7 +303,7 @@ message StopCondition { // Constrained / guided decoding spec. At most one of `guide` should be set. // The engine enforces the constraint during sampling via its grammar backend -// (xgrammar / outlines / llguidance); the orchestrator cannot apply it post-hoc. +// (xgrammar / outlines / llguidance); clients cannot apply it post-hoc. message GuidedDecoding { oneof guide { string json_schema = 1; // output conforms to this JSON schema