diff --git a/README.md b/README.md index aadc6fc..5451c29 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,7 @@ The canonical schema is organized by domain under | Area | What the contract provides | | --------------------- | ----------------------------------------------------------------------------------------------------------------- | | Portable generation | Text or token input, sampling, stopping, priorities, multiple sequences, and deterministic seeds | +| Non-generative tasks | Typed embedding, classification, and grouped query/candidate scoring with stable correlation | | Structured output | JSON Schema, JSON object, regex, EBNF grammar, structural tags, and fixed choices | | Token information | Prompt and output logprobs, ranks, candidate-token selection, per-token records, and streamed text deltas | | Discovery | Engine identity, schema revision, role, model limits, topology, parser configuration, and generation capabilities | @@ -168,7 +169,7 @@ depend on it. Expect direct schema refinement during this phase. The intended adoption path is incremental: -1. Aggregated text generation, discovery, health, abort, and drain. +1. Aggregated generation and non-generative tasks, discovery, health, abort, and drain. 2. Prefill/decode roles, KV handoff, rank affinity, and KV event integration. 3. Logprobs, guided decoding, LoRA, and multimodal input as needed. diff --git a/docs/api.md b/docs/api.md index 8f5c3da..427441e 100644 --- a/docs/api.md +++ b/docs/api.md @@ -24,6 +24,11 @@ service OpenEngine { // Core inference path. rpc Generate(GenerateRequest) returns (stream GenerateResponse); + // Non-generative inference paths. + rpc Embed(EmbedRequest) returns (EmbedResponse); + rpc Classify(ClassifyRequest) returns (ClassifyResponse); + rpc Score(ScoreRequest) returns (ScoreResponse); + // Runtime metadata and scheduling state. rpc GetEngineInfo(GetEngineInfoRequest) returns (EngineInfo); rpc GetModelInfo(GetModelInfoRequest) returns (ModelInfo); @@ -143,6 +148,7 @@ message ModelInfo { string reasoning_parser = 25; string tool_call_parser = 26; + TaskCapabilities tasks = 27; } message GenerationCapabilities { @@ -198,6 +204,353 @@ the LoRA lifecycle RPCs on `OpenEngine`. --- +## Non-generative task API + +`Embed`, `Classify`, and `Score` are unary inference operations. They share a +request context and typed inputs, but retain task-specific options and outputs. +Support is optional per model and advertised through `ModelInfo.tasks`. + +```protobuf +message TaskRequestContext { + string request_id = 1; + string model = 2; + string lora_name = 3; + optional int32 priority = 4; + map metadata = 5; +} + +message TaskInput { + string item_id = 1; + oneof input { + string text = 2; + TokenIds token_ids = 3; + MultimodalTaskInput multimodal = 4; + } +} + +message MultimodalTaskInput { + oneof prompt { + string text = 1; + TokenIds token_ids = 2; + } + repeated MediaItem media = 3; +} + +message DenseFloatTensor { + repeated uint64 shape = 1; + repeated float values = 2; +} + +message SparseFloatTensor { + repeated uint64 shape = 1; + repeated uint64 indices = 2; + repeated float values = 3; +} + +message TaskUsage { + uint64 input_tokens = 1; + optional uint64 cached_input_tokens = 2; +} + +enum TaskInputType { + TASK_INPUT_TYPE_UNSPECIFIED = 0; + TASK_INPUT_TYPE_TEXT = 1; + TASK_INPUT_TYPE_TOKEN_IDS = 2; + TASK_INPUT_TYPE_MULTIMODAL = 3; +} + +enum TaskOutputGranularity { + TASK_OUTPUT_GRANULARITY_UNSPECIFIED = 0; + TASK_OUTPUT_GRANULARITY_SEQUENCE = 1; + TASK_OUTPUT_GRANULARITY_TOKEN = 2; +} + +enum EmbeddingEncoding { + EMBEDDING_ENCODING_UNSPECIFIED = 0; + EMBEDDING_ENCODING_DENSE = 1; + EMBEDDING_ENCODING_SPARSE = 2; +} + +enum TaskValueSemantics { + TASK_VALUE_SEMANTICS_UNSPECIFIED = 0; + TASK_VALUE_SEMANTICS_LOGITS = 1; + TASK_VALUE_SEMANTICS_PROBABILITIES = 2; + TASK_VALUE_SEMANTICS_LOG_PROBABILITIES = 3; + TASK_VALUE_SEMANTICS_SIMILARITY = 4; + TASK_VALUE_SEMANTICS_RELEVANCE = 5; + TASK_VALUE_SEMANTICS_REWARD = 6; + TASK_VALUE_SEMANTICS_MODEL_DEFINED = 7; +} + +enum ScoreNormalization { + SCORE_NORMALIZATION_UNSPECIFIED = 0; + SCORE_NORMALIZATION_NONE = 1; + SCORE_NORMALIZATION_SOFTMAX = 2; +} +``` + +`TaskRequestContext.request_id` and `model` are required and non-empty. Request +IDs share the same namespace and abort semantics as generation request IDs. +`priority` uses the generation ordering convention: larger values have higher +priority. A non-empty `lora_name` selects an already loaded adapter. Clients +must use either option only when the corresponding task capability advertises +support. + +Each request batch must be non-empty. `item_id` is required and unique within +an embed/classify batch, and every query/candidate item ID is unique within one +score group. Exactly one `TaskInput.input` variant is set. A multimodal input must +contain at least one `MediaItem`; its optional prompt is either text or token +IDs, never both. Media ordering and validation follow `GenerateRequest.media`. + +`DenseFloatTensor` is row-major FP32 data. Every dimension is greater than zero, +the product of `shape` equals `values.size()`, and every value is finite. +`SparseFloatTensor` uses flattened row-major indices; `indices` and `values` +have equal length, indices are unique and strictly increasing, and every index +is smaller than the product of `shape`. A scalar is encoded with shape `[1]`, +not an empty shape. `cached_input_tokens`, when present, does not exceed +`input_tokens`. + +### Embedding + +```protobuf +message EmbedRequest { + TaskRequestContext context = 1; + repeated TaskInput inputs = 2; + EmbedOptions options = 3; +} + +message EmbedOptions { + optional TaskOutputGranularity granularity = 1; + optional bool normalize = 2; + optional uint32 dimensions = 3; + optional EmbeddingEncoding encoding = 4; +} + +message EmbedResponse { + string request_id = 1; + repeated EmbeddingOutput outputs = 2; + TaskUsage usage = 3; +} + +message EmbeddingOutput { + string item_id = 1; + optional uint32 input_index = 2; + TaskOutputGranularity granularity = 3; + oneof embedding { + DenseFloatTensor dense = 4; + SparseFloatTensor sparse = 5; + } + repeated uint32 token_ids = 6; +} +``` + +Absent embedding options select model defaults. An explicit granularity, +normalization, or encoding request must be implemented exactly or rejected. +`dimensions` must be greater than zero and requests dimensionality reduction; +it does not permit padding a smaller model output. + +Sequence embeddings have shape `[dimension]`. Token embeddings have shape +`[token_count, dimension]`; `token_ids`, when returned, has `token_count` +entries aligned to the first tensor dimension. Outputs preserve request order, +and `input_index` is present even for index zero and identifies the matching +input. A successful response contains exactly one output for every request +input. + +### Classification + +```protobuf +message ClassifyRequest { + TaskRequestContext context = 1; + repeated TaskInput inputs = 2; + ClassifyOptions options = 3; +} + +message ClassifyOptions { + optional TaskOutputGranularity granularity = 1; + optional TaskValueSemantics output_semantics = 2; +} + +message ClassifyResponse { + string request_id = 1; + repeated ClassificationOutput outputs = 2; + TaskUsage usage = 3; +} + +message ClassificationOutput { + string item_id = 1; + optional uint32 input_index = 2; + TaskOutputGranularity granularity = 3; + TaskValueSemantics semantics = 4; + DenseFloatTensor scores = 5; + repeated string labels = 6; + repeated uint32 token_ids = 7; +} +``` + +Classification normally returns logits, probabilities, log probabilities, or +model-defined values. The engine reports the actual non-`UNSPECIFIED` +semantics. Sequence classification has shape `[class_count]`; token +classification has shape `[token_count, class_count]`. When labels are +returned, their count equals `class_count` and their order matches the final +tensor dimension. Token IDs follow the same alignment rule as token +embeddings. Outputs preserve request order and correlation. A successful +response contains exactly one output for every request input, and `input_index` +is present even for index zero. + +### Scoring + +```protobuf +message ScoreRequest { + TaskRequestContext context = 1; + repeated ScoreGroup groups = 2; + ScoreOptions options = 3; +} + +message ScoreGroup { + string group_id = 1; + TaskInput query = 2; + repeated TaskInput candidates = 3; +} + +message ScoreOptions { + optional TaskOutputGranularity granularity = 1; + optional TaskValueSemantics output_semantics = 2; + ScoreNormalization normalization = 3; + repeated uint32 label_token_ids = 4; + string instruction = 5; +} + +message ScoreResponse { + string request_id = 1; + repeated ScoreGroupOutput groups = 2; + TaskOutputGranularity granularity = 3; + TaskValueSemantics semantics = 4; + optional bool higher_is_better = 5; + ScoreNormalization normalization = 6; + repeated uint32 label_token_ids = 7; + TaskUsage usage = 8; +} + +message ScoreGroupOutput { + string group_id = 1; + optional uint32 group_index = 2; + repeated ScoreCandidateOutput candidates = 3; +} + +message ScoreCandidateOutput { + string candidate_id = 1; + optional uint32 candidate_index = 2; + DenseFloatTensor scores = 3; + repeated uint32 token_ids = 4; +} +``` + +Every score request has at least one group. Group IDs are unique, every group +has one query and at least one candidate, and candidate IDs are unique within +the group. Repeating groups represents N:N paired scoring; one group with many +candidates represents the optimized 1:N path used by rerankers and multi-item +scoring engines. + +Absent granularity and output semantics select model defaults. An explicit +value must be supported. `SCORE_NORMALIZATION_UNSPECIFIED` selects the model +default, `NONE` requests native unnormalized values, and `SOFTMAX` requests +normalization across each returned label vector. Duplicate label token IDs are +invalid. When `label_token_ids` is non-empty, the engine performs causal-model +label-token scoring and the response repeats those IDs in the same order. +When it is empty, the engine performs its advertised model-native scoring +operation. A non-empty `instruction` is valid only when instruction support is +advertised; template selection and rendering remain engine-owned. + +`ScoreResponse` reports the actual granularity, semantics, normalization, and +label-token order. Group and candidate results preserve request order and carry +their present zero-based original indexes, including index zero. A successful +response contains every group and candidate exactly once; partial success is +not represented. Token-granularity candidate tensors may return aligned token +IDs. The engine never sorts candidates or echoes source documents. + +A gateway may derive reranking only when every candidate has exactly one score +and `higher_is_better` is present. It stable-sorts using that direction, breaks +ties by `candidate_index`, applies its external `top_n`, and attaches documents +from gateway-owned request state. Reranking is response shaping, not a separate +OpenEngine inference capability. + +### Task capability discovery + +```protobuf +message TaskCapabilities { + EmbedCapabilities embed = 1; + ClassifyCapabilities classify = 2; + ScoreCapabilities score = 3; +} + +message EmbedCapabilities { + optional bool supported = 1; + repeated TaskInputType input_types = 2; + repeated TaskOutputGranularity granularities = 3; + repeated EmbeddingEncoding encodings = 4; + optional uint32 dimension = 5; + optional uint32 max_batch_size = 6; + optional uint64 max_output_values_per_item = 7; + optional bool supports_priority = 8; + optional bool supports_lora = 9; + optional bool supports_normalization = 10; + optional bool supports_dimension_override = 11; + repeated Modality modalities = 12; +} + +message ClassifyCapabilities { + optional bool supported = 1; + repeated TaskInputType input_types = 2; + repeated TaskOutputGranularity granularities = 3; + repeated TaskValueSemantics semantics = 4; + optional uint32 max_batch_size = 5; + optional uint64 max_output_values_per_item = 6; + optional bool supports_priority = 7; + optional bool supports_lora = 8; + repeated Modality modalities = 9; +} + +message ScoreCapabilities { + optional bool supported = 1; + repeated TaskInputType input_types = 2; + repeated TaskOutputGranularity granularities = 3; + repeated TaskValueSemantics semantics = 4; + repeated ScoreNormalization normalizations = 5; + optional bool supports_label_token_scoring = 6; + optional bool supports_instruction = 7; + optional uint32 max_groups = 8; + optional uint32 max_candidates_per_group = 9; + optional uint64 max_output_values_per_candidate = 10; + optional bool supports_priority = 11; + optional bool supports_lora = 12; + optional bool higher_is_better = 13; + repeated Modality modalities = 14; + optional uint32 max_label_token_ids = 15; +} +``` + +An absent `ModelInfo.tasks` means the model does not advertise non-generative +task support. Within it, an absent task capability is unreported; a present +capability uses `supported` presence to distinguish unreported support from an +explicit `true` or `false`. Capability lists never contain `UNSPECIFIED`. +Reported dimensions and limits are greater than zero. `dimension` is present +only when a model has one fixed native embedding dimension. A rankable native +score advertises its default direction through `higher_is_better`; the response +still reports the actual direction for each request. `modalities` is meaningful +only when the task includes `TASK_INPUT_TYPE_MULTIMODAL`, and never contains +`MODALITY_UNSPECIFIED`. +For dense embeddings, `max_output_values_per_item` limits the flattened element +count; for sparse embeddings, it limits the number of returned nonzero values. + +The server rejects an unsupported task with `FAILED_PRECONDITION`, an unknown +model with `NOT_FOUND`, malformed input or unsupported explicit options with +`INVALID_ARGUMENT`, and admission/capacity exhaustion with +`RESOURCE_EXHAUSTED`. A successful unary response is terminal. Client +cancellation or deadline expiration stops queued or running work, and +`Abort(request_id)` follows the same semantics as generation. + +--- + ## LoRA lifecycle ```protobuf diff --git a/proto/openengine/v1/README.md b/proto/openengine/v1/README.md index 9b451ca..6c97034 100644 --- a/proto/openengine/v1/README.md +++ b/proto/openengine/v1/README.md @@ -11,10 +11,14 @@ share the same package and together define the API. | File | Area | | --- | --- | | [`openengine.proto`](openengine.proto) | `OpenEngine` service and RPC declarations | +| [`input.proto`](input.proto) | Shared text, token, and multimodal inputs | | [`engine.proto`](engine.proto) | Engine identity, roles, and parallelism | -| [`model.proto`](model.proto) | Model metadata and generation capabilities | -| [`generation.proto`](generation.proto) | Generation requests, streamed events, and usage | -| [`generation_params.proto`](generation_params.proto) | Sampling, stopping, response, KV, and guided-decoding parameters | +| [`model.proto`](model.proto) | Model metadata and inference capabilities | +| [`generation.proto`](generation.proto) | Generation requests, parameters, streamed events, and usage | +| [`tasks.proto`](tasks.proto) | Shared non-generative task request and output vocabulary | +| [`embedding.proto`](embedding.proto) | Dense and sparse embedding inference | +| [`classification.proto`](classification.proto) | Sequence and token classification inference | +| [`scoring.proto`](scoring.proto) | Grouped query and candidate scoring inference | | [`lora.proto`](lora.proto) | LoRA adapter lifecycle | | [`kv.proto`](kv.proto) | KV sessions, connector discovery, and cache events | | [`lifecycle.proto`](lifecycle.proto) | Health, abort, and drain operations | diff --git a/proto/openengine/v1/classification.proto b/proto/openengine/v1/classification.proto new file mode 100644 index 0000000..20b65af --- /dev/null +++ b/proto/openengine/v1/classification.proto @@ -0,0 +1,50 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Portable sequence and token classification inference. + +syntax = "proto3"; + +package openengine.v1; + +import "openengine/v1/input.proto"; +import "openengine/v1/tasks.proto"; + +message ClassifyCapabilities { + optional bool supported = 1; + repeated TaskInputType input_types = 2; + repeated TaskOutputGranularity granularities = 3; + repeated TaskValueSemantics semantics = 4; + optional uint32 max_batch_size = 5; + optional uint64 max_output_values_per_item = 6; + optional bool supports_priority = 7; + optional bool supports_lora = 8; + repeated Modality modalities = 9; +} + +message ClassifyRequest { + TaskRequestContext context = 1; + repeated TaskInput inputs = 2; + ClassifyOptions options = 3; +} + +message ClassifyOptions { + optional TaskOutputGranularity granularity = 1; + optional TaskValueSemantics output_semantics = 2; +} + +message ClassifyResponse { + string request_id = 1; + repeated ClassificationOutput outputs = 2; + TaskUsage usage = 3; +} + +message ClassificationOutput { + string item_id = 1; + optional uint32 input_index = 2; + TaskOutputGranularity granularity = 3; + TaskValueSemantics semantics = 4; + DenseFloatTensor scores = 5; + repeated string labels = 6; + repeated uint32 token_ids = 7; +} diff --git a/proto/openengine/v1/embedding.proto b/proto/openengine/v1/embedding.proto new file mode 100644 index 0000000..70b7fe6 --- /dev/null +++ b/proto/openengine/v1/embedding.proto @@ -0,0 +1,69 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Portable dense and sparse embedding inference. + +syntax = "proto3"; + +package openengine.v1; + +import "openengine/v1/input.proto"; +import "openengine/v1/tasks.proto"; + +enum EmbeddingEncoding { + EMBEDDING_ENCODING_UNSPECIFIED = 0; + EMBEDDING_ENCODING_DENSE = 1; + EMBEDDING_ENCODING_SPARSE = 2; +} + +// Row-major sparse FP32 tensor with flattened, strictly increasing indices. +message SparseFloatTensor { + repeated uint64 shape = 1; + repeated uint64 indices = 2; + repeated float values = 3; +} + +message EmbedCapabilities { + optional bool supported = 1; + repeated TaskInputType input_types = 2; + repeated TaskOutputGranularity granularities = 3; + repeated EmbeddingEncoding encodings = 4; + optional uint32 dimension = 5; + optional uint32 max_batch_size = 6; + optional uint64 max_output_values_per_item = 7; + optional bool supports_priority = 8; + optional bool supports_lora = 9; + optional bool supports_normalization = 10; + optional bool supports_dimension_override = 11; + repeated Modality modalities = 12; +} + +message EmbedRequest { + TaskRequestContext context = 1; + repeated TaskInput inputs = 2; + EmbedOptions options = 3; +} + +message EmbedOptions { + optional TaskOutputGranularity granularity = 1; + optional bool normalize = 2; + optional uint32 dimensions = 3; + optional EmbeddingEncoding encoding = 4; +} + +message EmbedResponse { + string request_id = 1; + repeated EmbeddingOutput outputs = 2; + TaskUsage usage = 3; +} + +message EmbeddingOutput { + string item_id = 1; + optional uint32 input_index = 2; + TaskOutputGranularity granularity = 3; + oneof embedding { + DenseFloatTensor dense = 4; + SparseFloatTensor sparse = 5; + } + repeated uint32 token_ids = 6; +} diff --git a/proto/openengine/v1/generation.proto b/proto/openengine/v1/generation.proto index 3d8b50b..34ed557 100644 --- a/proto/openengine/v1/generation.proto +++ b/proto/openengine/v1/generation.proto @@ -8,7 +8,7 @@ syntax = "proto3"; package openengine.v1; import "openengine/v1/error.proto"; -import "openengine/v1/generation_params.proto"; +import "openengine/v1/input.proto"; import "openengine/v1/kv.proto"; message GenerateRequest { @@ -44,29 +44,81 @@ message GenerateRequest { map metadata = 13; } -// Multimodal modality discriminator. 0 is treated as image for forward -// compatibility with senders that omit the field. -enum Modality { - MODALITY_UNSPECIFIED = 0; - MODALITY_IMAGE = 1; - MODALITY_VIDEO = 2; - MODALITY_AUDIO = 3; -} - -// A single multimodal input. Exactly one `source` should be set. The engine -// owns fetch, decode, and preprocessing, so pre-decoded or RDMA media -// descriptors are not represented here. -message MediaItem { - Modality modality = 1; - oneof source { - string url = 2; // http(s):// -- engine fetches - string data_uri = 3; // data:;base64,<...> -- engine decodes - bytes raw_bytes = 4; // pre-fetched bytes -- engine still preprocesses +message SamplingParams { + optional double temperature = 1; + optional double top_p = 2; + optional int32 top_k = 3; + optional double min_p = 4; + optional double frequency_penalty = 5; + optional double presence_penalty = 6; + optional double repetition_penalty = 7; + optional uint64 seed = 8; + optional uint32 num_sequences = 9; +} + +message StoppingOptions { + optional uint32 max_tokens = 1; + optional uint32 min_tokens = 2; + repeated StopCondition conditions = 3; + optional bool ignore_eos = 4; + optional bool include_stop_in_output = 5; +} + +message ResponseOptions { + optional bool return_prompt_logprobs = 1; + CandidateTokenSelection prompt_candidates = 2; + optional bool return_output_logprobs = 3; + CandidateTokenSelection output_candidates = 4; + optional uint32 prompt_logprob_start = 5; +} + +message CandidateTokenSelection { + oneof selection { + uint32 top_n = 1; + TokenIds token_ids = 2; + AllCandidates all = 3; + } +} + +message AllCandidates {} + +message KvOptions { + KvSessionRef session = 1; + optional uint32 data_parallel_rank = 2; + optional bool bypass_prefix_cache = 3; + optional string cache_salt = 4; +} + +message StopCondition { + oneof condition { + string stop_text = 1; + uint32 stop_token_id = 2; } - string mime_type = 5; // optional, hints raw_bytes decode - string uuid = 6; // optional caller id / mm_hash } +// Constrained / guided decoding spec. At most one of `guide` should be set. +// The engine enforces the constraint during sampling via its grammar backend +// (xgrammar / outlines / llguidance); clients cannot apply it post-hoc. +message GuidedDecoding { + oneof guide { + string json_schema = 1; // output conforms to this JSON schema + string regex = 2; // output matches this regex + string ebnf_grammar = 3; // output follows this EBNF / context-free grammar + string structural_tag = 4; // xgrammar structural-tag constraint (JSON string) + ChoiceConstraint choice = 5; + JsonObjectConstraint json_object = 6; + } + // Optional grammar backend override (e.g. "xgrammar", "outlines", + // "llguidance"). Empty = engine default. + string backend = 7; +} + +message ChoiceConstraint { + repeated string choices = 1; +} + +message JsonObjectConstraint {} + message GenerateResponse { string request_id = 1; diff --git a/proto/openengine/v1/generation_params.proto b/proto/openengine/v1/generation_params.proto deleted file mode 100644 index 48143c5..0000000 --- a/proto/openengine/v1/generation_params.proto +++ /dev/null @@ -1,89 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -// Portable generation parameters and reusable selection types. - -syntax = "proto3"; - -package openengine.v1; - -import "openengine/v1/kv.proto"; - -message TokenIds { - repeated uint32 ids = 1; -} - -message SamplingParams { - optional double temperature = 1; - optional double top_p = 2; - optional int32 top_k = 3; - optional double min_p = 4; - optional double frequency_penalty = 5; - optional double presence_penalty = 6; - optional double repetition_penalty = 7; - optional uint64 seed = 8; - optional uint32 num_sequences = 9; -} - -message StoppingOptions { - optional uint32 max_tokens = 1; - optional uint32 min_tokens = 2; - repeated StopCondition conditions = 3; - optional bool ignore_eos = 4; - optional bool include_stop_in_output = 5; -} - -message ResponseOptions { - optional bool return_prompt_logprobs = 1; - CandidateTokenSelection prompt_candidates = 2; - optional bool return_output_logprobs = 3; - CandidateTokenSelection output_candidates = 4; - optional uint32 prompt_logprob_start = 5; -} - -message CandidateTokenSelection { - oneof selection { - uint32 top_n = 1; - TokenIds token_ids = 2; - AllCandidates all = 3; - } -} - -message AllCandidates {} - -message KvOptions { - KvSessionRef session = 1; - optional uint32 data_parallel_rank = 2; - optional bool bypass_prefix_cache = 3; - optional string cache_salt = 4; -} - -message StopCondition { - oneof condition { - string stop_text = 1; - uint32 stop_token_id = 2; - } -} - -// Constrained / guided decoding spec. At most one of `guide` should be set. -// The engine enforces the constraint during sampling via its grammar backend -// (xgrammar / outlines / llguidance); clients cannot apply it post-hoc. -message GuidedDecoding { - oneof guide { - string json_schema = 1; // output conforms to this JSON schema - string regex = 2; // output matches this regex - string ebnf_grammar = 3; // output follows this EBNF / context-free grammar - string structural_tag = 4; // xgrammar structural-tag constraint (JSON string) - ChoiceConstraint choice = 5; - JsonObjectConstraint json_object = 6; - } - // Optional grammar backend override (e.g. "xgrammar", "outlines", - // "llguidance"). Empty = engine default. - string backend = 7; -} - -message ChoiceConstraint { - repeated string choices = 1; -} - -message JsonObjectConstraint {} diff --git a/proto/openengine/v1/input.proto b/proto/openengine/v1/input.proto new file mode 100644 index 0000000..5b384c1 --- /dev/null +++ b/proto/openengine/v1/input.proto @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Portable text, token, and multimodal inference inputs. + +syntax = "proto3"; + +package openengine.v1; + +message TokenIds { + repeated uint32 ids = 1; +} + +// Multimodal modality discriminator. 0 is treated as image for forward +// compatibility with senders that omit the field. +enum Modality { + MODALITY_UNSPECIFIED = 0; + MODALITY_IMAGE = 1; + MODALITY_VIDEO = 2; + MODALITY_AUDIO = 3; +} + +// A single multimodal input. Exactly one `source` should be set. The engine +// owns fetch, decode, and preprocessing, so pre-decoded or RDMA media +// descriptors are not represented here. +message MediaItem { + Modality modality = 1; + oneof source { + string url = 2; // http(s):// -- engine fetches + string data_uri = 3; // data:;base64,<...> -- engine decodes + bytes raw_bytes = 4; // pre-fetched bytes -- engine still preprocesses + } + string mime_type = 5; // optional, hints raw_bytes decode + string uuid = 6; // optional caller id / mm_hash +} + +// Text and token prompts may accompany media in a multimodal task input. +message MultimodalTaskInput { + oneof prompt { + string text = 1; + TokenIds token_ids = 2; + } + repeated MediaItem media = 3; +} + +// One independently correlated non-generative task input. +message TaskInput { + string item_id = 1; + oneof input { + string text = 2; + TokenIds token_ids = 3; + MultimodalTaskInput multimodal = 4; + } +} + +enum TaskInputType { + TASK_INPUT_TYPE_UNSPECIFIED = 0; + TASK_INPUT_TYPE_TEXT = 1; + TASK_INPUT_TYPE_TOKEN_IDS = 2; + TASK_INPUT_TYPE_MULTIMODAL = 3; +} diff --git a/proto/openengine/v1/model.proto b/proto/openengine/v1/model.proto index 828aac6..3cfb72c 100644 --- a/proto/openengine/v1/model.proto +++ b/proto/openengine/v1/model.proto @@ -1,12 +1,16 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -// Model metadata and portable generation capabilities. +// Model metadata and portable inference capabilities. syntax = "proto3"; package openengine.v1; +import "openengine/v1/classification.proto"; +import "openengine/v1/embedding.proto"; +import "openengine/v1/scoring.proto"; + message GetModelInfoRequest { string model = 1; } @@ -34,6 +38,15 @@ message ModelInfo { // Empty means no parser is configured for this model. string reasoning_parser = 25; string tool_call_parser = 26; + + // Optional non-generative task support for this model. + TaskCapabilities tasks = 27; +} + +message TaskCapabilities { + EmbedCapabilities embed = 1; + ClassifyCapabilities classify = 2; + ScoreCapabilities score = 3; } message GenerationCapabilities { diff --git a/proto/openengine/v1/openengine.proto b/proto/openengine/v1/openengine.proto index 71f26b8..062d68f 100644 --- a/proto/openengine/v1/openengine.proto +++ b/proto/openengine/v1/openengine.proto @@ -16,6 +16,8 @@ syntax = "proto3"; package openengine.v1; +import "openengine/v1/classification.proto"; +import "openengine/v1/embedding.proto"; import "openengine/v1/engine.proto"; import "openengine/v1/generation.proto"; import "openengine/v1/kv.proto"; @@ -23,11 +25,17 @@ import "openengine/v1/lifecycle.proto"; import "openengine/v1/lora.proto"; import "openengine/v1/model.proto"; import "openengine/v1/observability.proto"; +import "openengine/v1/scoring.proto"; service OpenEngine { // Core inference path. rpc Generate(GenerateRequest) returns (stream GenerateResponse); + // Non-generative inference paths. + rpc Embed(EmbedRequest) returns (EmbedResponse); + rpc Classify(ClassifyRequest) returns (ClassifyResponse); + rpc Score(ScoreRequest) returns (ScoreResponse); + // Runtime metadata and scheduling state. rpc GetEngineInfo(GetEngineInfoRequest) returns (EngineInfo); rpc GetModelInfo(GetModelInfoRequest) returns (ModelInfo); diff --git a/proto/openengine/v1/scoring.proto b/proto/openengine/v1/scoring.proto new file mode 100644 index 0000000..4887913 --- /dev/null +++ b/proto/openengine/v1/scoring.proto @@ -0,0 +1,80 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Portable grouped query and candidate scoring inference. + +syntax = "proto3"; + +package openengine.v1; + +import "openengine/v1/input.proto"; +import "openengine/v1/tasks.proto"; + +enum ScoreNormalization { + SCORE_NORMALIZATION_UNSPECIFIED = 0; + SCORE_NORMALIZATION_NONE = 1; + SCORE_NORMALIZATION_SOFTMAX = 2; +} + +message ScoreCapabilities { + optional bool supported = 1; + repeated TaskInputType input_types = 2; + repeated TaskOutputGranularity granularities = 3; + repeated TaskValueSemantics semantics = 4; + repeated ScoreNormalization normalizations = 5; + optional bool supports_label_token_scoring = 6; + optional bool supports_instruction = 7; + optional uint32 max_groups = 8; + optional uint32 max_candidates_per_group = 9; + optional uint64 max_output_values_per_candidate = 10; + optional bool supports_priority = 11; + optional bool supports_lora = 12; + optional bool higher_is_better = 13; + repeated Modality modalities = 14; + optional uint32 max_label_token_ids = 15; +} + +message ScoreRequest { + TaskRequestContext context = 1; + repeated ScoreGroup groups = 2; + ScoreOptions options = 3; +} + +// A query and one or more candidates scored as one correlation group. +message ScoreGroup { + string group_id = 1; + TaskInput query = 2; + repeated TaskInput candidates = 3; +} + +message ScoreOptions { + optional TaskOutputGranularity granularity = 1; + optional TaskValueSemantics output_semantics = 2; + ScoreNormalization normalization = 3; + repeated uint32 label_token_ids = 4; + string instruction = 5; +} + +message ScoreResponse { + string request_id = 1; + repeated ScoreGroupOutput groups = 2; + TaskOutputGranularity granularity = 3; + TaskValueSemantics semantics = 4; + optional bool higher_is_better = 5; + ScoreNormalization normalization = 6; + repeated uint32 label_token_ids = 7; + TaskUsage usage = 8; +} + +message ScoreGroupOutput { + string group_id = 1; + optional uint32 group_index = 2; + repeated ScoreCandidateOutput candidates = 3; +} + +message ScoreCandidateOutput { + string candidate_id = 1; + optional uint32 candidate_index = 2; + DenseFloatTensor scores = 3; + repeated uint32 token_ids = 4; +} diff --git a/proto/openengine/v1/tasks.proto b/proto/openengine/v1/tasks.proto new file mode 100644 index 0000000..2338885 --- /dev/null +++ b/proto/openengine/v1/tasks.proto @@ -0,0 +1,46 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Shared request and output vocabulary for non-generative inference tasks. + +syntax = "proto3"; + +package openengine.v1; + +// Request metadata shared by every non-generative task. +message TaskRequestContext { + string request_id = 1; + string model = 2; + string lora_name = 3; + optional int32 priority = 4; + map metadata = 5; +} + +// Row-major FP32 tensor. The product of shape must equal values.size(). +message DenseFloatTensor { + repeated uint64 shape = 1; + repeated float values = 2; +} + +// Token accounting for one complete unary task request. +message TaskUsage { + uint64 input_tokens = 1; + optional uint64 cached_input_tokens = 2; +} + +enum TaskOutputGranularity { + TASK_OUTPUT_GRANULARITY_UNSPECIFIED = 0; + TASK_OUTPUT_GRANULARITY_SEQUENCE = 1; + TASK_OUTPUT_GRANULARITY_TOKEN = 2; +} + +enum TaskValueSemantics { + TASK_VALUE_SEMANTICS_UNSPECIFIED = 0; + TASK_VALUE_SEMANTICS_LOGITS = 1; + TASK_VALUE_SEMANTICS_PROBABILITIES = 2; + TASK_VALUE_SEMANTICS_LOG_PROBABILITIES = 3; + TASK_VALUE_SEMANTICS_SIMILARITY = 4; + TASK_VALUE_SEMANTICS_RELEVANCE = 5; + TASK_VALUE_SEMANTICS_REWARD = 6; + TASK_VALUE_SEMANTICS_MODEL_DEFINED = 7; +}