From ec44bc6e05612a593709a221f5ae9d79ca434257 Mon Sep 17 00:00:00 2001 From: Connor Carpenter Date: Fri, 10 Jul 2026 13:23:43 -0700 Subject: [PATCH 1/3] feat(api): add non-generative task RPCs Signed-off-by: Connor Carpenter --- README.md | 3 +- docs/api.md | 353 +++++++++++++++++++++++++++ proto/openengine/v1/README.md | 1 + proto/openengine/v1/model.proto | 7 +- proto/openengine/v1/openengine.proto | 6 + proto/openengine/v1/tasks.proto | 248 +++++++++++++++++++ 6 files changed, 616 insertions(+), 2 deletions(-) create mode 100644 proto/openengine/v1/tasks.proto diff --git a/README.md b/README.md index aadc6fc..5451c29 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,7 @@ The canonical schema is organized by domain under | Area | What the contract provides | | --------------------- | ----------------------------------------------------------------------------------------------------------------- | | Portable generation | Text or token input, sampling, stopping, priorities, multiple sequences, and deterministic seeds | +| Non-generative tasks | Typed embedding, classification, and grouped query/candidate scoring with stable correlation | | Structured output | JSON Schema, JSON object, regex, EBNF grammar, structural tags, and fixed choices | | Token information | Prompt and output logprobs, ranks, candidate-token selection, per-token records, and streamed text deltas | | Discovery | Engine identity, schema revision, role, model limits, topology, parser configuration, and generation capabilities | @@ -168,7 +169,7 @@ depend on it. Expect direct schema refinement during this phase. The intended adoption path is incremental: -1. Aggregated text generation, discovery, health, abort, and drain. +1. Aggregated generation and non-generative tasks, discovery, health, abort, and drain. 2. Prefill/decode roles, KV handoff, rank affinity, and KV event integration. 3. Logprobs, guided decoding, LoRA, and multimodal input as needed. diff --git a/docs/api.md b/docs/api.md index 8f5c3da..427441e 100644 --- a/docs/api.md +++ b/docs/api.md @@ -24,6 +24,11 @@ service OpenEngine { // Core inference path. rpc Generate(GenerateRequest) returns (stream GenerateResponse); + // Non-generative inference paths. + rpc Embed(EmbedRequest) returns (EmbedResponse); + rpc Classify(ClassifyRequest) returns (ClassifyResponse); + rpc Score(ScoreRequest) returns (ScoreResponse); + // Runtime metadata and scheduling state. rpc GetEngineInfo(GetEngineInfoRequest) returns (EngineInfo); rpc GetModelInfo(GetModelInfoRequest) returns (ModelInfo); @@ -143,6 +148,7 @@ message ModelInfo { string reasoning_parser = 25; string tool_call_parser = 26; + TaskCapabilities tasks = 27; } message GenerationCapabilities { @@ -198,6 +204,353 @@ the LoRA lifecycle RPCs on `OpenEngine`. --- +## Non-generative task API + +`Embed`, `Classify`, and `Score` are unary inference operations. They share a +request context and typed inputs, but retain task-specific options and outputs. +Support is optional per model and advertised through `ModelInfo.tasks`. + +```protobuf +message TaskRequestContext { + string request_id = 1; + string model = 2; + string lora_name = 3; + optional int32 priority = 4; + map metadata = 5; +} + +message TaskInput { + string item_id = 1; + oneof input { + string text = 2; + TokenIds token_ids = 3; + MultimodalTaskInput multimodal = 4; + } +} + +message MultimodalTaskInput { + oneof prompt { + string text = 1; + TokenIds token_ids = 2; + } + repeated MediaItem media = 3; +} + +message DenseFloatTensor { + repeated uint64 shape = 1; + repeated float values = 2; +} + +message SparseFloatTensor { + repeated uint64 shape = 1; + repeated uint64 indices = 2; + repeated float values = 3; +} + +message TaskUsage { + uint64 input_tokens = 1; + optional uint64 cached_input_tokens = 2; +} + +enum TaskInputType { + TASK_INPUT_TYPE_UNSPECIFIED = 0; + TASK_INPUT_TYPE_TEXT = 1; + TASK_INPUT_TYPE_TOKEN_IDS = 2; + TASK_INPUT_TYPE_MULTIMODAL = 3; +} + +enum TaskOutputGranularity { + TASK_OUTPUT_GRANULARITY_UNSPECIFIED = 0; + TASK_OUTPUT_GRANULARITY_SEQUENCE = 1; + TASK_OUTPUT_GRANULARITY_TOKEN = 2; +} + +enum EmbeddingEncoding { + EMBEDDING_ENCODING_UNSPECIFIED = 0; + EMBEDDING_ENCODING_DENSE = 1; + EMBEDDING_ENCODING_SPARSE = 2; +} + +enum TaskValueSemantics { + TASK_VALUE_SEMANTICS_UNSPECIFIED = 0; + TASK_VALUE_SEMANTICS_LOGITS = 1; + TASK_VALUE_SEMANTICS_PROBABILITIES = 2; + TASK_VALUE_SEMANTICS_LOG_PROBABILITIES = 3; + TASK_VALUE_SEMANTICS_SIMILARITY = 4; + TASK_VALUE_SEMANTICS_RELEVANCE = 5; + TASK_VALUE_SEMANTICS_REWARD = 6; + TASK_VALUE_SEMANTICS_MODEL_DEFINED = 7; +} + +enum ScoreNormalization { + SCORE_NORMALIZATION_UNSPECIFIED = 0; + SCORE_NORMALIZATION_NONE = 1; + SCORE_NORMALIZATION_SOFTMAX = 2; +} +``` + +`TaskRequestContext.request_id` and `model` are required and non-empty. Request +IDs share the same namespace and abort semantics as generation request IDs. +`priority` uses the generation ordering convention: larger values have higher +priority. A non-empty `lora_name` selects an already loaded adapter. Clients +must use either option only when the corresponding task capability advertises +support. + +Each request batch must be non-empty. `item_id` is required and unique within +an embed/classify batch, and every query/candidate item ID is unique within one +score group. Exactly one `TaskInput.input` variant is set. A multimodal input must +contain at least one `MediaItem`; its optional prompt is either text or token +IDs, never both. Media ordering and validation follow `GenerateRequest.media`. + +`DenseFloatTensor` is row-major FP32 data. Every dimension is greater than zero, +the product of `shape` equals `values.size()`, and every value is finite. +`SparseFloatTensor` uses flattened row-major indices; `indices` and `values` +have equal length, indices are unique and strictly increasing, and every index +is smaller than the product of `shape`. A scalar is encoded with shape `[1]`, +not an empty shape. `cached_input_tokens`, when present, does not exceed +`input_tokens`. + +### Embedding + +```protobuf +message EmbedRequest { + TaskRequestContext context = 1; + repeated TaskInput inputs = 2; + EmbedOptions options = 3; +} + +message EmbedOptions { + optional TaskOutputGranularity granularity = 1; + optional bool normalize = 2; + optional uint32 dimensions = 3; + optional EmbeddingEncoding encoding = 4; +} + +message EmbedResponse { + string request_id = 1; + repeated EmbeddingOutput outputs = 2; + TaskUsage usage = 3; +} + +message EmbeddingOutput { + string item_id = 1; + optional uint32 input_index = 2; + TaskOutputGranularity granularity = 3; + oneof embedding { + DenseFloatTensor dense = 4; + SparseFloatTensor sparse = 5; + } + repeated uint32 token_ids = 6; +} +``` + +Absent embedding options select model defaults. An explicit granularity, +normalization, or encoding request must be implemented exactly or rejected. +`dimensions` must be greater than zero and requests dimensionality reduction; +it does not permit padding a smaller model output. + +Sequence embeddings have shape `[dimension]`. Token embeddings have shape +`[token_count, dimension]`; `token_ids`, when returned, has `token_count` +entries aligned to the first tensor dimension. Outputs preserve request order, +and `input_index` is present even for index zero and identifies the matching +input. A successful response contains exactly one output for every request +input. + +### Classification + +```protobuf +message ClassifyRequest { + TaskRequestContext context = 1; + repeated TaskInput inputs = 2; + ClassifyOptions options = 3; +} + +message ClassifyOptions { + optional TaskOutputGranularity granularity = 1; + optional TaskValueSemantics output_semantics = 2; +} + +message ClassifyResponse { + string request_id = 1; + repeated ClassificationOutput outputs = 2; + TaskUsage usage = 3; +} + +message ClassificationOutput { + string item_id = 1; + optional uint32 input_index = 2; + TaskOutputGranularity granularity = 3; + TaskValueSemantics semantics = 4; + DenseFloatTensor scores = 5; + repeated string labels = 6; + repeated uint32 token_ids = 7; +} +``` + +Classification normally returns logits, probabilities, log probabilities, or +model-defined values. The engine reports the actual non-`UNSPECIFIED` +semantics. Sequence classification has shape `[class_count]`; token +classification has shape `[token_count, class_count]`. When labels are +returned, their count equals `class_count` and their order matches the final +tensor dimension. Token IDs follow the same alignment rule as token +embeddings. Outputs preserve request order and correlation. A successful +response contains exactly one output for every request input, and `input_index` +is present even for index zero. + +### Scoring + +```protobuf +message ScoreRequest { + TaskRequestContext context = 1; + repeated ScoreGroup groups = 2; + ScoreOptions options = 3; +} + +message ScoreGroup { + string group_id = 1; + TaskInput query = 2; + repeated TaskInput candidates = 3; +} + +message ScoreOptions { + optional TaskOutputGranularity granularity = 1; + optional TaskValueSemantics output_semantics = 2; + ScoreNormalization normalization = 3; + repeated uint32 label_token_ids = 4; + string instruction = 5; +} + +message ScoreResponse { + string request_id = 1; + repeated ScoreGroupOutput groups = 2; + TaskOutputGranularity granularity = 3; + TaskValueSemantics semantics = 4; + optional bool higher_is_better = 5; + ScoreNormalization normalization = 6; + repeated uint32 label_token_ids = 7; + TaskUsage usage = 8; +} + +message ScoreGroupOutput { + string group_id = 1; + optional uint32 group_index = 2; + repeated ScoreCandidateOutput candidates = 3; +} + +message ScoreCandidateOutput { + string candidate_id = 1; + optional uint32 candidate_index = 2; + DenseFloatTensor scores = 3; + repeated uint32 token_ids = 4; +} +``` + +Every score request has at least one group. Group IDs are unique, every group +has one query and at least one candidate, and candidate IDs are unique within +the group. Repeating groups represents N:N paired scoring; one group with many +candidates represents the optimized 1:N path used by rerankers and multi-item +scoring engines. + +Absent granularity and output semantics select model defaults. An explicit +value must be supported. `SCORE_NORMALIZATION_UNSPECIFIED` selects the model +default, `NONE` requests native unnormalized values, and `SOFTMAX` requests +normalization across each returned label vector. Duplicate label token IDs are +invalid. When `label_token_ids` is non-empty, the engine performs causal-model +label-token scoring and the response repeats those IDs in the same order. +When it is empty, the engine performs its advertised model-native scoring +operation. A non-empty `instruction` is valid only when instruction support is +advertised; template selection and rendering remain engine-owned. + +`ScoreResponse` reports the actual granularity, semantics, normalization, and +label-token order. Group and candidate results preserve request order and carry +their present zero-based original indexes, including index zero. A successful +response contains every group and candidate exactly once; partial success is +not represented. Token-granularity candidate tensors may return aligned token +IDs. The engine never sorts candidates or echoes source documents. + +A gateway may derive reranking only when every candidate has exactly one score +and `higher_is_better` is present. It stable-sorts using that direction, breaks +ties by `candidate_index`, applies its external `top_n`, and attaches documents +from gateway-owned request state. Reranking is response shaping, not a separate +OpenEngine inference capability. + +### Task capability discovery + +```protobuf +message TaskCapabilities { + EmbedCapabilities embed = 1; + ClassifyCapabilities classify = 2; + ScoreCapabilities score = 3; +} + +message EmbedCapabilities { + optional bool supported = 1; + repeated TaskInputType input_types = 2; + repeated TaskOutputGranularity granularities = 3; + repeated EmbeddingEncoding encodings = 4; + optional uint32 dimension = 5; + optional uint32 max_batch_size = 6; + optional uint64 max_output_values_per_item = 7; + optional bool supports_priority = 8; + optional bool supports_lora = 9; + optional bool supports_normalization = 10; + optional bool supports_dimension_override = 11; + repeated Modality modalities = 12; +} + +message ClassifyCapabilities { + optional bool supported = 1; + repeated TaskInputType input_types = 2; + repeated TaskOutputGranularity granularities = 3; + repeated TaskValueSemantics semantics = 4; + optional uint32 max_batch_size = 5; + optional uint64 max_output_values_per_item = 6; + optional bool supports_priority = 7; + optional bool supports_lora = 8; + repeated Modality modalities = 9; +} + +message ScoreCapabilities { + optional bool supported = 1; + repeated TaskInputType input_types = 2; + repeated TaskOutputGranularity granularities = 3; + repeated TaskValueSemantics semantics = 4; + repeated ScoreNormalization normalizations = 5; + optional bool supports_label_token_scoring = 6; + optional bool supports_instruction = 7; + optional uint32 max_groups = 8; + optional uint32 max_candidates_per_group = 9; + optional uint64 max_output_values_per_candidate = 10; + optional bool supports_priority = 11; + optional bool supports_lora = 12; + optional bool higher_is_better = 13; + repeated Modality modalities = 14; + optional uint32 max_label_token_ids = 15; +} +``` + +An absent `ModelInfo.tasks` means the model does not advertise non-generative +task support. Within it, an absent task capability is unreported; a present +capability uses `supported` presence to distinguish unreported support from an +explicit `true` or `false`. Capability lists never contain `UNSPECIFIED`. +Reported dimensions and limits are greater than zero. `dimension` is present +only when a model has one fixed native embedding dimension. A rankable native +score advertises its default direction through `higher_is_better`; the response +still reports the actual direction for each request. `modalities` is meaningful +only when the task includes `TASK_INPUT_TYPE_MULTIMODAL`, and never contains +`MODALITY_UNSPECIFIED`. +For dense embeddings, `max_output_values_per_item` limits the flattened element +count; for sparse embeddings, it limits the number of returned nonzero values. + +The server rejects an unsupported task with `FAILED_PRECONDITION`, an unknown +model with `NOT_FOUND`, malformed input or unsupported explicit options with +`INVALID_ARGUMENT`, and admission/capacity exhaustion with +`RESOURCE_EXHAUSTED`. A successful unary response is terminal. Client +cancellation or deadline expiration stops queued or running work, and +`Abort(request_id)` follows the same semantics as generation. + +--- + ## LoRA lifecycle ```protobuf diff --git a/proto/openengine/v1/README.md b/proto/openengine/v1/README.md index 9b451ca..084772e 100644 --- a/proto/openengine/v1/README.md +++ b/proto/openengine/v1/README.md @@ -15,6 +15,7 @@ share the same package and together define the API. | [`model.proto`](model.proto) | Model metadata and generation capabilities | | [`generation.proto`](generation.proto) | Generation requests, streamed events, and usage | | [`generation_params.proto`](generation_params.proto) | Sampling, stopping, response, KV, and guided-decoding parameters | +| [`tasks.proto`](tasks.proto) | Embedding, classification, scoring, and task capability discovery | | [`lora.proto`](lora.proto) | LoRA adapter lifecycle | | [`kv.proto`](kv.proto) | KV sessions, connector discovery, and cache events | | [`lifecycle.proto`](lifecycle.proto) | Health, abort, and drain operations | diff --git a/proto/openengine/v1/model.proto b/proto/openengine/v1/model.proto index 828aac6..c2cf582 100644 --- a/proto/openengine/v1/model.proto +++ b/proto/openengine/v1/model.proto @@ -1,12 +1,14 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -// Model metadata and portable generation capabilities. +// Model metadata and portable inference capabilities. syntax = "proto3"; package openengine.v1; +import "openengine/v1/tasks.proto"; + message GetModelInfoRequest { string model = 1; } @@ -34,6 +36,9 @@ message ModelInfo { // Empty means no parser is configured for this model. string reasoning_parser = 25; string tool_call_parser = 26; + + // Optional non-generative task support for this model. + TaskCapabilities tasks = 27; } message GenerationCapabilities { diff --git a/proto/openengine/v1/openengine.proto b/proto/openengine/v1/openengine.proto index 71f26b8..dd8d566 100644 --- a/proto/openengine/v1/openengine.proto +++ b/proto/openengine/v1/openengine.proto @@ -23,11 +23,17 @@ import "openengine/v1/lifecycle.proto"; import "openengine/v1/lora.proto"; import "openengine/v1/model.proto"; import "openengine/v1/observability.proto"; +import "openengine/v1/tasks.proto"; service OpenEngine { // Core inference path. rpc Generate(GenerateRequest) returns (stream GenerateResponse); + // Non-generative inference paths. + rpc Embed(EmbedRequest) returns (EmbedResponse); + rpc Classify(ClassifyRequest) returns (ClassifyResponse); + rpc Score(ScoreRequest) returns (ScoreResponse); + // Runtime metadata and scheduling state. rpc GetEngineInfo(GetEngineInfoRequest) returns (EngineInfo); rpc GetModelInfo(GetModelInfoRequest) returns (ModelInfo); diff --git a/proto/openengine/v1/tasks.proto b/proto/openengine/v1/tasks.proto new file mode 100644 index 0000000..5a1c7c5 --- /dev/null +++ b/proto/openengine/v1/tasks.proto @@ -0,0 +1,248 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Portable non-generative inference tasks and model capabilities. + +syntax = "proto3"; + +package openengine.v1; + +import "openengine/v1/generation.proto"; +import "openengine/v1/generation_params.proto"; + +// Request metadata shared by every non-generative task. +message TaskRequestContext { + string request_id = 1; + string model = 2; + string lora_name = 3; + optional int32 priority = 4; + map metadata = 5; +} + +// Text and token prompts may accompany media in a multimodal task input. +message MultimodalTaskInput { + oneof prompt { + string text = 1; + TokenIds token_ids = 2; + } + repeated MediaItem media = 3; +} + +// One independently correlated task input. +message TaskInput { + string item_id = 1; + oneof input { + string text = 2; + TokenIds token_ids = 3; + MultimodalTaskInput multimodal = 4; + } +} + +// Row-major FP32 tensor. The product of shape must equal values.size(). +message DenseFloatTensor { + repeated uint64 shape = 1; + repeated float values = 2; +} + +// Row-major sparse FP32 tensor with flattened, strictly increasing indices. +message SparseFloatTensor { + repeated uint64 shape = 1; + repeated uint64 indices = 2; + repeated float values = 3; +} + +// Token accounting for one complete unary task request. +message TaskUsage { + uint64 input_tokens = 1; + optional uint64 cached_input_tokens = 2; +} + +enum TaskInputType { + TASK_INPUT_TYPE_UNSPECIFIED = 0; + TASK_INPUT_TYPE_TEXT = 1; + TASK_INPUT_TYPE_TOKEN_IDS = 2; + TASK_INPUT_TYPE_MULTIMODAL = 3; +} + +enum TaskOutputGranularity { + TASK_OUTPUT_GRANULARITY_UNSPECIFIED = 0; + TASK_OUTPUT_GRANULARITY_SEQUENCE = 1; + TASK_OUTPUT_GRANULARITY_TOKEN = 2; +} + +enum EmbeddingEncoding { + EMBEDDING_ENCODING_UNSPECIFIED = 0; + EMBEDDING_ENCODING_DENSE = 1; + EMBEDDING_ENCODING_SPARSE = 2; +} + +enum TaskValueSemantics { + TASK_VALUE_SEMANTICS_UNSPECIFIED = 0; + TASK_VALUE_SEMANTICS_LOGITS = 1; + TASK_VALUE_SEMANTICS_PROBABILITIES = 2; + TASK_VALUE_SEMANTICS_LOG_PROBABILITIES = 3; + TASK_VALUE_SEMANTICS_SIMILARITY = 4; + TASK_VALUE_SEMANTICS_RELEVANCE = 5; + TASK_VALUE_SEMANTICS_REWARD = 6; + TASK_VALUE_SEMANTICS_MODEL_DEFINED = 7; +} + +enum ScoreNormalization { + SCORE_NORMALIZATION_UNSPECIFIED = 0; + SCORE_NORMALIZATION_NONE = 1; + SCORE_NORMALIZATION_SOFTMAX = 2; +} + +// Per-model support and limits for non-generative tasks. +message TaskCapabilities { + EmbedCapabilities embed = 1; + ClassifyCapabilities classify = 2; + ScoreCapabilities score = 3; +} + +message EmbedCapabilities { + optional bool supported = 1; + repeated TaskInputType input_types = 2; + repeated TaskOutputGranularity granularities = 3; + repeated EmbeddingEncoding encodings = 4; + optional uint32 dimension = 5; + optional uint32 max_batch_size = 6; + optional uint64 max_output_values_per_item = 7; + optional bool supports_priority = 8; + optional bool supports_lora = 9; + optional bool supports_normalization = 10; + optional bool supports_dimension_override = 11; + repeated Modality modalities = 12; +} + +message ClassifyCapabilities { + optional bool supported = 1; + repeated TaskInputType input_types = 2; + repeated TaskOutputGranularity granularities = 3; + repeated TaskValueSemantics semantics = 4; + optional uint32 max_batch_size = 5; + optional uint64 max_output_values_per_item = 6; + optional bool supports_priority = 7; + optional bool supports_lora = 8; + repeated Modality modalities = 9; +} + +message ScoreCapabilities { + optional bool supported = 1; + repeated TaskInputType input_types = 2; + repeated TaskOutputGranularity granularities = 3; + repeated TaskValueSemantics semantics = 4; + repeated ScoreNormalization normalizations = 5; + optional bool supports_label_token_scoring = 6; + optional bool supports_instruction = 7; + optional uint32 max_groups = 8; + optional uint32 max_candidates_per_group = 9; + optional uint64 max_output_values_per_candidate = 10; + optional bool supports_priority = 11; + optional bool supports_lora = 12; + optional bool higher_is_better = 13; + repeated Modality modalities = 14; + optional uint32 max_label_token_ids = 15; +} + +message EmbedRequest { + TaskRequestContext context = 1; + repeated TaskInput inputs = 2; + EmbedOptions options = 3; +} + +message EmbedOptions { + optional TaskOutputGranularity granularity = 1; + optional bool normalize = 2; + optional uint32 dimensions = 3; + optional EmbeddingEncoding encoding = 4; +} + +message EmbedResponse { + string request_id = 1; + repeated EmbeddingOutput outputs = 2; + TaskUsage usage = 3; +} + +message EmbeddingOutput { + string item_id = 1; + optional uint32 input_index = 2; + TaskOutputGranularity granularity = 3; + oneof embedding { + DenseFloatTensor dense = 4; + SparseFloatTensor sparse = 5; + } + repeated uint32 token_ids = 6; +} + +message ClassifyRequest { + TaskRequestContext context = 1; + repeated TaskInput inputs = 2; + ClassifyOptions options = 3; +} + +message ClassifyOptions { + optional TaskOutputGranularity granularity = 1; + optional TaskValueSemantics output_semantics = 2; +} + +message ClassifyResponse { + string request_id = 1; + repeated ClassificationOutput outputs = 2; + TaskUsage usage = 3; +} + +message ClassificationOutput { + string item_id = 1; + optional uint32 input_index = 2; + TaskOutputGranularity granularity = 3; + TaskValueSemantics semantics = 4; + DenseFloatTensor scores = 5; + repeated string labels = 6; + repeated uint32 token_ids = 7; +} + +message ScoreRequest { + TaskRequestContext context = 1; + repeated ScoreGroup groups = 2; + ScoreOptions options = 3; +} + +// A query and one or more candidates scored as one correlation group. +message ScoreGroup { + string group_id = 1; + TaskInput query = 2; + repeated TaskInput candidates = 3; +} + +message ScoreOptions { + optional TaskOutputGranularity granularity = 1; + optional TaskValueSemantics output_semantics = 2; + ScoreNormalization normalization = 3; + repeated uint32 label_token_ids = 4; + string instruction = 5; +} + +message ScoreResponse { + string request_id = 1; + repeated ScoreGroupOutput groups = 2; + TaskOutputGranularity granularity = 3; + TaskValueSemantics semantics = 4; + optional bool higher_is_better = 5; + ScoreNormalization normalization = 6; + repeated uint32 label_token_ids = 7; + TaskUsage usage = 8; +} + +message ScoreGroupOutput { + string group_id = 1; + optional uint32 group_index = 2; + repeated ScoreCandidateOutput candidates = 3; +} + +message ScoreCandidateOutput { + string candidate_id = 1; + optional uint32 candidate_index = 2; + DenseFloatTensor scores = 3; + repeated uint32 token_ids = 4; +} From 7688b68cd5151f11645d71c89840f57fb0dd0e20 Mon Sep 17 00:00:00 2001 From: Connor Carpenter Date: Sun, 12 Jul 2026 10:11:55 -0700 Subject: [PATCH 2/3] refactor(proto): split inference task domains Signed-off-by: Connor Carpenter --- proto/openengine/v1/README.md | 8 +- proto/openengine/v1/classification.proto | 50 +++++ proto/openengine/v1/embedding.proto | 69 +++++++ proto/openengine/v1/generation.proto | 24 +-- proto/openengine/v1/generation_params.proto | 5 +- proto/openengine/v1/input.proto | 61 ++++++ proto/openengine/v1/model.proto | 10 +- proto/openengine/v1/openengine.proto | 4 +- proto/openengine/v1/scoring.proto | 80 ++++++++ proto/openengine/v1/tasks.proto | 204 +------------------- 10 files changed, 281 insertions(+), 234 deletions(-) create mode 100644 proto/openengine/v1/classification.proto create mode 100644 proto/openengine/v1/embedding.proto create mode 100644 proto/openengine/v1/input.proto create mode 100644 proto/openengine/v1/scoring.proto diff --git a/proto/openengine/v1/README.md b/proto/openengine/v1/README.md index 084772e..ab7eaef 100644 --- a/proto/openengine/v1/README.md +++ b/proto/openengine/v1/README.md @@ -11,11 +11,15 @@ share the same package and together define the API. | File | Area | | --- | --- | | [`openengine.proto`](openengine.proto) | `OpenEngine` service and RPC declarations | +| [`input.proto`](input.proto) | Shared text, token, and multimodal inputs | | [`engine.proto`](engine.proto) | Engine identity, roles, and parallelism | -| [`model.proto`](model.proto) | Model metadata and generation capabilities | +| [`model.proto`](model.proto) | Model metadata and inference capabilities | | [`generation.proto`](generation.proto) | Generation requests, streamed events, and usage | | [`generation_params.proto`](generation_params.proto) | Sampling, stopping, response, KV, and guided-decoding parameters | -| [`tasks.proto`](tasks.proto) | Embedding, classification, scoring, and task capability discovery | +| [`tasks.proto`](tasks.proto) | Shared non-generative task request and output vocabulary | +| [`embedding.proto`](embedding.proto) | Dense and sparse embedding inference | +| [`classification.proto`](classification.proto) | Sequence and token classification inference | +| [`scoring.proto`](scoring.proto) | Grouped query and candidate scoring inference | | [`lora.proto`](lora.proto) | LoRA adapter lifecycle | | [`kv.proto`](kv.proto) | KV sessions, connector discovery, and cache events | | [`lifecycle.proto`](lifecycle.proto) | Health, abort, and drain operations | diff --git a/proto/openengine/v1/classification.proto b/proto/openengine/v1/classification.proto new file mode 100644 index 0000000..20b65af --- /dev/null +++ b/proto/openengine/v1/classification.proto @@ -0,0 +1,50 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Portable sequence and token classification inference. + +syntax = "proto3"; + +package openengine.v1; + +import "openengine/v1/input.proto"; +import "openengine/v1/tasks.proto"; + +message ClassifyCapabilities { + optional bool supported = 1; + repeated TaskInputType input_types = 2; + repeated TaskOutputGranularity granularities = 3; + repeated TaskValueSemantics semantics = 4; + optional uint32 max_batch_size = 5; + optional uint64 max_output_values_per_item = 6; + optional bool supports_priority = 7; + optional bool supports_lora = 8; + repeated Modality modalities = 9; +} + +message ClassifyRequest { + TaskRequestContext context = 1; + repeated TaskInput inputs = 2; + ClassifyOptions options = 3; +} + +message ClassifyOptions { + optional TaskOutputGranularity granularity = 1; + optional TaskValueSemantics output_semantics = 2; +} + +message ClassifyResponse { + string request_id = 1; + repeated ClassificationOutput outputs = 2; + TaskUsage usage = 3; +} + +message ClassificationOutput { + string item_id = 1; + optional uint32 input_index = 2; + TaskOutputGranularity granularity = 3; + TaskValueSemantics semantics = 4; + DenseFloatTensor scores = 5; + repeated string labels = 6; + repeated uint32 token_ids = 7; +} diff --git a/proto/openengine/v1/embedding.proto b/proto/openengine/v1/embedding.proto new file mode 100644 index 0000000..70b7fe6 --- /dev/null +++ b/proto/openengine/v1/embedding.proto @@ -0,0 +1,69 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Portable dense and sparse embedding inference. + +syntax = "proto3"; + +package openengine.v1; + +import "openengine/v1/input.proto"; +import "openengine/v1/tasks.proto"; + +enum EmbeddingEncoding { + EMBEDDING_ENCODING_UNSPECIFIED = 0; + EMBEDDING_ENCODING_DENSE = 1; + EMBEDDING_ENCODING_SPARSE = 2; +} + +// Row-major sparse FP32 tensor with flattened, strictly increasing indices. +message SparseFloatTensor { + repeated uint64 shape = 1; + repeated uint64 indices = 2; + repeated float values = 3; +} + +message EmbedCapabilities { + optional bool supported = 1; + repeated TaskInputType input_types = 2; + repeated TaskOutputGranularity granularities = 3; + repeated EmbeddingEncoding encodings = 4; + optional uint32 dimension = 5; + optional uint32 max_batch_size = 6; + optional uint64 max_output_values_per_item = 7; + optional bool supports_priority = 8; + optional bool supports_lora = 9; + optional bool supports_normalization = 10; + optional bool supports_dimension_override = 11; + repeated Modality modalities = 12; +} + +message EmbedRequest { + TaskRequestContext context = 1; + repeated TaskInput inputs = 2; + EmbedOptions options = 3; +} + +message EmbedOptions { + optional TaskOutputGranularity granularity = 1; + optional bool normalize = 2; + optional uint32 dimensions = 3; + optional EmbeddingEncoding encoding = 4; +} + +message EmbedResponse { + string request_id = 1; + repeated EmbeddingOutput outputs = 2; + TaskUsage usage = 3; +} + +message EmbeddingOutput { + string item_id = 1; + optional uint32 input_index = 2; + TaskOutputGranularity granularity = 3; + oneof embedding { + DenseFloatTensor dense = 4; + SparseFloatTensor sparse = 5; + } + repeated uint32 token_ids = 6; +} diff --git a/proto/openengine/v1/generation.proto b/proto/openengine/v1/generation.proto index 3d8b50b..ab33446 100644 --- a/proto/openengine/v1/generation.proto +++ b/proto/openengine/v1/generation.proto @@ -9,6 +9,7 @@ package openengine.v1; import "openengine/v1/error.proto"; import "openengine/v1/generation_params.proto"; +import "openengine/v1/input.proto"; import "openengine/v1/kv.proto"; message GenerateRequest { @@ -44,29 +45,6 @@ message GenerateRequest { map metadata = 13; } -// Multimodal modality discriminator. 0 is treated as image for forward -// compatibility with senders that omit the field. -enum Modality { - MODALITY_UNSPECIFIED = 0; - MODALITY_IMAGE = 1; - MODALITY_VIDEO = 2; - MODALITY_AUDIO = 3; -} - -// A single multimodal input. Exactly one `source` should be set. The engine -// owns fetch, decode, and preprocessing, so pre-decoded or RDMA media -// descriptors are not represented here. -message MediaItem { - Modality modality = 1; - oneof source { - string url = 2; // http(s):// -- engine fetches - string data_uri = 3; // data:;base64,<...> -- engine decodes - bytes raw_bytes = 4; // pre-fetched bytes -- engine still preprocesses - } - string mime_type = 5; // optional, hints raw_bytes decode - string uuid = 6; // optional caller id / mm_hash -} - message GenerateResponse { string request_id = 1; diff --git a/proto/openengine/v1/generation_params.proto b/proto/openengine/v1/generation_params.proto index 48143c5..fc19ada 100644 --- a/proto/openengine/v1/generation_params.proto +++ b/proto/openengine/v1/generation_params.proto @@ -7,12 +7,9 @@ syntax = "proto3"; package openengine.v1; +import "openengine/v1/input.proto"; import "openengine/v1/kv.proto"; -message TokenIds { - repeated uint32 ids = 1; -} - message SamplingParams { optional double temperature = 1; optional double top_p = 2; diff --git a/proto/openengine/v1/input.proto b/proto/openengine/v1/input.proto new file mode 100644 index 0000000..5b384c1 --- /dev/null +++ b/proto/openengine/v1/input.proto @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Portable text, token, and multimodal inference inputs. + +syntax = "proto3"; + +package openengine.v1; + +message TokenIds { + repeated uint32 ids = 1; +} + +// Multimodal modality discriminator. 0 is treated as image for forward +// compatibility with senders that omit the field. +enum Modality { + MODALITY_UNSPECIFIED = 0; + MODALITY_IMAGE = 1; + MODALITY_VIDEO = 2; + MODALITY_AUDIO = 3; +} + +// A single multimodal input. Exactly one `source` should be set. The engine +// owns fetch, decode, and preprocessing, so pre-decoded or RDMA media +// descriptors are not represented here. +message MediaItem { + Modality modality = 1; + oneof source { + string url = 2; // http(s):// -- engine fetches + string data_uri = 3; // data:;base64,<...> -- engine decodes + bytes raw_bytes = 4; // pre-fetched bytes -- engine still preprocesses + } + string mime_type = 5; // optional, hints raw_bytes decode + string uuid = 6; // optional caller id / mm_hash +} + +// Text and token prompts may accompany media in a multimodal task input. +message MultimodalTaskInput { + oneof prompt { + string text = 1; + TokenIds token_ids = 2; + } + repeated MediaItem media = 3; +} + +// One independently correlated non-generative task input. +message TaskInput { + string item_id = 1; + oneof input { + string text = 2; + TokenIds token_ids = 3; + MultimodalTaskInput multimodal = 4; + } +} + +enum TaskInputType { + TASK_INPUT_TYPE_UNSPECIFIED = 0; + TASK_INPUT_TYPE_TEXT = 1; + TASK_INPUT_TYPE_TOKEN_IDS = 2; + TASK_INPUT_TYPE_MULTIMODAL = 3; +} diff --git a/proto/openengine/v1/model.proto b/proto/openengine/v1/model.proto index c2cf582..3cfb72c 100644 --- a/proto/openengine/v1/model.proto +++ b/proto/openengine/v1/model.proto @@ -7,7 +7,9 @@ syntax = "proto3"; package openengine.v1; -import "openengine/v1/tasks.proto"; +import "openengine/v1/classification.proto"; +import "openengine/v1/embedding.proto"; +import "openengine/v1/scoring.proto"; message GetModelInfoRequest { string model = 1; @@ -41,6 +43,12 @@ message ModelInfo { TaskCapabilities tasks = 27; } +message TaskCapabilities { + EmbedCapabilities embed = 1; + ClassifyCapabilities classify = 2; + ScoreCapabilities score = 3; +} + message GenerationCapabilities { LogprobCapabilities prompt_logprobs = 1; LogprobCapabilities output_logprobs = 2; diff --git a/proto/openengine/v1/openengine.proto b/proto/openengine/v1/openengine.proto index dd8d566..062d68f 100644 --- a/proto/openengine/v1/openengine.proto +++ b/proto/openengine/v1/openengine.proto @@ -16,6 +16,8 @@ syntax = "proto3"; package openengine.v1; +import "openengine/v1/classification.proto"; +import "openengine/v1/embedding.proto"; import "openengine/v1/engine.proto"; import "openengine/v1/generation.proto"; import "openengine/v1/kv.proto"; @@ -23,7 +25,7 @@ import "openengine/v1/lifecycle.proto"; import "openengine/v1/lora.proto"; import "openengine/v1/model.proto"; import "openengine/v1/observability.proto"; -import "openengine/v1/tasks.proto"; +import "openengine/v1/scoring.proto"; service OpenEngine { // Core inference path. diff --git a/proto/openengine/v1/scoring.proto b/proto/openengine/v1/scoring.proto new file mode 100644 index 0000000..4887913 --- /dev/null +++ b/proto/openengine/v1/scoring.proto @@ -0,0 +1,80 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Portable grouped query and candidate scoring inference. + +syntax = "proto3"; + +package openengine.v1; + +import "openengine/v1/input.proto"; +import "openengine/v1/tasks.proto"; + +enum ScoreNormalization { + SCORE_NORMALIZATION_UNSPECIFIED = 0; + SCORE_NORMALIZATION_NONE = 1; + SCORE_NORMALIZATION_SOFTMAX = 2; +} + +message ScoreCapabilities { + optional bool supported = 1; + repeated TaskInputType input_types = 2; + repeated TaskOutputGranularity granularities = 3; + repeated TaskValueSemantics semantics = 4; + repeated ScoreNormalization normalizations = 5; + optional bool supports_label_token_scoring = 6; + optional bool supports_instruction = 7; + optional uint32 max_groups = 8; + optional uint32 max_candidates_per_group = 9; + optional uint64 max_output_values_per_candidate = 10; + optional bool supports_priority = 11; + optional bool supports_lora = 12; + optional bool higher_is_better = 13; + repeated Modality modalities = 14; + optional uint32 max_label_token_ids = 15; +} + +message ScoreRequest { + TaskRequestContext context = 1; + repeated ScoreGroup groups = 2; + ScoreOptions options = 3; +} + +// A query and one or more candidates scored as one correlation group. +message ScoreGroup { + string group_id = 1; + TaskInput query = 2; + repeated TaskInput candidates = 3; +} + +message ScoreOptions { + optional TaskOutputGranularity granularity = 1; + optional TaskValueSemantics output_semantics = 2; + ScoreNormalization normalization = 3; + repeated uint32 label_token_ids = 4; + string instruction = 5; +} + +message ScoreResponse { + string request_id = 1; + repeated ScoreGroupOutput groups = 2; + TaskOutputGranularity granularity = 3; + TaskValueSemantics semantics = 4; + optional bool higher_is_better = 5; + ScoreNormalization normalization = 6; + repeated uint32 label_token_ids = 7; + TaskUsage usage = 8; +} + +message ScoreGroupOutput { + string group_id = 1; + optional uint32 group_index = 2; + repeated ScoreCandidateOutput candidates = 3; +} + +message ScoreCandidateOutput { + string candidate_id = 1; + optional uint32 candidate_index = 2; + DenseFloatTensor scores = 3; + repeated uint32 token_ids = 4; +} diff --git a/proto/openengine/v1/tasks.proto b/proto/openengine/v1/tasks.proto index 5a1c7c5..2338885 100644 --- a/proto/openengine/v1/tasks.proto +++ b/proto/openengine/v1/tasks.proto @@ -1,15 +1,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -// Portable non-generative inference tasks and model capabilities. +// Shared request and output vocabulary for non-generative inference tasks. syntax = "proto3"; package openengine.v1; -import "openengine/v1/generation.proto"; -import "openengine/v1/generation_params.proto"; - // Request metadata shared by every non-generative task. message TaskRequestContext { string request_id = 1; @@ -19,63 +16,24 @@ message TaskRequestContext { map metadata = 5; } -// Text and token prompts may accompany media in a multimodal task input. -message MultimodalTaskInput { - oneof prompt { - string text = 1; - TokenIds token_ids = 2; - } - repeated MediaItem media = 3; -} - -// One independently correlated task input. -message TaskInput { - string item_id = 1; - oneof input { - string text = 2; - TokenIds token_ids = 3; - MultimodalTaskInput multimodal = 4; - } -} - // Row-major FP32 tensor. The product of shape must equal values.size(). message DenseFloatTensor { repeated uint64 shape = 1; repeated float values = 2; } -// Row-major sparse FP32 tensor with flattened, strictly increasing indices. -message SparseFloatTensor { - repeated uint64 shape = 1; - repeated uint64 indices = 2; - repeated float values = 3; -} - // Token accounting for one complete unary task request. message TaskUsage { uint64 input_tokens = 1; optional uint64 cached_input_tokens = 2; } -enum TaskInputType { - TASK_INPUT_TYPE_UNSPECIFIED = 0; - TASK_INPUT_TYPE_TEXT = 1; - TASK_INPUT_TYPE_TOKEN_IDS = 2; - TASK_INPUT_TYPE_MULTIMODAL = 3; -} - enum TaskOutputGranularity { TASK_OUTPUT_GRANULARITY_UNSPECIFIED = 0; TASK_OUTPUT_GRANULARITY_SEQUENCE = 1; TASK_OUTPUT_GRANULARITY_TOKEN = 2; } -enum EmbeddingEncoding { - EMBEDDING_ENCODING_UNSPECIFIED = 0; - EMBEDDING_ENCODING_DENSE = 1; - EMBEDDING_ENCODING_SPARSE = 2; -} - enum TaskValueSemantics { TASK_VALUE_SEMANTICS_UNSPECIFIED = 0; TASK_VALUE_SEMANTICS_LOGITS = 1; @@ -86,163 +44,3 @@ enum TaskValueSemantics { TASK_VALUE_SEMANTICS_REWARD = 6; TASK_VALUE_SEMANTICS_MODEL_DEFINED = 7; } - -enum ScoreNormalization { - SCORE_NORMALIZATION_UNSPECIFIED = 0; - SCORE_NORMALIZATION_NONE = 1; - SCORE_NORMALIZATION_SOFTMAX = 2; -} - -// Per-model support and limits for non-generative tasks. -message TaskCapabilities { - EmbedCapabilities embed = 1; - ClassifyCapabilities classify = 2; - ScoreCapabilities score = 3; -} - -message EmbedCapabilities { - optional bool supported = 1; - repeated TaskInputType input_types = 2; - repeated TaskOutputGranularity granularities = 3; - repeated EmbeddingEncoding encodings = 4; - optional uint32 dimension = 5; - optional uint32 max_batch_size = 6; - optional uint64 max_output_values_per_item = 7; - optional bool supports_priority = 8; - optional bool supports_lora = 9; - optional bool supports_normalization = 10; - optional bool supports_dimension_override = 11; - repeated Modality modalities = 12; -} - -message ClassifyCapabilities { - optional bool supported = 1; - repeated TaskInputType input_types = 2; - repeated TaskOutputGranularity granularities = 3; - repeated TaskValueSemantics semantics = 4; - optional uint32 max_batch_size = 5; - optional uint64 max_output_values_per_item = 6; - optional bool supports_priority = 7; - optional bool supports_lora = 8; - repeated Modality modalities = 9; -} - -message ScoreCapabilities { - optional bool supported = 1; - repeated TaskInputType input_types = 2; - repeated TaskOutputGranularity granularities = 3; - repeated TaskValueSemantics semantics = 4; - repeated ScoreNormalization normalizations = 5; - optional bool supports_label_token_scoring = 6; - optional bool supports_instruction = 7; - optional uint32 max_groups = 8; - optional uint32 max_candidates_per_group = 9; - optional uint64 max_output_values_per_candidate = 10; - optional bool supports_priority = 11; - optional bool supports_lora = 12; - optional bool higher_is_better = 13; - repeated Modality modalities = 14; - optional uint32 max_label_token_ids = 15; -} - -message EmbedRequest { - TaskRequestContext context = 1; - repeated TaskInput inputs = 2; - EmbedOptions options = 3; -} - -message EmbedOptions { - optional TaskOutputGranularity granularity = 1; - optional bool normalize = 2; - optional uint32 dimensions = 3; - optional EmbeddingEncoding encoding = 4; -} - -message EmbedResponse { - string request_id = 1; - repeated EmbeddingOutput outputs = 2; - TaskUsage usage = 3; -} - -message EmbeddingOutput { - string item_id = 1; - optional uint32 input_index = 2; - TaskOutputGranularity granularity = 3; - oneof embedding { - DenseFloatTensor dense = 4; - SparseFloatTensor sparse = 5; - } - repeated uint32 token_ids = 6; -} - -message ClassifyRequest { - TaskRequestContext context = 1; - repeated TaskInput inputs = 2; - ClassifyOptions options = 3; -} - -message ClassifyOptions { - optional TaskOutputGranularity granularity = 1; - optional TaskValueSemantics output_semantics = 2; -} - -message ClassifyResponse { - string request_id = 1; - repeated ClassificationOutput outputs = 2; - TaskUsage usage = 3; -} - -message ClassificationOutput { - string item_id = 1; - optional uint32 input_index = 2; - TaskOutputGranularity granularity = 3; - TaskValueSemantics semantics = 4; - DenseFloatTensor scores = 5; - repeated string labels = 6; - repeated uint32 token_ids = 7; -} - -message ScoreRequest { - TaskRequestContext context = 1; - repeated ScoreGroup groups = 2; - ScoreOptions options = 3; -} - -// A query and one or more candidates scored as one correlation group. -message ScoreGroup { - string group_id = 1; - TaskInput query = 2; - repeated TaskInput candidates = 3; -} - -message ScoreOptions { - optional TaskOutputGranularity granularity = 1; - optional TaskValueSemantics output_semantics = 2; - ScoreNormalization normalization = 3; - repeated uint32 label_token_ids = 4; - string instruction = 5; -} - -message ScoreResponse { - string request_id = 1; - repeated ScoreGroupOutput groups = 2; - TaskOutputGranularity granularity = 3; - TaskValueSemantics semantics = 4; - optional bool higher_is_better = 5; - ScoreNormalization normalization = 6; - repeated uint32 label_token_ids = 7; - TaskUsage usage = 8; -} - -message ScoreGroupOutput { - string group_id = 1; - optional uint32 group_index = 2; - repeated ScoreCandidateOutput candidates = 3; -} - -message ScoreCandidateOutput { - string candidate_id = 1; - optional uint32 candidate_index = 2; - DenseFloatTensor scores = 3; - repeated uint32 token_ids = 4; -} From cd0f3922fb906c698b1857b6958aedb08657e95f Mon Sep 17 00:00:00 2001 From: Connor Carpenter Date: Sun, 12 Jul 2026 10:23:55 -0700 Subject: [PATCH 3/3] refactor(proto): consolidate generation schema Signed-off-by: Connor Carpenter --- proto/openengine/v1/README.md | 3 +- proto/openengine/v1/generation.proto | 76 +++++++++++++++++- proto/openengine/v1/generation_params.proto | 86 --------------------- 3 files changed, 76 insertions(+), 89 deletions(-) delete mode 100644 proto/openengine/v1/generation_params.proto diff --git a/proto/openengine/v1/README.md b/proto/openengine/v1/README.md index ab7eaef..6c97034 100644 --- a/proto/openengine/v1/README.md +++ b/proto/openengine/v1/README.md @@ -14,8 +14,7 @@ share the same package and together define the API. | [`input.proto`](input.proto) | Shared text, token, and multimodal inputs | | [`engine.proto`](engine.proto) | Engine identity, roles, and parallelism | | [`model.proto`](model.proto) | Model metadata and inference capabilities | -| [`generation.proto`](generation.proto) | Generation requests, streamed events, and usage | -| [`generation_params.proto`](generation_params.proto) | Sampling, stopping, response, KV, and guided-decoding parameters | +| [`generation.proto`](generation.proto) | Generation requests, parameters, streamed events, and usage | | [`tasks.proto`](tasks.proto) | Shared non-generative task request and output vocabulary | | [`embedding.proto`](embedding.proto) | Dense and sparse embedding inference | | [`classification.proto`](classification.proto) | Sequence and token classification inference | diff --git a/proto/openengine/v1/generation.proto b/proto/openengine/v1/generation.proto index ab33446..34ed557 100644 --- a/proto/openengine/v1/generation.proto +++ b/proto/openengine/v1/generation.proto @@ -8,7 +8,6 @@ syntax = "proto3"; package openengine.v1; import "openengine/v1/error.proto"; -import "openengine/v1/generation_params.proto"; import "openengine/v1/input.proto"; import "openengine/v1/kv.proto"; @@ -45,6 +44,81 @@ message GenerateRequest { map metadata = 13; } +message SamplingParams { + optional double temperature = 1; + optional double top_p = 2; + optional int32 top_k = 3; + optional double min_p = 4; + optional double frequency_penalty = 5; + optional double presence_penalty = 6; + optional double repetition_penalty = 7; + optional uint64 seed = 8; + optional uint32 num_sequences = 9; +} + +message StoppingOptions { + optional uint32 max_tokens = 1; + optional uint32 min_tokens = 2; + repeated StopCondition conditions = 3; + optional bool ignore_eos = 4; + optional bool include_stop_in_output = 5; +} + +message ResponseOptions { + optional bool return_prompt_logprobs = 1; + CandidateTokenSelection prompt_candidates = 2; + optional bool return_output_logprobs = 3; + CandidateTokenSelection output_candidates = 4; + optional uint32 prompt_logprob_start = 5; +} + +message CandidateTokenSelection { + oneof selection { + uint32 top_n = 1; + TokenIds token_ids = 2; + AllCandidates all = 3; + } +} + +message AllCandidates {} + +message KvOptions { + KvSessionRef session = 1; + optional uint32 data_parallel_rank = 2; + optional bool bypass_prefix_cache = 3; + optional string cache_salt = 4; +} + +message StopCondition { + oneof condition { + string stop_text = 1; + uint32 stop_token_id = 2; + } +} + +// Constrained / guided decoding spec. At most one of `guide` should be set. +// The engine enforces the constraint during sampling via its grammar backend +// (xgrammar / outlines / llguidance); clients cannot apply it post-hoc. +message GuidedDecoding { + oneof guide { + string json_schema = 1; // output conforms to this JSON schema + string regex = 2; // output matches this regex + string ebnf_grammar = 3; // output follows this EBNF / context-free grammar + string structural_tag = 4; // xgrammar structural-tag constraint (JSON string) + ChoiceConstraint choice = 5; + JsonObjectConstraint json_object = 6; + } + // Optional grammar backend override (e.g. "xgrammar", "outlines", + // "llguidance"). Empty = engine default. + string backend = 7; +} + +message ChoiceConstraint { + repeated string choices = 1; +} + +message JsonObjectConstraint {} + message GenerateResponse { string request_id = 1; diff --git a/proto/openengine/v1/generation_params.proto b/proto/openengine/v1/generation_params.proto deleted file mode 100644 index fc19ada..0000000 --- a/proto/openengine/v1/generation_params.proto +++ /dev/null @@ -1,86 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -// Portable generation parameters and reusable selection types. - -syntax = "proto3"; - -package openengine.v1; - -import "openengine/v1/input.proto"; -import "openengine/v1/kv.proto"; - -message SamplingParams { - optional double temperature = 1; - optional double top_p = 2; - optional int32 top_k = 3; - optional double min_p = 4; - optional double frequency_penalty = 5; - optional double presence_penalty = 6; - optional double repetition_penalty = 7; - optional uint64 seed = 8; - optional uint32 num_sequences = 9; -} - -message StoppingOptions { - optional uint32 max_tokens = 1; - optional uint32 min_tokens = 2; - repeated StopCondition conditions = 3; - optional bool ignore_eos = 4; - optional bool include_stop_in_output = 5; -} - -message ResponseOptions { - optional bool return_prompt_logprobs = 1; - CandidateTokenSelection prompt_candidates = 2; - optional bool return_output_logprobs = 3; - CandidateTokenSelection output_candidates = 4; - optional uint32 prompt_logprob_start = 5; -} - -message CandidateTokenSelection { - oneof selection { - uint32 top_n = 1; - TokenIds token_ids = 2; - AllCandidates all = 3; - } -} - -message AllCandidates {} - -message KvOptions { - KvSessionRef session = 1; - optional uint32 data_parallel_rank = 2; - optional bool bypass_prefix_cache = 3; - optional string cache_salt = 4; -} - -message StopCondition { - oneof condition { - string stop_text = 1; - uint32 stop_token_id = 2; - } -} - -// Constrained / guided decoding spec. At most one of `guide` should be set. -// The engine enforces the constraint during sampling via its grammar backend -// (xgrammar / outlines / llguidance); clients cannot apply it post-hoc. -message GuidedDecoding { - oneof guide { - string json_schema = 1; // output conforms to this JSON schema - string regex = 2; // output matches this regex - string ebnf_grammar = 3; // output follows this EBNF / context-free grammar - string structural_tag = 4; // xgrammar structural-tag constraint (JSON string) - ChoiceConstraint choice = 5; - JsonObjectConstraint json_object = 6; - } - // Optional grammar backend override (e.g. "xgrammar", "outlines", - // "llguidance"). Empty = engine default. - string backend = 7; -} - -message ChoiceConstraint { - repeated string choices = 1; -} - -message JsonObjectConstraint {}