Add internal text embedding subsystem for embedding text

schemas/dab.draft.schema.json

@@ -642,6 +642,145 @@
               "default": 4
             }
           }
+        },
+        "embeddings": {
+          "type": "object",
+          "description": "Configuration for text embedding/vectorization service. Supports OpenAI and Azure OpenAI providers.",
+          "additionalProperties": false,
+          "properties": {
+            "enabled": {
+              "type": "boolean",
+              "description": "Whether the embedding service is enabled. Defaults to true.",
+              "default": true
+            },
+            "provider": {
+              "type": "string",
+              "description": "The embedding provider type.",
+              "enum": ["azure-openai", "openai"]
+            },
+            "base-url": {
+              "type": "string",
+              "description": "The provider base URL. For Azure OpenAI, use the Azure resource endpoint. For OpenAI, use https://api.openai.com."
+            },
+            "api-key": {
+              "type": "string",
+              "description": "The API key for authentication. Supports environment variable substitution with @env('VAR_NAME')."
+            },
+            "model": {
+              "type": "string",
+              "description": "The model or deployment name. Required for Azure OpenAI (deployment name). For OpenAI, defaults to 'text-embedding-3-small' if not specified."
+            },
+            "api-version": {
+              "type": "string",
+              "description": "Azure API version. Only used for Azure OpenAI provider.",
+              "default": "2024-02-01"
+            },
+            "dimensions": {
+              "type": "integer",
+              "description": "Output vector dimensions. Optional, uses model default if not specified. Useful for Redis schema alignment.",
+              "minimum": 1
+            },
+            "timeout-ms": {
+              "type": "integer",
+              "description": "Request timeout in milliseconds.",
+              "default": 30000,
+              "minimum": 1,
+              "maximum": 300000
+            },
+            "endpoint": {
+              "type": "object",
+              "description": "REST endpoint configuration for the embedding service.",
+              "additionalProperties": false,
+              "properties": {
+                "enabled": {
+                  "type": "boolean",
+                  "description": "Whether the /embed REST endpoint is enabled. Defaults to false.",
+                  "default": false
+                },
+                "path": {
+                  "type": "string",
+                  "description": "The endpoint path. Defaults to '/embed'.",
+                  "default": "/embed"
+                },
+                "roles": {
+                  "type": "array",
+                  "description": "The roles allowed to access the embedding endpoint. In development mode, defaults to ['anonymous'].",
+                  "items": {
+                    "type": "string"
+                  }
+                }
+              }
+            },
+            "health": {
+              "type": "object",
+              "description": "Health check configuration for the embedding service.",
+              "additionalProperties": false,
+              "properties": {
+                "enabled": {
+                  "type": "boolean",
+                  "description": "Whether health checks are enabled for embeddings. Defaults to true.",
+                  "default": true
+                },
+                "threshold-ms": {
+                  "type": "integer",
+                  "description": "The maximum response time in milliseconds to be considered healthy.",
+                  "default": 5000,
+                  "minimum": 1,
+                  "maximum": 300000
+                },
+                "test-text": {
+                  "type": "string",
+                  "description": "The text to use for health check validation.",
+                  "default": "health check"
+                },
+                "expected-dimensions": {
+                  "type": "integer",
+                  "description": "The expected number of dimensions in the embedding result. If specified, dimension validation is performed.",
+                  "minimum": 1
+                }
+              }
+            }
+          },
+          "required": ["provider", "base-url", "api-key"],
+          "allOf": [
+            {
+              "$comment": "Azure OpenAI requires the model (deployment name) to be specified.",
+              "if": {
+                "properties": {
+                  "provider": {
+                    "const": "azure-openai"
+                  }
+                },
+                "required": ["provider"]
+              },
+              "then": {
+                "required": ["model"],
+                "properties": {
+                  "api-version": {
+                    "type": "string",
+                    "description": "Azure API version. Required for Azure OpenAI provider.",
+                    "default": "2024-02-01"
+                  }
+                }
+              }
+            },
+            {
+              "$comment": "OpenAI does not require model (defaults to text-embedding-3-small) and does not use api-version.",
+              "if": {
+                "properties": {
+                  "provider": {
+                    "const": "openai"
+                  }
+                },
+                "required": ["provider"]
+              },
+              "then": {
+                "properties": {
+                  "api-version": false
+                }
+              }
+            }
+          ]
         }
       }
     },

src/Cli/Commands/ConfigureOptions.cs

@@ -4,6 +4,7 @@
 using System.IO.Abstractions;
 using Azure.DataApiBuilder.Config;
 using Azure.DataApiBuilder.Config.ObjectModel;
+using Azure.DataApiBuilder.Config.ObjectModel.Embeddings;
 using Azure.DataApiBuilder.Product;
 using Cli.Constants;
 using CommandLine;
@@ -71,6 +72,21 @@ public ConfigureOptions(
             RollingInterval? fileSinkRollingInterval = null,
             int? fileSinkRetainedFileCountLimit = null,
             long? fileSinkFileSizeLimitBytes = null,
+            CliBool? runtimeEmbeddingsEnabled = null,
+            EmbeddingProviderType? runtimeEmbeddingsProvider = null,
+            string? runtimeEmbeddingsBaseUrl = null,
+            string? runtimeEmbeddingsApiKey = null,
+            string? runtimeEmbeddingsModel = null,
+            string? runtimeEmbeddingsApiVersion = null,
+            int? runtimeEmbeddingsDimensions = null,
+            int? runtimeEmbeddingsTimeoutMs = null,
+            CliBool? runtimeEmbeddingsEndpointEnabled = null,
+            string? runtimeEmbeddingsEndpointPath = null,
+            IEnumerable<string>? runtimeEmbeddingsEndpointRoles = null,
+            CliBool? runtimeEmbeddingsHealthEnabled = null,
+            int? runtimeEmbeddingsHealthThresholdMs = null,
+            string? runtimeEmbeddingsHealthTestText = null,
+            int? runtimeEmbeddingsHealthExpectedDimensions = null,
             string? config = null)
             : base(config)
         {
@@ -132,6 +148,24 @@ public ConfigureOptions(
             FileSinkRollingInterval = fileSinkRollingInterval;
             FileSinkRetainedFileCountLimit = fileSinkRetainedFileCountLimit;
             FileSinkFileSizeLimitBytes = fileSinkFileSizeLimitBytes;
+            // Embeddings
+            RuntimeEmbeddingsEnabled = runtimeEmbeddingsEnabled;
+            RuntimeEmbeddingsProvider = runtimeEmbeddingsProvider;
+            RuntimeEmbeddingsBaseUrl = runtimeEmbeddingsBaseUrl;
+            RuntimeEmbeddingsApiKey = runtimeEmbeddingsApiKey;
+            RuntimeEmbeddingsModel = runtimeEmbeddingsModel;
+            RuntimeEmbeddingsApiVersion = runtimeEmbeddingsApiVersion;
+            RuntimeEmbeddingsDimensions = runtimeEmbeddingsDimensions;
+            RuntimeEmbeddingsTimeoutMs = runtimeEmbeddingsTimeoutMs;
+            // Embeddings Endpoint
+            RuntimeEmbeddingsEndpointEnabled = runtimeEmbeddingsEndpointEnabled;
+            RuntimeEmbeddingsEndpointPath = runtimeEmbeddingsEndpointPath;
+            RuntimeEmbeddingsEndpointRoles = runtimeEmbeddingsEndpointRoles;
+            // Embeddings Health
+            RuntimeEmbeddingsHealthEnabled = runtimeEmbeddingsHealthEnabled;
+            RuntimeEmbeddingsHealthThresholdMs = runtimeEmbeddingsHealthThresholdMs;
+            RuntimeEmbeddingsHealthTestText = runtimeEmbeddingsHealthTestText;
+            RuntimeEmbeddingsHealthExpectedDimensions = runtimeEmbeddingsHealthExpectedDimensions;
         }
 
         [Option("data-source.database-type", Required = false, HelpText = "Database type. Allowed values: MSSQL, PostgreSQL, CosmosDB_NoSQL, MySQL.")]
@@ -281,6 +315,51 @@ public ConfigureOptions(
         [Option("runtime.telemetry.file.file-size-limit-bytes", Required = false, HelpText = "Configure maximum file size limit in bytes. Default: 1048576")]
         public long? FileSinkFileSizeLimitBytes { get; }
 
+        [Option("runtime.embeddings.enabled", Required = false, HelpText = "Enable/disable the embedding service. Default: true")]
+        public CliBool? RuntimeEmbeddingsEnabled { get; }
+
+        [Option("runtime.embeddings.provider", Required = false, HelpText = "Configure embedding provider type. Allowed values: azure-openai, openai.")]
+        public EmbeddingProviderType? RuntimeEmbeddingsProvider { get; }
+
+        [Option("runtime.embeddings.base-url", Required = false, HelpText = "Configure the embedding provider base URL.")]
+        public string? RuntimeEmbeddingsBaseUrl { get; }
+
+        [Option("runtime.embeddings.api-key", Required = false, HelpText = "Configure the embedding API key for authentication.")]
+        public string? RuntimeEmbeddingsApiKey { get; }
+
+        [Option("runtime.embeddings.model", Required = false, HelpText = "Configure the model/deployment name. Required for Azure OpenAI, defaults to text-embedding-3-small for OpenAI.")]
+        public string? RuntimeEmbeddingsModel { get; }
+
+        [Option("runtime.embeddings.api-version", Required = false, HelpText = "Configure the Azure API version. Only used for Azure OpenAI provider. Default: 2024-02-01")]
+        public string? RuntimeEmbeddingsApiVersion { get; }
+
+        [Option("runtime.embeddings.dimensions", Required = false, HelpText = "Configure the output vector dimensions. Optional, uses model default if not specified.")]
+        public int? RuntimeEmbeddingsDimensions { get; }
+
+        [Option("runtime.embeddings.timeout-ms", Required = false, HelpText = "Configure the request timeout in milliseconds. Default: 30000")]
+        public int? RuntimeEmbeddingsTimeoutMs { get; }
+
+        [Option("runtime.embeddings.endpoint.enabled", Required = false, HelpText = "Enable/disable the endpoint for embeddings. Default: false")]
+        public CliBool? RuntimeEmbeddingsEndpointEnabled { get; }
+
+        [Option("runtime.embeddings.endpoint.path", Required = false, HelpText = "Configure the endpoint path for embeddings. Default: /embed")]
+        public string? RuntimeEmbeddingsEndpointPath { get; }
+
+        [Option("runtime.embeddings.endpoint.roles", Required = false, Separator = ',', HelpText = "Configure the roles allowed to access the embedding endpoint. Comma-separated list. In development mode defaults to 'anonymous'.")]
+        public IEnumerable<string>? RuntimeEmbeddingsEndpointRoles { get; }
+
+        [Option("runtime.embeddings.health.enabled", Required = false, HelpText = "Enable/disable health checks for the embedding service. Default: true")]
+        public CliBool? RuntimeEmbeddingsHealthEnabled { get; }
+
+        [Option("runtime.embeddings.health.threshold-ms", Required = false, HelpText = "Configure the health check threshold in milliseconds. Default: 5000")]
+        public int? RuntimeEmbeddingsHealthThresholdMs { get; }
+
+        [Option("runtime.embeddings.health.test-text", Required = false, HelpText = "Configure the test text for health check validation. Default: 'health check'")]
+        public string? RuntimeEmbeddingsHealthTestText { get; }
+
+        [Option("runtime.embeddings.health.expected-dimensions", Required = false, HelpText = "Configure the expected dimensions for health check validation. Optional.")]
+        public int? RuntimeEmbeddingsHealthExpectedDimensions { get; }
+
         public int Handler(ILogger logger, FileSystemRuntimeConfigLoader loader, IFileSystem fileSystem)
         {
             logger.LogInformation("{productName} {version}", PRODUCT_NAME, ProductInfo.GetProductVersion());

src/Cli/ConfigGenerator.cs

@@ -8,6 +8,7 @@
 using Azure.DataApiBuilder.Config.Converters;
 using Azure.DataApiBuilder.Config.NamingPolicies;
 using Azure.DataApiBuilder.Config.ObjectModel;
+using Azure.DataApiBuilder.Config.ObjectModel.Embeddings;
 using Azure.DataApiBuilder.Core;
 using Azure.DataApiBuilder.Core.Configurations;
 using Azure.DataApiBuilder.Service;
@@ -908,6 +909,27 @@ options.FileSinkRetainedFileCountLimit is not null ||
                 }
             }
 
+            // Embeddings: Provider, Endpoint, ApiKey, Model, ApiVersion, Dimensions, TimeoutMs, Enabled
+            if (options.RuntimeEmbeddingsProvider is not null ||
+                options.RuntimeEmbeddingsBaseUrl is not null ||
+                options.RuntimeEmbeddingsApiKey is not null ||
+                options.RuntimeEmbeddingsModel is not null ||
+                options.RuntimeEmbeddingsApiVersion is not null ||
+                options.RuntimeEmbeddingsDimensions is not null ||
+                options.RuntimeEmbeddingsTimeoutMs is not null ||
+                options.RuntimeEmbeddingsEnabled is not null)
+            {
+                bool status = TryUpdateConfiguredEmbeddingsValues(options, runtimeConfig?.Runtime?.Embeddings, out EmbeddingsOptions? updatedEmbeddingsOptions);
+                if (status && updatedEmbeddingsOptions is not null)
+                {
+                    runtimeConfig = runtimeConfig! with { Runtime = runtimeConfig.Runtime! with { Embeddings = updatedEmbeddingsOptions } };
+                }
+                else
+                {
+                    return false;
+                }
+            }
+
             return runtimeConfig != null;
         }
 
@@ -1522,6 +1544,97 @@ private static bool TryUpdateConfiguredFileOptions(
             }
         }
 
+        /// <summary>
+        /// Attempts to update the embeddings configuration based on the provided options.
+        /// Creates a new EmbeddingsOptions object if the configuration is valid.
+        /// Provider, endpoint, and API key are required when configuring embeddings.
+        /// </summary>
+        /// <param name="options">The configuration options provided by the user.</param>
+        /// <param name="existingEmbeddingsOptions">The existing embeddings options from the runtime configuration.</param>
+        /// <param name="updatedEmbeddingsOptions">The resulting embeddings options if successful.</param>
+        /// <returns>True if the embeddings options were successfully configured; otherwise, false.</returns>
+        private static bool TryUpdateConfiguredEmbeddingsValues(
+            ConfigureOptions options,
+            EmbeddingsOptions? existingEmbeddingsOptions,
+            out EmbeddingsOptions? updatedEmbeddingsOptions)
+        {
+            updatedEmbeddingsOptions = null;
+
+            try
+            {
+                // Get values from options or fall back to existing configuration
+                EmbeddingProviderType? provider = options.RuntimeEmbeddingsProvider ?? existingEmbeddingsOptions?.Provider;
+                string? baseUrl = options.RuntimeEmbeddingsBaseUrl ?? existingEmbeddingsOptions?.BaseUrl;
+                string? apiKey = options.RuntimeEmbeddingsApiKey ?? existingEmbeddingsOptions?.ApiKey;
+                string? model = options.RuntimeEmbeddingsModel ?? existingEmbeddingsOptions?.Model;
+                string? apiVersion = options.RuntimeEmbeddingsApiVersion ?? existingEmbeddingsOptions?.ApiVersion;
+                int? dimensions = options.RuntimeEmbeddingsDimensions ?? existingEmbeddingsOptions?.Dimensions;
+                int? timeoutMs = options.RuntimeEmbeddingsTimeoutMs ?? existingEmbeddingsOptions?.TimeoutMs;
+                bool? enabled = options.RuntimeEmbeddingsEnabled.HasValue
+                    ? options.RuntimeEmbeddingsEnabled.Value == CliBool.True
+                    : existingEmbeddingsOptions?.Enabled;
+
+                // Validate required fields
+                if (provider is null)
+                {
+                    _logger.LogError("Failed to configure embeddings: provider is required. Use --runtime.embeddings.provider to specify the provider (azure-openai or openai).");
+                    return false;
+                }
+
+                if (string.IsNullOrEmpty(baseUrl))
+                {
+                    _logger.LogError("Failed to configure embeddings: base-url is required. Use --runtime.embeddings.base-url to specify the provider base URL.");
+                    return false;
+                }
+
+                if (string.IsNullOrEmpty(apiKey))
+                {
+                    _logger.LogError("Failed to configure embeddings: api-key is required. Use --runtime.embeddings.api-key to specify the authentication key.");
+                    return false;
+                }
+
+                // Validate Azure OpenAI requires model/deployment name
+                if (provider == EmbeddingProviderType.AzureOpenAI && string.IsNullOrEmpty(model))
+                {
+                    _logger.LogError("Failed to configure embeddings: model/deployment name is required for Azure OpenAI provider. Use --runtime.embeddings.model to specify the deployment name.");
+                    return false;
+                }
+
+                // Validate dimensions if provided
+                if (dimensions is not null && dimensions <= 0)
+                {
+                    _logger.LogError("Failed to configure embeddings: dimensions must be a positive integer.");
+                    return false;
+                }
+
+                // Validate timeout if provided
+                if (timeoutMs is not null && timeoutMs <= 0)
+                {
+                    _logger.LogError("Failed to configure embeddings: timeout-ms must be a positive integer.");
+                    return false;
+                }
+
+                // Create the embeddings options
+                updatedEmbeddingsOptions = new EmbeddingsOptions(
+                    Provider: (EmbeddingProviderType)provider,
+                    BaseUrl: baseUrl,
+                    ApiKey: apiKey,
+                    Enabled: enabled,
+                    Model: model,
+                    ApiVersion: apiVersion,
+                    Dimensions: dimensions,
+                    TimeoutMs: timeoutMs);
+
+                _logger.LogInformation("Updated RuntimeConfig with Runtime.Embeddings configuration.");
+                return true;
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError("Failed to update RuntimeConfig.Embeddings with exception message: {exceptionMessage}.", ex.Message);
+                return false;
+            }
+        }
+
         /// <summary>
         /// Parse permission string to create PermissionSetting array.
         /// </summary>