Let a provider add document declarations and bind the indexing chain

config-model-api/abi-spec.json

@@ -886,6 +886,64 @@
     ],
     "fields" : [ ]
   },
+  "com.yahoo.config.model.api.AdditionalContent$DocumentTypeDeclaration" : {
+    "superClass" : "java.lang.Record",
+    "interfaces" : [ ],
+    "attributes" : [
+      "public",
+      "final",
+      "record"
+    ],
+    "methods" : [
+      "public void <init>(java.lang.String, com.yahoo.config.model.api.AdditionalContent$Mode, boolean)",
+      "public final java.lang.String toString()",
+      "public final int hashCode()",
+      "public final boolean equals(java.lang.Object)",
+      "public java.lang.String type()",
+      "public com.yahoo.config.model.api.AdditionalContent$Mode mode()",
+      "public boolean global()"
+    ],
+    "fields" : [ ]
+  },
+  "com.yahoo.config.model.api.AdditionalContent$Mode" : {
+    "superClass" : "java.lang.Enum",
+    "interfaces" : [ ],
+    "attributes" : [
+      "public",
+      "final",
+      "enum"
+    ],
+    "methods" : [
+      "public static com.yahoo.config.model.api.AdditionalContent$Mode[] values()",
+      "public static com.yahoo.config.model.api.AdditionalContent$Mode valueOf(java.lang.String)",
+      "public java.lang.String xmlValue()"
+    ],
+    "fields" : [
+      "public static final enum com.yahoo.config.model.api.AdditionalContent$Mode INDEX",
+      "public static final enum com.yahoo.config.model.api.AdditionalContent$Mode STREAMING",
+      "public static final enum com.yahoo.config.model.api.AdditionalContent$Mode STORE_ONLY"
+    ]
+  },
+  "com.yahoo.config.model.api.AdditionalContent" : {
+    "superClass" : "java.lang.Record",
+    "interfaces" : [ ],
+    "attributes" : [
+      "public",
+      "final",
+      "record"
+    ],
+    "methods" : [
+      "public void <init>(java.util.List, java.util.List)",
+      "public static com.yahoo.config.model.api.AdditionalContent none()",
+      "public boolean isEmpty()",
+      "public final java.lang.String toString()",
+      "public final int hashCode()",
+      "public final boolean equals(java.lang.Object)",
+      "public java.util.List schemas()",
+      "public java.util.List documents()"
+    ],
+    "fields" : [ ]
+  },
   "com.yahoo.config.model.api.ApplicationClusterEndpoint$AuthMethod" : {
     "superClass" : "java.lang.Enum",
     "interfaces" : [ ],
@@ -1041,7 +1099,7 @@
     ],
     "fields" : [ ]
   },
-  "com.yahoo.config.model.api.CommerceDiscoverySchemaProvider" : {
+  "com.yahoo.config.model.api.CommerceDiscoveryProvider" : {
     "superClass" : "java.lang.Object",
     "interfaces" : [ ],
     "attributes" : [
@@ -1050,7 +1108,7 @@
       "abstract"
     ],
     "methods" : [
-      "public abstract java.util.List schemas(com.yahoo.config.application.api.ApplicationPackage)"
+      "public abstract com.yahoo.config.model.api.AdditionalContent additionalContent(com.yahoo.config.application.api.ApplicationPackage)"
     ],
     "fields" : [ ]
   },

config-model-api/src/main/java/com/yahoo/config/model/api/AdditionalContent.java

@@ -0,0 +1,97 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.api;
+
+import com.yahoo.io.reader.NamedReader;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * Content to add to an application on top of what its package declares: schemas, and the
+ * content cluster document declarations of those schemas. The two are supplied together
+ * because they must match: every declared type must be one of the supplied schemas, checked
+ * here by schema file name, so an application never gets a declaration of a type it has no
+ * schema for, nor a schema it cannot store.
+ *
+ * <p>The declarations are the programmatic equivalent of {@code <document>} entries in the
+ * {@code <documents>} element of services.xml. They name no content cluster: every content
+ * cluster of the application declares every type listed here, as if each {@code <documents>}
+ * element contained them. Document types the application already declares are left as
+ * declared.</p>
+ *
+ * @author sebasabe
+ */
+public record AdditionalContent(List<NamedReader> schemas, List<DocumentTypeDeclaration> documents) {
+
+    public AdditionalContent {
+        schemas = List.copyOf(schemas);
+        documents = List.copyOf(documents);
+        Set<String> schemaNames = new HashSet<>();
+        for (NamedReader schema : schemas)
+            schemaNames.add(schemaName(schema.getName()));
+        List<String> undeclared = new ArrayList<>();
+        for (DocumentTypeDeclaration declaration : documents)
+            if ( ! schemaNames.contains(declaration.type())) undeclared.add(declaration.type());
+        if ( ! undeclared.isEmpty())
+            throw new IllegalArgumentException("Additional document declarations " + undeclared + " have no schema among " +
+                                               "the additional schemas " + schemaNames.stream().sorted().toList() +
+                                               ": a declared type must have a schema file named '<type>.sd' supplied with it");
+    }
+
+    /**
+     * The schema name a schema file name denotes: the name without directories and extension, so
+     * {@code schemas/product.sd} names {@code product}. The same rule the schema parser applies
+     * when it requires the file name to match the schema it contains.
+     */
+    private static String schemaName(String fileName) {
+        int slash = fileName.lastIndexOf('/');
+        if (slash != -1) fileName = fileName.substring(slash + 1);
+        int dot = fileName.lastIndexOf('.');
+        if (dot != -1) fileName = fileName.substring(0, dot);
+        return fileName;
+    }
+
+    /**
+     * The equivalent of a {@code <document type="..." mode="..." global="..."/>} declaration
+     * in a content cluster's {@code <documents>} element. The {@code selection} attribute is not
+     * carried: a provider adds a type to have it stored, not a subset of it.
+     */
+    public record DocumentTypeDeclaration(String type, Mode mode, boolean global) {
+
+        public DocumentTypeDeclaration {
+            if (type == null || type.isBlank())
+                throw new IllegalArgumentException("A document type declaration must name a type");
+            Objects.requireNonNull(mode, "A document type declaration must have a mode");
+        }
+
+    }
+
+    /**
+     * The values of the {@code mode} attribute of {@code <document>}. Declarations added through this
+     * record bypass the services.xml schema validation hand-written ones get, so the values are fixed
+     * here instead. Mirrors {@code attribute mode} in {@code content.rnc} and {@code SchemaInfo.IndexMode}
+     * in config-model; AdditionalContentTest there should fail if they drift.
+     */
+    public enum Mode {
+
+        INDEX("index"), STREAMING("streaming"), STORE_ONLY("store-only");
+
+        private final String xmlValue;
+
+        Mode(String xmlValue) { this.xmlValue = xmlValue; }
+
+        /** The value as written in services.xml. */
+        public String xmlValue() { return xmlValue; }
+
+    }
+
+    public static AdditionalContent none() {
+        return new AdditionalContent(List.of(), List.of());
+    }
+
+    public boolean isEmpty() { return schemas.isEmpty() && documents.isEmpty(); }
+
+}

config-model-api/src/main/java/com/yahoo/config/model/api/CommerceDiscoveryProvider.java

@@ -0,0 +1,34 @@
+// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.model.api;
+
+import com.yahoo.config.application.api.ApplicationPackage;
+
+/**
+ * Provides the platform-owned Vespa Commerce Discovery content. Consulted for every application
+ * built in hosted Vespa when the {@code commerce-discovery} feature flag is enabled for the
+ * application — whether the application uses the {@code <commerce-discovery>} services.xml
+ * element is not checked by the caller. Provider implementations must detect it from the
+ * application package. At most one provider may be registered.
+ *
+ * @author sebasabe
+ */
+public interface CommerceDiscoveryProvider {
+
+    /**
+     * Returns the content to add to this application on top of its package: schemas, and the
+     * content cluster document declarations of those schemas. Implementations decide from the
+     * package what to provide: {@link AdditionalContent#none()} for an application that does not
+     * use the {@code <commerce-discovery>} element, and otherwise the content keyed on the
+     * element's version, so element versions can carry different schema families. Schemas and
+     * declarations come in one result because they must match, which
+     * {@link AdditionalContent} checks.
+     *
+     * The declarations apply to every content cluster of the application, so a provider must
+     * either require the application to have a single content cluster, or accept that each cluster
+     * stores every declared type. Version 1.0 of {@code <commerce-discovery>} requires a single
+     * content cluster; an element version supporting several must extend
+     * {@link AdditionalContent.DocumentTypeDeclaration} with a cluster reference.
+     */
+    AdditionalContent additionalContent(ApplicationPackage applicationPackage);
+
+}

config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java

@@ -12,6 +12,7 @@
 import com.yahoo.config.application.api.UnparsedConfigDefinition;
 import com.yahoo.config.application.api.ValidationOverrides;
 import com.yahoo.config.model.ConfigModelContext.ApplicationType;
+import com.yahoo.config.model.api.AdditionalContent;
 import com.yahoo.config.model.api.ConfigDefinitionRepo;
 import com.yahoo.config.model.api.ContainerEndpoint;
 import com.yahoo.config.model.api.EndpointCertificateSecrets;
@@ -33,7 +34,6 @@
 import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.io.IOUtils;
-import com.yahoo.io.reader.NamedReader;
 import com.yahoo.schema.Application;
 import com.yahoo.schema.ApplicationBuilder;
 import com.yahoo.schema.RankProfileRegistry;
@@ -96,6 +96,7 @@ public class DeployState implements ConfigDefinitionStore {
     private final ExecutorService executor;
     private final OnnxModelCost onnxModelCost;
     private final Optional<SidecarProvider> sidecarProvider;
+    private final AdditionalContent additionalContent;
 
     private DeployState(Application application,
                         RankProfileRegistry rankProfileRegistry,
@@ -120,7 +121,8 @@ private DeployState(Application application,
                         Reindexing reindexing,
                         Optional<ValidationOverrides> validationOverrides,
                         OnnxModelCost onnxModelCost,
-                        Optional<SidecarProvider> sidecarProvider) {
+                        Optional<SidecarProvider> sidecarProvider,
+                        AdditionalContent additionalContent) {
         this.logger = deployLogger;
         this.fileRegistry = fileRegistry;
         this.executor = executor;
@@ -150,6 +152,7 @@ private DeployState(Application application,
         this.reindexing = reindexing;
         this.onnxModelCost = onnxModelCost;
         this.sidecarProvider = sidecarProvider;
+        this.additionalContent = additionalContent;
     }
 
     public static HostProvisioner getDefaultModelHostProvisioner(ApplicationPackage applicationPackage) {
@@ -312,6 +315,12 @@ public Optional<String> tlsClientAuthority() {
     /** Returns the sidecar provider to consult when building container clusters, if any. */
     public Optional<SidecarProvider> getSidecarProvider() { return sidecarProvider; }
 
+    /**
+     * Returns the content added to this application on top of its package. Its schemas were added to the
+     * application when this was built; its document declarations are added by the content cluster builders.
+     */
+    public AdditionalContent getAdditionalContent() { return additionalContent; }
+
     public boolean isHostedTenantApplication(ApplicationType type) {
         boolean isTesterApplication = getProperties().applicationId().instance().isTester();
         return isHosted() && type == ApplicationType.DEFAULT && !isTesterApplication
@@ -347,7 +356,7 @@ public static class Builder {
         private Optional<ValidationOverrides> validationOverrides = Optional.empty();
         private OnnxModelCost onnxModelCost = OnnxModelCost.disabled();
         private Optional<SidecarProvider> sidecarProvider = Optional.empty();
-        private List<NamedReader> additionalSchemas = List.of();
+        private AdditionalContent additionalContent = AdditionalContent.none();
 
         public Builder() {}
 
@@ -472,8 +481,12 @@ public Builder sidecarProvider(SidecarProvider sidecarProvider) {
             return this;
         }
 
-        public Builder additionalSchemas(List<NamedReader> schemas) {
-            this.additionalSchemas = List.copyOf(schemas);
+        /**
+         * Sets content to add to the application on top of its package: schemas and their document declarations.
+         * The schemas are added to the application by {@link #build}; the declarations by the content cluster builders.
+         */
+        public Builder additionalContent(AdditionalContent content) {
+            this.additionalContent = Objects.requireNonNull(content, "additional content cannot be null");
             return this;
         }
 
@@ -487,7 +500,7 @@ public DeployState build(ValidationParameters validationParameters) {
             SemanticRules semanticRules = new SemanticRuleBuilder().build(applicationPackage);
             ApplicationBuilder applicationBuilder = new ApplicationBuilder(applicationPackage, fileRegistry, logger, properties,
                                                                             rankProfileRegistry, queryProfiles.getRegistry());
-            additionalSchemas.forEach(applicationBuilder::addSchema);
+            additionalContent.schemas().forEach(applicationBuilder::addSchema);
             Application application = applicationBuilder.build(! validationParameters.ignoreValidationErrors());
             return new DeployState(application,
                                    rankProfileRegistry,
@@ -512,7 +525,8 @@ public DeployState build(ValidationParameters validationParameters) {
                                    reindexing,
                                    validationOverrides,
                                    onnxModelCost,
-                                   sidecarProvider);
+                                   sidecarProvider,
+                                   additionalContent);
         }
 
     }