- Set the complete list of the config where it contains the complete …

…world. - Append where it is intended. This is a workaround for not having a map type in config. Sometimes it works to have multiple producers produce the same config, sometimes not. This tries to make it more deterministic. Currently other tricks have been employed to avoid appending more to the list than intended.
a-dubaj · Mar 6, 2024 · bd1de86 · bd1de86
1 parent 266b744
commit bd1de86
Show file tree

Hide file tree

Showing 18 changed files with 91 additions and 73 deletions.
diff --git a/config-model/src/main/java/com/yahoo/schema/derived/ImportedFields.java b/config-model/src/main/java/com/yahoo/schema/derived/ImportedFields.java
@@ -9,6 +9,8 @@
 import com.yahoo.schema.document.ImportedField;
 import com.yahoo.vespa.config.search.ImportedFieldsConfig;
 
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Optional;
 
 import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isArrayOfSimpleStruct;
@@ -40,56 +42,58 @@ protected String getDerivedName() {
 
     @Override
     public void getConfig(ImportedFieldsConfig.Builder builder) {
+        // Replace
         if (importedFields.isPresent()) {
-            importedFields.get().fields().forEach( (name, field) -> considerField(builder, field));
+            List<ImportedField> imported = new ArrayList<>();
+            importedFields.get().fields().forEach( (name, field) -> considerField(imported, field));
+            builder.attribute(imported.stream().map(ImportedFields::createAttributeBuilder).toList());
         }
     }
 
     private static boolean isNestedFieldName(String fieldName) {
         return fieldName.indexOf('.') != -1;
     }
 
-    private static void considerField(ImportedFieldsConfig.Builder builder, ImportedField field) {
+    private static void considerField(List<ImportedField> importedFields, ImportedField field) {
         if (field instanceof ImportedComplexField) {
-            considerComplexField(builder, (ImportedComplexField) field);
+            considerComplexField(importedFields, (ImportedComplexField) field);
         } else {
-            considerSimpleField(builder, field);
+            considerSimpleField(importedFields, field);
         }
     }
 
-    private static void considerComplexField(ImportedFieldsConfig.Builder builder, ImportedComplexField field) {
+    private static void considerComplexField(List<ImportedField> importedFields, ImportedComplexField field) {
         ImmutableSDField targetField = field.targetField();
         if (GeoPos.isAnyPos(targetField)) {
             // no action needed
         } else if (isArrayOfSimpleStruct(targetField)) {
-            considerNestedFields(builder, field);
+            considerNestedFields(importedFields, field);
         } else if (isMapOfSimpleStruct(targetField)) {
-            considerSimpleField(builder, field.getNestedField("key"));
-            considerNestedFields(builder, field.getNestedField("value"));
+            considerSimpleField(importedFields, field.getNestedField("key"));
+            considerNestedFields(importedFields, field.getNestedField("value"));
         } else if (isMapOfPrimitiveType(targetField)) {
-            considerSimpleField(builder, field.getNestedField("key"));
-            considerSimpleField(builder, field.getNestedField("value"));
+            considerSimpleField(importedFields, field.getNestedField("key"));
+            considerSimpleField(importedFields, field.getNestedField("value"));
         }
     }
 
-    private static void considerNestedFields(ImportedFieldsConfig.Builder builder, ImportedField field) {
-        if (field instanceof ImportedComplexField) {
-            ImportedComplexField complexField = (ImportedComplexField) field;
-            complexField.getNestedFields().forEach(nestedField -> considerSimpleField(builder, nestedField));
+    private static void considerNestedFields(List<ImportedField> importedFields, ImportedField field) {
+        if (field instanceof ImportedComplexField complexField) {
+            complexField.getNestedFields().forEach(nestedField -> considerSimpleField(importedFields, nestedField));
         }
     }
 
-    private static void considerSimpleField(ImportedFieldsConfig.Builder builder, ImportedField field) {
+    private static void considerSimpleField(List<ImportedField> importedFields, ImportedField field) {
         ImmutableSDField targetField = field.targetField();
         String targetFieldName = targetField.getName();
         if (!isNestedFieldName(targetFieldName)) {
             if (targetField.doesAttributing()) {
-                builder.attribute.add(createAttributeBuilder(field));
+                importedFields.add(field);
             }
         } else {
             Attribute attribute = targetField.getAttribute();
             if (attribute != null) {
-                builder.attribute.add(createAttributeBuilder(field));
+                importedFields.add(field);
             }
         }
     }

diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java
@@ -316,6 +316,7 @@ private boolean notInCommands(String index) {
 
     @Override
     public void getConfig(IndexInfoConfig.Builder builder) {
+        // Append
         IndexInfoConfig.Indexinfo.Builder iiB = new IndexInfoConfig.Indexinfo.Builder();
         iiB.name(getName());
         for (IndexCommand command : commands) {

diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexSchema.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexSchema.java
@@ -109,30 +109,33 @@ protected String getDerivedName() {
         return "indexschema";
     }
 
-    @Override
-    public void getConfig(IndexschemaConfig.Builder icB) {
-        for (IndexField f : fields) {
-            IndexschemaConfig.Indexfield.Builder ifB = new IndexschemaConfig.Indexfield.Builder()
+    private static IndexschemaConfig.Indexfield.Builder createIndexFieldConfig(IndexField f) {
+        var ifB = new IndexschemaConfig.Indexfield.Builder()
                 .name(f.getName())
                 .datatype(IndexschemaConfig.Indexfield.Datatype.Enum.valueOf(f.getType()))
                 .prefix(f.hasPrefix())
                 .phrases(false)
                 .positions(true)
                 .interleavedfeatures(f.useInterleavedFeatures());
-            if (!f.getCollectionType().equals("SINGLE")) {
-                ifB.collectiontype(IndexschemaConfig.Indexfield.Collectiontype.Enum.valueOf(f.getCollectionType()));
-            }
-            icB.indexfield(ifB);
-        }
-        for (FieldSet fieldSet : fieldSets.values()) {
-            IndexschemaConfig.Fieldset.Builder fsB = new IndexschemaConfig.Fieldset.Builder()
-                .name(fieldSet.getName());
-            for (String f : fieldSet.getFieldNames()) {
-                fsB.field(new IndexschemaConfig.Fieldset.Field.Builder()
-                        .name(f));
-            }
-            icB.fieldset(fsB);
+        if (!f.getCollectionType().equals("SINGLE")) {
+            ifB.collectiontype(IndexschemaConfig.Indexfield.Collectiontype.Enum.valueOf(f.getCollectionType()));
         }
+        return ifB;
+    }
+
+    private static IndexschemaConfig.Fieldset.Builder createFieldSetConfig(FieldSet fieldSet) {
+        var fsB = new IndexschemaConfig.Fieldset.Builder().name(fieldSet.getName());
+        for (String f : fieldSet.getFieldNames()) {
+            fsB.field(new IndexschemaConfig.Fieldset.Field.Builder().name(f));
+        }
+        return fsB;
+    }
+
+    @Override
+    public void getConfig(IndexschemaConfig.Builder icB) {
+        // Replace
+        icB.indexfield(fields.stream().map(IndexSchema::createIndexFieldConfig).toList());
+        icB.fieldset(fieldSets.values().stream().map(IndexSchema::createFieldSetConfig).toList());
     }
 
     static List<Field> flattenField(Field field) {

diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java
@@ -95,6 +95,7 @@ public String getDerivedName() {
 
     @Override
     public void getConfig(IlscriptsConfig.Builder configBuilder) {
+        // Append
         IlscriptsConfig.Ilscript.Builder ilscriptBuilder = new IlscriptsConfig.Ilscript.Builder();
         ilscriptBuilder.doctype(getName());
         ilscriptBuilder.docfield(docFields);

diff --git a/config-model/src/main/java/com/yahoo/schema/derived/Juniperrc.java b/config-model/src/main/java/com/yahoo/schema/derived/Juniperrc.java
@@ -44,18 +44,21 @@ protected void derive(Schema schema) {
     @Override
     protected String getDerivedName() { return "juniperrc"; }
 
+    private static JuniperrcConfig.Override.Builder createOverride(String name) {
+        return new JuniperrcConfig.Override.Builder()
+                .fieldname(name)
+                .length(64*Mb)
+                .max_matches(1)
+                .min_length(8192)
+                .surround_max(64*Mb);
+    }
+
     @Override
     public void getConfig(JuniperrcConfig.Builder builder) {
-        if (boldingFields.size() != 0) {
+        // Replace
+        if (!boldingFields.isEmpty()) {
             builder.prefix(true);
-            for (String name : boldingFields) {
-                builder.override(new JuniperrcConfig.Override.Builder()
-                    .fieldname(name)
-                    .length(64*Mb)
-                    .max_matches(1)
-                    .min_length(8192)
-                    .surround_max(64*Mb));
-            }
+            builder.override(boldingFields.stream().map(Juniperrc::createOverride).toList());
         }
     }
 

diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java b/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java
@@ -65,6 +65,7 @@ private Map<String, RankProfileInfo> toRankProfiles(Collection<RankProfile> rank
 
     @Override
     public void getConfig(SchemaInfoConfig.Builder builder) {
+        // Append
         var schemaBuilder = new SchemaInfoConfig.Schema.Builder();
         schemaBuilder.name(schema.getName());
         addFieldsConfig(schemaBuilder);

diff --git a/config-model/src/main/java/com/yahoo/schema/derived/Summaries.java b/config-model/src/main/java/com/yahoo/schema/derived/Summaries.java
@@ -41,11 +41,10 @@ public Summaries(Schema schema, DeployLogger deployLogger, ModelContext.FeatureF
 
     @Override
     public void getConfig(SummaryConfig.Builder builder) {
+        // Replace
         builder.defaultsummaryid(summaries.isEmpty() ? -1 : summaries.get(0).hashCode());
         builder.usev8geopositions(useV8GeoPositions);
-        for (SummaryClass summaryClass : summaries) {
-            builder.classes(summaryClass.getSummaryClassConfig());
-        }
+        builder.classes(summaries.stream().map(SummaryClass::getSummaryClassConfig).toList());
     }    
 
 }
diff --git a/config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java b/config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java
@@ -108,12 +108,9 @@ public String getDerivedName() {
 
     @Override
     public void getConfig(VsmfieldsConfig.Builder vsB) {
-        for (StreamingField streamingField : fields.values()) {
-            vsB.fieldspec(streamingField.getFieldSpecConfig());
-        }
-        for (StreamingDocumentType streamingDocType : doctypes.values()) {
-            vsB.documenttype(streamingDocType.getDocTypeConfig());
-        }
+        // Replace
+        vsB.fieldspec(fields.values().stream().map(StreamingField::getFieldSpecConfig).toList());
+        vsB.documenttype(doctypes.values().stream().map(StreamingDocumentType::getDocTypeConfig).toList());
     }
 
     private static boolean isAttributeField(ImmutableSDField field, boolean isStructField, boolean ignoreAttributeAspect) {

diff --git a/config-model/src/main/java/com/yahoo/schema/derived/VsmSummary.java b/config-model/src/main/java/com/yahoo/schema/derived/VsmSummary.java
@@ -96,14 +96,14 @@ public String getDerivedName() {
 
     @Override
     public void getConfig(VsmsummaryConfig.Builder vB) {
-        for (Map.Entry<SummaryField, List<String>> entry : summaryMap.entrySet()) {
-            VsmsummaryConfig.Fieldmap.Builder fmB = new VsmsummaryConfig.Fieldmap.Builder().summary(entry.getKey().getName());
-            for (String field : entry.getValue()) {
-                fmB.document(new VsmsummaryConfig.Fieldmap.Document.Builder().field(field));
-            }
-            fmB.command(VsmsummaryConfig.Fieldmap.Command.Enum.valueOf(entry.getKey().getVsmCommand().toString()));
-            vB.fieldmap(fmB);
-        }
+        // Replace
+        vB.fieldmap(
+                summaryMap.entrySet().stream().map(entry -> new VsmsummaryConfig.Fieldmap.Builder()
+                        .summary(entry.getKey().getName())
+                        .document(entry.getValue().stream().map(field -> new VsmsummaryConfig.Fieldmap.Document.Builder().field(field)).toList())
+                        .command(VsmsummaryConfig.Fieldmap.Command.Enum.valueOf(entry.getKey().getVsmCommand().toString()))
+                ).toList()
+        );
     }
 
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java
@@ -173,12 +173,11 @@ public void getConfig(SchemaInfoConfig.Builder builder) {
     public void getConfig(QrSearchersConfig.Builder builder) {
         for (int i = 0; i < searchClusters.size(); i++) {
             SearchCluster sys = findClusterWithId(searchClusters, i);
-            QrSearchersConfig.Searchcluster.Builder scB = new QrSearchersConfig.Searchcluster.Builder().
-                    name(sys.getClusterName());
+            var scB = new QrSearchersConfig.Searchcluster.Builder().name(sys.getClusterName());
             for (SchemaInfo spec : sys.schemas().values()) {
                 scB.searchdef(spec.fullSchema().getName());
             }
-            scB.rankprofiles(new QrSearchersConfig.Searchcluster.Rankprofiles.Builder().configid(sys.getConfigId()));
+            scB.rankprofiles_configid(sys.getConfigId());
             scB.indexingmode(QrSearchersConfig.Searchcluster.Indexingmode.Enum.valueOf(sys.getIndexingModeName()));
             if ( ! (sys instanceof IndexedSearchCluster)) {
                 scB.storagecluster(new QrSearchersConfig.Searchcluster.Storagecluster.Builder().

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java b/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java
@@ -61,18 +61,25 @@ public String getSchemaName() {
     public DerivedConfiguration getDerivedConfiguration() {
         return derivedCfg;
     }
+    // These methods will append to the config
     @Override public void getConfig(IndexInfoConfig.Builder builder) { derivedCfg.getIndexInfo().getConfig(builder); }
     @Override public void getConfig(IlscriptsConfig.Builder builder) { derivedCfg.getIndexingScript().getConfig(builder); }
+    @Override public void getConfig(SchemaInfoConfig.Builder builder) { derivedCfg.getSchemaInfo().getConfig(builder); }
+
+    // These methods append as multiple databases join config => TODO will loose information - not good
     @Override public void getConfig(AttributesConfig.Builder builder) { derivedCfg.getConfig(builder); }
     @Override public void getConfig(RankProfilesConfig.Builder builder) { derivedCfg.getRankProfileList().getConfig(builder); }
+
+    // These methods append, TODO unknown usage and consequences
     @Override public void getConfig(RankingExpressionsConfig.Builder builder) { derivedCfg.getRankProfileList().getConfig(builder); }
     @Override public void getConfig(RankingConstantsConfig.Builder builder) { derivedCfg.getRankProfileList().getConfig(builder); }
     @Override public void getConfig(OnnxModelsConfig.Builder builder) { derivedCfg.getRankProfileList().getConfig(builder); }
+
+    // Below methods will replace config completely
     @Override public void getConfig(IndexschemaConfig.Builder builder) { derivedCfg.getIndexSchema().getConfig(builder); }
     @Override public void getConfig(JuniperrcConfig.Builder builder) { derivedCfg.getJuniperrc().getConfig(builder); }
     @Override public void getConfig(SummaryConfig.Builder builder) { derivedCfg.getSummaries().getConfig(builder); }
     @Override public void getConfig(ImportedFieldsConfig.Builder builder) { derivedCfg.getImportedFields().getConfig(builder); }
-    @Override public void getConfig(SchemaInfoConfig.Builder builder) { derivedCfg.getSchemaInfo().getConfig(builder); }
     @Override public void getConfig(VsmsummaryConfig.Builder builder) { derivedCfg.getVsmSummary().getConfig(builder); }
     @Override public void getConfig(VsmfieldsConfig.Builder builder) { derivedCfg.getVsmFields().getConfig(builder); }
 

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java
@@ -60,6 +60,15 @@ public boolean hasDocumentDB(String name) {
         return false;
     }
 
+    public String getConfigId(String name) {
+        for (DocumentDatabase db : documentDbs) {
+            if (db.getName().equals(name)) {
+                return db.getConfigId();
+            }
+        }
+        return "";
+    }
+
     /** Returns the schemas that should be active in this cluster. Note: These are added during processing. */
     public Map<String, SchemaInfo> schemas() { return Collections.unmodifiableMap(schemas); }
 

diff --git a/container-core/src/main/resources/configdefinitions/container.qr-searchers.def b/container-core/src/main/resources/configdefinitions/container.qr-searchers.def
@@ -64,7 +64,7 @@ searchcluster[].name string
 searchcluster[].searchdef[] string
 
 ## configid that may be used to get rank-profiles config for the cluster.
-searchcluster[].rankprofiles.configid reference default=""
+searchcluster[].rankprofiles_configid string default=""
 
 ## Indexing mode of search cluster.
 searchcluster[].indexingmode enum { REALTIME, STREAMING } default=REALTIME

diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java
@@ -150,7 +150,7 @@ private static StreamingSearcher streamingCluster(String serverId,
                                                searchClusterConfig.searchdef());
         ClusterParams clusterParams = makeClusterParams(searchclusterIndex);
         StreamingSearcher searcher = new StreamingSearcher(access);
-        searcher.setSearchClusterName(searchClusterConfig.rankprofiles().configid());
+        searcher.setSearchClusterName(searchClusterConfig.rankprofiles_configid());
         searcher.setDocumentType(searchClusterConfig.searchdef(0));
         searcher.setStorageClusterRouteSpec(searchClusterConfig.storagecluster().routespec());
         searcher.init(serverId, docSumParams, clusterParams, documentdbInfoConfig, schemaInfo);

diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/NonPhrasingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/NonPhrasingSearcher.java
@@ -44,7 +44,7 @@ public NonPhrasingSearcher(String phraseAutomatonFile) {
     }
 
     private void setupAutomatonFile(String phraseAutomatonFile) {
-        if (phraseAutomatonFile == null || phraseAutomatonFile.trim().equals("")) {
+        if (phraseAutomatonFile == null || phraseAutomatonFile.trim().isEmpty()) {
             //no file, just use dummy matcher
             phraseMatcher = PhraseMatcher.getNullMatcher();
         } else {

diff --git a/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java b/container-search/src/main/java/com/yahoo/search/schema/SchemaInfoConfigurer.java
@@ -3,7 +3,6 @@
 
 import com.yahoo.container.QrSearchersConfig;
 import com.yahoo.search.config.SchemaInfoConfig;
-import com.yahoo.tensor.TensorType;
 
 import java.util.ArrayList;
 import java.util.List;

diff --git a/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java b/container-search/src/test/java/com/yahoo/search/schema/SchemaInfoTester.java
@@ -3,12 +3,8 @@
 
 import com.yahoo.container.QrSearchersConfig;
 import com.yahoo.search.Query;
-import com.yahoo.search.config.IndexInfoConfig;
 import com.yahoo.search.config.SchemaInfoConfig;
-import com.yahoo.search.schema.RankProfile;
 import com.yahoo.search.schema.RankProfile.InputType;
-import com.yahoo.search.schema.Schema;
-import com.yahoo.search.schema.SchemaInfo;
 import com.yahoo.tensor.TensorType;
 
 import java.util.ArrayList;

diff --git a/...ner-search/src/test/java/com/yahoo/vespa/streamingvisitors/StreamingSearcherTestCase.java b/...ner-search/src/test/java/com/yahoo/vespa/streamingvisitors/StreamingSearcherTestCase.java
@@ -28,7 +28,6 @@
 import org.junit.jupiter.api.Test;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;