Merge branch 'main' into parker/logsdb-random-test-querying

Author: parkertimmins

.buildkite/pipelines/intake.yml

@@ -65,7 +65,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.17.10", "8.18.5", "9.0.5", "9.1.1", "9.2.0"]
+            BWC_VERSION: ["8.17.10", "8.18.5", "8.19.0", "9.0.5", "9.1.1", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404

.buildkite/pipelines/periodic-packaging.yml

@@ -334,6 +334,22 @@ steps:
         env:
           BWC_VERSION: 8.18.5
 
+      - label: "{{matrix.image}} / 8.19.0 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.19.0
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2404
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.19.0
+
       - label: "{{matrix.image}} / 9.0.5 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.5
         timeout_in_minutes: 300

.buildkite/pipelines/periodic.yml

@@ -363,6 +363,25 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
+      - label: 8.19.0 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.19.0#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2404
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+          preemptible: true
+        env:
+          BWC_VERSION: 8.19.0
+        retry:
+          automatic:
+            - exit_status: "-1"
+              limit: 3
+              signal_reason: none
+            - signal_reason: agent_stop
+              limit: 3
+
       - label: 9.0.5 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.5#bwcTest
         timeout_in_minutes: 300
@@ -487,7 +506,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.17.10", "8.18.5", "9.0.5", "9.1.1", "9.2.0"]
+            BWC_VERSION: ["8.17.10", "8.18.5", "8.19.0", "9.0.5", "9.1.1", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404
@@ -531,7 +550,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.17.10", "8.18.5", "9.0.5", "9.1.1", "9.2.0"]
+            BWC_VERSION: ["8.17.10", "8.18.5", "8.19.0", "9.0.5", "9.1.1", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404

.ci/bwcVersions

@@ -18,6 +18,7 @@ BWC_VERSION:
   - "8.16.6"
   - "8.17.10"
   - "8.18.5"
+  - "8.19.0"
   - "9.0.5"
   - "9.1.1"
   - "9.2.0"

.ci/snapshotBwcVersions

@@ -1,6 +1,7 @@
 BWC_VERSION:
   - "8.17.10"
   - "8.18.5"
+  - "8.19.0"
   - "9.0.5"
   - "9.1.1"
   - "9.2.0"

muted-tests.yml

@@ -545,6 +545,60 @@ tests:
 - class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
   method: testDenseVectorMappingUpdate {initialType=int4_flat updateType=int8_hnsw}
   issue: https://github.com/elastic/elasticsearch/issues/132117
+- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
+  method: testDenseVectorMappingUpdate {initialType=flat updateType=hnsw}
+  issue: https://github.com/elastic/elasticsearch/issues/132119
+- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
+  method: testDenseVectorMappingUpdate {initialType=flat updateType=bbq_flat}
+  issue: https://github.com/elastic/elasticsearch/issues/132120
+- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
+  method: testDenseVectorMappingUpdate {initialType=int8_flat updateType=bbq_disk}
+  issue: https://github.com/elastic/elasticsearch/issues/132122
+- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
+  method: testDenseVectorMappingUpdate {initialType=int8_flat updateType=int4_hnsw}
+  issue: https://github.com/elastic/elasticsearch/issues/132123
+- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
+  method: testDenseVectorMappingUpdate {initialType=int8_flat updateType=hnsw}
+  issue: https://github.com/elastic/elasticsearch/issues/132124
+- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
+  method: testDenseVectorMappingUpdate {initialType=int8_flat updateType=int4_flat}
+  issue: https://github.com/elastic/elasticsearch/issues/132125
+- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
+  method: testDenseVectorMappingUpdate {initialType=flat updateType=int8_hnsw}
+  issue: https://github.com/elastic/elasticsearch/issues/132126
+- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
+  method: testDenseVectorMappingUpdate {initialType=bbq_disk updateType=bbq_disk}
+  issue: https://github.com/elastic/elasticsearch/issues/132127
+- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
+  method: testDenseVectorMappingUpdate {initialType=flat updateType=int8_flat}
+  issue: https://github.com/elastic/elasticsearch/issues/132129
+- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
+  method: "testDenseVectorMappingUpdate {initialType=flat updateType=bbq_disk #2}"
+  issue: https://github.com/elastic/elasticsearch/issues/132130
+- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
+  method: testDenseVectorMappingUpdate {initialType=int4_flat updateType=bbq_flat}
+  issue: https://github.com/elastic/elasticsearch/issues/132132
+- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
+  method: testDenseVectorMappingUpdate {initialType=int8_hnsw updateType=int4_hnsw}
+  issue: https://github.com/elastic/elasticsearch/issues/132133
+- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
+  method: testDenseVectorMappingUpdate {initialType=bbq_flat updateType=bbq_disk}
+  issue: https://github.com/elastic/elasticsearch/issues/132134
+- class: org.elasticsearch.upgrades.RunningSnapshotIT
+  method: testRunningSnapshotCompleteAfterUpgrade {upgradedNodes=1}
+  issue: https://github.com/elastic/elasticsearch/issues/132135
+- class: org.elasticsearch.upgrades.RunningSnapshotIT
+  method: testRunningSnapshotCompleteAfterUpgrade {upgradedNodes=2}
+  issue: https://github.com/elastic/elasticsearch/issues/132136
+- class: org.elasticsearch.upgrades.RunningSnapshotIT
+  method: testRunningSnapshotCompleteAfterUpgrade {upgradedNodes=3}
+  issue: https://github.com/elastic/elasticsearch/issues/132137
+- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
+  method: testDenseVectorMappingUpdate {initialType=int4_flat updateType=int4_hnsw}
+  issue: https://github.com/elastic/elasticsearch/issues/132140
+- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
+  method: testDenseVectorMappingUpdate {initialType=int8_flat updateType=bbq_hnsw}
+  issue: https://github.com/elastic/elasticsearch/issues/132141
 
 # Examples:
 #

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -333,7 +333,6 @@ static TransportVersion def(int id) {
     public static final TransportVersion ESQL_SPLIT_ON_BIG_VALUES_9_1 = def(9_112_0_01);
     public static final TransportVersion ESQL_FIXED_INDEX_LIKE_9_1 = def(9_112_0_02);
     public static final TransportVersion ESQL_SAMPLE_OPERATOR_STATUS_9_1 = def(9_112_0_03);
-    // Below is the first version in 9.2 and NOT in 9.1.
     public static final TransportVersion INITIAL_ELASTICSEARCH_9_1_1 = def(9_112_0_04);
     public static final TransportVersion PROJECT_STATE_REGISTRY_RECORDS_DELETIONS = def(9_113_0_00);
     public static final TransportVersion ESQL_SERIALIZE_TIMESERIES_FIELD_TYPE = def(9_114_0_00);

server/src/main/java/org/elasticsearch/Version.java

@@ -212,6 +212,7 @@ public class Version implements VersionId<Version>, ToXContentFragment {
     public static final Version V_8_18_3 = new Version(8_18_03_99);
     public static final Version V_8_18_4 = new Version(8_18_04_99);
     public static final Version V_8_18_5 = new Version(8_18_05_99);
+    public static final Version V_8_19_0 = new Version(8_19_00_99);
     public static final Version V_9_0_0 = new Version(9_00_00_99);
     public static final Version V_9_0_1 = new Version(9_00_01_99);
     public static final Version V_9_0_2 = new Version(9_00_02_99);

x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapper.java

@@ -33,8 +33,6 @@
 import java.util.List;
 import java.util.Map;
 
-import static org.elasticsearch.index.IndexSettings.USE_DOC_VALUES_SKIPPER;
-
 /**
  * A {@link FieldMapper} that assigns every document the same value.
  */
@@ -62,25 +60,12 @@ public static class Builder extends FieldMapper.Builder {
         private final IndexSettings indexSettings;
         private final Parameter<Map<String, String>> meta = Parameter.metaParam();
         private final TextParams.Analyzers analyzers;
-        private final boolean enableDocValuesSkipper;
 
         public Builder(String name, MappingParserContext context) {
-            this(
-                name,
-                context.indexVersionCreated(),
-                context.getIndexSettings(),
-                context.getIndexAnalyzers(),
-                USE_DOC_VALUES_SKIPPER.get(context.getSettings())
-            );
+            this(name, context.indexVersionCreated(), context.getIndexSettings(), context.getIndexAnalyzers());
         }
 
-        public Builder(
-            String name,
-            IndexVersion indexCreatedVersion,
-            IndexSettings indexSettings,
-            IndexAnalyzers indexAnalyzers,
-            boolean enableDocValuesSkipper
-        ) {
+        public Builder(String name, IndexVersion indexCreatedVersion, IndexSettings indexSettings, IndexAnalyzers indexAnalyzers) {
             super(name);
             this.indexCreatedVersion = indexCreatedVersion;
             this.indexSettings = indexSettings;
@@ -90,7 +75,6 @@ public Builder(
                 m -> ((PatternedTextFieldMapper) m).positionIncrementGap,
                 indexCreatedVersion
             );
-            this.enableDocValuesSkipper = enableDocValuesSkipper;
         }
 
         @Override
@@ -120,8 +104,8 @@ public PatternedTextFieldMapper build(MapperBuilderContext context) {
                 patternedTextFieldType.templateIdFieldName(),
                 indexSettings.getMode(),
                 indexCreatedVersion,
-                enableDocValuesSkipper
-            ).build(context);
+                true
+            ).indexed(false).build(context);
             return new PatternedTextFieldMapper(leafName(), patternedTextFieldType, builderParams, this, templateIdMapper);
         }
     }
@@ -132,7 +116,6 @@ public PatternedTextFieldMapper build(MapperBuilderContext context) {
     private final IndexAnalyzers indexAnalyzers;
     private final IndexSettings indexSettings;
     private final NamedAnalyzer indexAnalyzer;
-    private final boolean enableDocValuesSkipper;
     private final int positionIncrementGap;
     private final FieldType fieldType;
     private final KeywordFieldMapper templateIdMapper;
@@ -152,7 +135,6 @@ private PatternedTextFieldMapper(
         this.indexAnalyzers = builder.analyzers.indexAnalyzers;
         this.indexAnalyzer = builder.analyzers.getIndexAnalyzer();
         this.indexSettings = builder.indexSettings;
-        this.enableDocValuesSkipper = builder.enableDocValuesSkipper;
         this.positionIncrementGap = builder.analyzers.positionIncrementGap.getValue();
         this.templateIdMapper = templateIdMapper;
     }
@@ -164,7 +146,7 @@ public Map<String, NamedAnalyzer> indexAnalyzers() {
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(leafName(), indexCreatedVersion, indexSettings, indexAnalyzers, enableDocValuesSkipper).init(this);
+        return new Builder(leafName(), indexCreatedVersion, indexSettings, indexAnalyzers).init(this);
     }
 
     @Override

x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapperTests.java

@@ -24,7 +24,6 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Tuple;
-import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.DateFieldMapper;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
@@ -131,10 +130,7 @@ protected void minimalStoreMapping(XContentBuilder b) throws IOException {
     }
 
     public void testDefaults() throws IOException {
-        boolean enabledDocValuesSkipper = randomBoolean();
-        var indexSettings = getIndexSettingsBuilder().put(IndexSettings.USE_DOC_VALUES_SKIPPER.getKey(), enabledDocValuesSkipper).build();
-
-        DocumentMapper mapper = createMapperService(indexSettings, fieldMapping(this::minimalMapping)).documentMapper();
+        DocumentMapper mapper = createMapperService(fieldMapping(this::minimalMapping)).documentMapper();
         assertEquals(Strings.toString(fieldMapping(this::minimalMapping)), mapper.mappingSource().toString());
 
         ParsedDocument doc = mapper.parse(source(b -> b.field("field", "1234")));
@@ -162,7 +158,7 @@ public void testDefaults() throws IOException {
             assertThat(fieldType.omitNorms(), equalTo(true));
             assertFalse(fieldType.tokenized());
             assertFalse(fieldType.stored());
-            assertThat(fieldType.indexOptions(), equalTo(enabledDocValuesSkipper ? IndexOptions.NONE : IndexOptions.DOCS));
+            assertThat(fieldType.indexOptions(), equalTo(IndexOptions.NONE));
             assertThat(fieldType.storeTermVectors(), equalTo(false));
             assertThat(fieldType.storeTermVectorOffsets(), equalTo(false));
             assertThat(fieldType.storeTermVectorPositions(), equalTo(false));