Merge remote-tracking branch 'upstream/main' into fix-rolling-upgrade-tests-2

Author: jordan-powers

docs/changelog/135370.yaml

@@ -0,0 +1,5 @@
+pr: 135370
+summary: Add new `pattern_text` field mapper in tech preview
+area: Mapping
+type: feature
+issues: []

muted-tests.yml

@@ -73,9 +73,6 @@ tests:
 - class: org.elasticsearch.xpack.apmdata.APMYamlTestSuiteIT
   method: test {yaml=/10_apm/Test template reinstallation}
   issue: https://github.com/elastic/elasticsearch/issues/116445
-- class: org.elasticsearch.versioning.ConcurrentSeqNoVersioningIT
-  method: testSeqNoCASLinearizability
-  issue: https://github.com/elastic/elasticsearch/issues/117249
 - class: org.elasticsearch.xpack.inference.InferenceRestIT
   method: test {p0=inference/40_semantic_text_query/Query a field that uses the default ELSER 2 endpoint}
   issue: https://github.com/elastic/elasticsearch/issues/117027
@@ -621,9 +618,6 @@ tests:
 - class: org.elasticsearch.xpack.esql.qa.multi_node.GenerativeIT
   method: test
   issue: https://github.com/elastic/elasticsearch/issues/134407
-- class: org.elasticsearch.index.codec.vectors.cluster.HierarchicalKMeansTests
-  method: testFewDifferentValues
-  issue: https://github.com/elastic/elasticsearch/issues/135538
 
 # Examples:
 #

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -192,7 +192,6 @@ static TransportVersion def(int id) {
     public static final TransportVersion ML_INFERENCE_ELASTIC_RERANK_ADDED_8_19 = def(8_841_0_48);
     public static final TransportVersion NONE_CHUNKING_STRATEGY_8_19 = def(8_841_0_49);
     public static final TransportVersion IDP_CUSTOM_SAML_ATTRIBUTES_ALLOW_LIST_8_19 = def(8_841_0_50);
-    public static final TransportVersion SETTINGS_IN_DATA_STREAMS_8_19 = def(8_841_0_51);
     public static final TransportVersion INITIAL_ELASTICSEARCH_9_0 = def(9_000_0_00);
     public static final TransportVersion REMOVE_SNAPSHOT_FAILURES_90 = def(9_000_0_01);
     public static final TransportVersion TRANSPORT_STATS_HANDLING_TIME_REQUIRED_90 = def(9_000_0_02);
@@ -263,7 +262,6 @@ static TransportVersion def(int id) {
     public static final TransportVersion SYNONYMS_REFRESH_PARAM = def(9_060_0_00);
     public static final TransportVersion DOC_FIELDS_AS_LIST = def(9_061_0_00);
     public static final TransportVersion DENSE_VECTOR_OFF_HEAP_STATS = def(9_062_00_0);
-    public static final TransportVersion SETTINGS_IN_DATA_STREAMS = def(9_064_0_00);
     public static final TransportVersion INTRODUCE_FAILURES_LIFECYCLE = def(9_065_0_00);
     public static final TransportVersion PROJECT_METADATA_SETTINGS = def(9_066_0_00);
     public static final TransportVersion AGGREGATE_METRIC_DOUBLE_BLOCK = def(9_067_0_00);
@@ -280,7 +278,6 @@ static TransportVersion def(int id) {
     public static final TransportVersion ML_INFERENCE_HUGGING_FACE_CHAT_COMPLETION_ADDED = def(9_078_0_00);
     public static final TransportVersion NODES_STATS_SUPPORTS_MULTI_PROJECT = def(9_079_0_00);
     public static final TransportVersion ML_INFERENCE_HUGGING_FACE_RERANK_ADDED = def(9_080_0_00);
-    public static final TransportVersion SETTINGS_IN_DATA_STREAMS_DRY_RUN = def(9_081_0_00);
     public static final TransportVersion ML_INFERENCE_SAGEMAKER_CHAT_COMPLETION = def(9_082_0_00);
     public static final TransportVersion ML_INFERENCE_VERTEXAI_CHATCOMPLETION_ADDED = def(9_083_0_00);
     public static final TransportVersion INFERENCE_CUSTOM_SERVICE_ADDED = def(9_084_0_00);

server/src/main/java/org/elasticsearch/action/datastreams/UpdateDataStreamSettingsAction.java

@@ -10,7 +10,6 @@
 package org.elasticsearch.action.datastreams;
 
 import org.elasticsearch.TransportVersion;
-import org.elasticsearch.TransportVersions;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.IndicesRequest;
@@ -42,6 +41,7 @@ public class UpdateDataStreamSettingsAction extends ActionType<UpdateDataStreamS
     public static final String NAME = "indices:admin/data_stream/settings/update";
     public static final UpdateDataStreamSettingsAction INSTANCE = new UpdateDataStreamSettingsAction();
 
+    private static final TransportVersion SETTINGS_IN_DATA_STREAMS = TransportVersion.fromName("settings_in_data_streams");
     private static final TransportVersion DATA_STREAM_WRITE_INDEX_ONLY_SETTINGS = TransportVersion.fromName(
         "data_stream_write_index_only_settings"
     );
@@ -88,8 +88,7 @@ public Request(StreamInput in) throws IOException {
             super(in);
             this.dataStreamNames = in.readStringArray();
             this.settings = Settings.readSettingsFromStream(in);
-            if (in.getTransportVersion().onOrAfter(TransportVersions.SETTINGS_IN_DATA_STREAMS_DRY_RUN)
-                || in.getTransportVersion().isPatchFrom(TransportVersions.SETTINGS_IN_DATA_STREAMS_8_19)) {
+            if (in.getTransportVersion().supports(SETTINGS_IN_DATA_STREAMS)) {
                 this.dryRun = in.readBoolean();
             } else {
                 this.dryRun = false;
@@ -101,8 +100,7 @@ public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
             out.writeStringArray(dataStreamNames);
             settings.writeTo(out);
-            if (out.getTransportVersion().onOrAfter(TransportVersions.SETTINGS_IN_DATA_STREAMS_DRY_RUN)
-                || out.getTransportVersion().isPatchFrom(TransportVersions.SETTINGS_IN_DATA_STREAMS_8_19)) {
+            if (out.getTransportVersion().supports(SETTINGS_IN_DATA_STREAMS)) {
                 out.writeBoolean(dryRun);
             }
         }

server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java

@@ -86,6 +86,7 @@ public final class DataStream implements SimpleDiffable<DataStream>, ToXContentO
 
     private static final Logger LOGGER = LogManager.getLogger(DataStream.class);
 
+    private static final TransportVersion SETTINGS_IN_DATA_STREAMS = TransportVersion.fromName("settings_in_data_streams");
     private static final TransportVersion MAPPINGS_IN_DATA_STREAMS = TransportVersion.fromName("mappings_in_data_streams");
 
     public static final NodeFeature DATA_STREAM_FAILURE_STORE_FEATURE = new NodeFeature("data_stream.failure_store");
@@ -334,8 +335,7 @@ public static DataStream read(StreamInput in) throws IOException {
             dataStreamOptions = failureStoreEnabled ? DataStreamOptions.FAILURE_STORE_ENABLED : null;
         }
         final Settings settings;
-        if (in.getTransportVersion().onOrAfter(TransportVersions.SETTINGS_IN_DATA_STREAMS)
-            || in.getTransportVersion().isPatchFrom(TransportVersions.SETTINGS_IN_DATA_STREAMS_8_19)) {
+        if (in.getTransportVersion().supports(SETTINGS_IN_DATA_STREAMS)) {
             settings = Settings.readSettingsFromStream(in);
         } else {
             settings = Settings.EMPTY;
@@ -1479,8 +1479,7 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getTransportVersion().onOrAfter(DataStream.ADD_DATA_STREAM_OPTIONS_VERSION)) {
             out.writeOptionalWriteable(dataStreamOptions.isEmpty() ? null : dataStreamOptions);
         }
-        if (out.getTransportVersion().onOrAfter(TransportVersions.SETTINGS_IN_DATA_STREAMS)
-            || out.getTransportVersion().isPatchFrom(TransportVersions.SETTINGS_IN_DATA_STREAMS_8_19)) {
+        if (out.getTransportVersion().supports(SETTINGS_IN_DATA_STREAMS)) {
             settings.writeTo(out);
         }
         if (out.getTransportVersion().supports(MAPPINGS_IN_DATA_STREAMS)) {

server/src/main/java/org/elasticsearch/index/codec/vectors/cluster/HierarchicalKMeans.java

@@ -24,6 +24,7 @@ public class HierarchicalKMeans {
     public static final int MAX_ITERATIONS_DEFAULT = 6;
     public static final int SAMPLES_PER_CLUSTER_DEFAULT = 64;
     public static final float DEFAULT_SOAR_LAMBDA = 1.0f;
+    public static final int NO_SOAR_ASSIGNMENT = -1;
 
     final int dimension;
     final int maxIterations;

server/src/main/java/org/elasticsearch/index/codec/vectors/cluster/KMeansLocal.java

@@ -19,6 +19,8 @@
 import java.util.Arrays;
 import java.util.Random;
 
+import static org.elasticsearch.index.codec.vectors.cluster.HierarchicalKMeans.NO_SOAR_ASSIGNMENT;
+
 /**
  * k-means implementation specific to the needs of the {@link HierarchicalKMeans} algorithm that deals specifically
  * with finalizing nearby pre-established clusters and generate
@@ -240,7 +242,7 @@ private void assignSpilled(
             // TODO: cache these?
             float vectorCentroidDist = VectorUtil.squareDistance(vector, currentCentroid);
             if (vectorCentroidDist <= SOAR_MIN_DISTANCE) {
-                spilledAssignments[i] = -1; // no SOAR assignment
+                spilledAssignments[i] = NO_SOAR_ASSIGNMENT; // no SOAR assignment
                 continue;
             }
 

server/src/main/java/org/elasticsearch/index/codec/vectors/diskbbq/ES920DiskBBQVectorsWriter.java

@@ -38,6 +38,8 @@
 import java.util.AbstractList;
 import java.util.Arrays;
 
+import static org.elasticsearch.index.codec.vectors.cluster.HierarchicalKMeans.NO_SOAR_ASSIGNMENT;
+
 /**
  * Default implementation of {@link IVFVectorsWriter}. It uses {@link HierarchicalKMeans} algorithm to
  * partition the vector space, and then stores the centroids and posting list in a sequential
@@ -75,7 +77,7 @@ CentroidOffsetAndLength buildAndWritePostingsLists(
         for (int i = 0; i < assignments.length; i++) {
             centroidVectorCount[assignments[i]]++;
             // if soar assignments are present, count them as well
-            if (overspillAssignments.length > i && overspillAssignments[i] != -1) {
+            if (overspillAssignments.length > i && overspillAssignments[i] != NO_SOAR_ASSIGNMENT) {
                 centroidVectorCount[overspillAssignments[i]]++;
             }
         }
@@ -95,7 +97,7 @@ CentroidOffsetAndLength buildAndWritePostingsLists(
             // if soar assignments are present, add them to the cluster as well
             if (overspillAssignments.length > i) {
                 int s = overspillAssignments[i];
-                if (s != -1) {
+                if (s != NO_SOAR_ASSIGNMENT) {
                     assignmentsByCluster[s][centroidVectorCount[s]++] = i;
                 }
             }
@@ -187,7 +189,7 @@ CentroidOffsetAndLength buildAndWritePostingsLists(
                 int c = assignments[i];
                 float[] centroid = centroidSupplier.centroid(c);
                 float[] vector = floatVectorValues.vectorValue(i);
-                boolean overspill = overspillAssignments.length > i && overspillAssignments[i] != -1;
+                boolean overspill = overspillAssignments.length > i && overspillAssignments[i] != NO_SOAR_ASSIGNMENT;
                 OptimizedScalarQuantizer.QuantizationResult result = quantizer.scalarQuantize(
                     vector,
                     scratch,
@@ -220,7 +222,7 @@ CentroidOffsetAndLength buildAndWritePostingsLists(
         for (int i = 0; i < assignments.length; i++) {
             centroidVectorCount[assignments[i]]++;
             // if soar assignments are present, count them as well
-            if (overspillAssignments.length > i && overspillAssignments[i] != -1) {
+            if (overspillAssignments.length > i && overspillAssignments[i] != NO_SOAR_ASSIGNMENT) {
                 centroidVectorCount[overspillAssignments[i]]++;
             }
         }
@@ -242,7 +244,7 @@ CentroidOffsetAndLength buildAndWritePostingsLists(
             // if soar assignments are present, add them to the cluster as well
             if (overspillAssignments.length > i) {
                 int s = overspillAssignments[i];
-                if (s != -1) {
+                if (s != NO_SOAR_ASSIGNMENT) {
                     assignmentsByCluster[s][centroidVectorCount[s]] = i;
                     isOverspillByCluster[s][centroidVectorCount[s]++] = true;
                 }

server/src/main/resources/transport/definitions/referable/settings_in_data_streams.csv

@@ -0,0 +1 @@
+9064000,8841051

server/src/main/resources/transport/definitions/unreferable/dead_settings_in_data_streams.csv

@@ -0,0 +1 @@
+9081000