Merge branch 'main' into correct-seed-data-in-reload-test

Author: ebarlas

modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java

@@ -783,7 +783,10 @@ static Tuple<Long, Long> numberOfMultiparts(final long totalSize, final long par
         }
     }
 
-    private class CompareAndExchangeOperation {
+    /**
+     * An implementation of {@link BlobContainer#compareAndExchangeRegister} based on strongly-consistent multipart upload APIs.
+     */
+    private class MultipartUploadCompareAndExchangeOperation {
 
         private final OperationPurpose purpose;
         private final S3Client client;
@@ -792,7 +795,13 @@ private class CompareAndExchangeOperation {
         private final String blobKey;
         private final ThreadPool threadPool;
 
-        CompareAndExchangeOperation(OperationPurpose purpose, S3Client client, String bucket, String key, ThreadPool threadPool) {
+        MultipartUploadCompareAndExchangeOperation(
+            OperationPurpose purpose,
+            S3Client client,
+            String bucket,
+            String key,
+            ThreadPool threadPool
+        ) {
             this.purpose = purpose;
             this.client = client;
             this.bucket = bucket;
@@ -802,6 +811,23 @@ private class CompareAndExchangeOperation {
         }
 
         void run(BytesReference expected, BytesReference updated, ActionListener<OptionalBytesReference> listener) throws Exception {
+            innerRun(expected, updated, listener.delegateResponse((delegate, e) -> {
+                logger.trace(() -> Strings.format("[%s]: compareAndExchangeRegister failed", rawKey), e);
+                if (e instanceof AwsServiceException awsServiceException
+                    && (awsServiceException.statusCode() == 404
+                        || awsServiceException.statusCode() == 200
+                            && "NoSuchUpload".equals(awsServiceException.awsErrorDetails().errorCode()))) {
+                    // An uncaught 404 means that our multipart upload was aborted by a concurrent operation before we could complete it.
+                    // Also (rarely) S3 can start processing the request during a concurrent abort and this can result in a 200 OK with an
+                    // <Error><Code>NoSuchUpload</Code>... in the response. Either way, this means that our write encountered contention:
+                    delegate.onResponse(OptionalBytesReference.MISSING);
+                } else {
+                    delegate.onFailure(e);
+                }
+            }));
+        }
+
+        void innerRun(BytesReference expected, BytesReference updated, ActionListener<OptionalBytesReference> listener) throws Exception {
             BlobContainerUtils.ensureValidRegisterContent(updated);
 
             if (hasPreexistingUploads()) {
@@ -1094,25 +1120,15 @@ public void compareAndExchangeRegister(
         ActionListener<OptionalBytesReference> listener
     ) {
         final var clientReference = blobStore.clientReference();
-        ActionListener.run(ActionListener.releaseAfter(listener.delegateResponse((delegate, e) -> {
-            logger.trace(() -> Strings.format("[%s]: compareAndExchangeRegister failed", key), e);
-            if (e instanceof AwsServiceException awsServiceException
-                && (awsServiceException.statusCode() == 404
-                    || awsServiceException.statusCode() == 200
-                        && "NoSuchUpload".equals(awsServiceException.awsErrorDetails().errorCode()))) {
-                // An uncaught 404 means that our multipart upload was aborted by a concurrent operation before we could complete it.
-                // Also (rarely) S3 can start processing the request during a concurrent abort and this can result in a 200 OK with an
-                // <Error><Code>NoSuchUpload</Code>... in the response. Either way, this means that our write encountered contention:
-                delegate.onResponse(OptionalBytesReference.MISSING);
-            } else {
-                delegate.onFailure(e);
-            }
-        }), clientReference),
-            l -> new CompareAndExchangeOperation(purpose, clientReference.client(), blobStore.bucket(), key, blobStore.getThreadPool()).run(
-                expected,
-                updated,
-                l
-            )
+        ActionListener.run(
+            ActionListener.releaseBefore(clientReference, listener),
+            l -> new MultipartUploadCompareAndExchangeOperation(
+                purpose,
+                clientReference.client(),
+                blobStore.bucket(),
+                key,
+                blobStore.getThreadPool()
+            ).run(expected, updated, l)
         );
     }
 

muted-tests.yml

@@ -420,6 +420,12 @@ tests:
 - class: org.elasticsearch.smoketest.MlWithSecurityIT
   method: test {yaml=ml/start_data_frame_analytics/Test start classification analysis when the dependent variable cardinality is too low}
   issue: https://github.com/elastic/elasticsearch/issues/138409
+- class: org.elasticsearch.xpack.test.rest.XPackRestIT
+  method: test {p0=esql/60_usage/Basic ESQL usage output (telemetry) non-snapshot version}
+  issue: https://github.com/elastic/elasticsearch/issues/138426
+- class: org.elasticsearch.xpack.ml.integration.RevertModelSnapshotIT
+  method: testRevertModelSnapshot
+  issue: https://github.com/elastic/elasticsearch/issues/132733
 
 # Examples:
 #

qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/FeatureUpgradeApiIT.java

@@ -29,7 +29,7 @@ public class FeatureUpgradeApiIT extends AbstractSystemIndicesIT {
 
     @After
     public void resetFeatures() throws Exception {
-        client().performRequest(new Request("POST", "/_features/_reset"));
+        performPostFeaturesReset(client());
     }
 
     public void testCreatingSystemIndex() throws Exception {

qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/NetNewSystemIndicesIT.java

@@ -94,6 +94,6 @@ public void testSearch() throws Exception {
 
     @After
     public void resetFeatures() throws Exception {
-        client().performRequest(new Request("POST", "/_features/_reset"));
+        performPostFeaturesReset(client());
     }
 }

qa/system-indices/src/javaRestTest/java/org/elasticsearch/system/indices/SystemAliasIT.java

@@ -26,7 +26,7 @@ public class SystemAliasIT extends AbstractSystemIndicesIT {
 
     @After
     public void resetFeatures() throws Exception {
-        client().performRequest(new Request("POST", "/_features/_reset"));
+        performPostFeaturesReset(client());
     }
 
     public void testCreatingSystemIndexWithAlias() throws Exception {

server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingStatsTests.java

@@ -99,7 +99,7 @@ public class MappingStatsTests extends AbstractWireSerializingTestCase<MappingSt
 
     private static final String SCRIPT_1 = scriptAsJSON("doc['field'] + doc.field + params._source.field");
     private static final String SCRIPT_2 = scriptAsJSON("doc['field']");
-    private static final String SCRIPT_3 = scriptAsJSON("params._source.field + params._source.field \n + params._source.field");
+    private static final String SCRIPT_3 = scriptAsJSON("params._source.field + params._source.field + params._source.field");
     private static final String SCRIPT_4 = scriptAsJSON("params._source.field");
 
     public void testToXContent() {
@@ -114,8 +114,8 @@ public void testToXContent() {
               "mappings" : {
                 "total_field_count" : 12,
                 "total_deduplicated_field_count" : 6,
-                "total_deduplicated_mapping_size" : "260b",
-                "total_deduplicated_mapping_size_in_bytes" : 260,
+                "total_deduplicated_mapping_size" : "255b",
+                "total_deduplicated_mapping_size_in_bytes" : 255,
                 "field_types" : [
                   {
                     "name" : "dense_vector",
@@ -159,10 +159,10 @@ public void testToXContent() {
                     "lang" : [
                       "painless"
                     ],
-                    "lines_max" : 2,
-                    "lines_total" : 6,
-                    "chars_max" : 68,
-                    "chars_total" : 176,
+                    "lines_max" : 1,
+                    "lines_total" : 4,
+                    "chars_max" : 66,
+                    "chars_total" : 172,
                     "source_max" : 3,
                     "source_total" : 8,
                     "doc_max" : 0,
@@ -203,10 +203,10 @@ public void testToXContent() {
                     "lang" : [
                       "painless"
                     ],
-                    "lines_max" : 2,
-                    "lines_total" : 6,
-                    "chars_max" : 68,
-                    "chars_total" : 176,
+                    "lines_max" : 1,
+                    "lines_total" : 4,
+                    "chars_max" : 66,
+                    "chars_total" : 172,
                     "source_max" : 3,
                     "source_total" : 8,
                     "doc_max" : 0,
@@ -246,8 +246,8 @@ public void testToXContentWithSomeSharedMappings() {
               "mappings" : {
                 "total_field_count" : 18,
                 "total_deduplicated_field_count" : 12,
-                "total_deduplicated_mapping_size" : "519b",
-                "total_deduplicated_mapping_size_in_bytes" : 519,
+                "total_deduplicated_mapping_size" : "513b",
+                "total_deduplicated_mapping_size_in_bytes" : 513,
                 "field_types" : [
                   {
                     "name" : "dense_vector",
@@ -291,10 +291,10 @@ public void testToXContentWithSomeSharedMappings() {
                     "lang" : [
                       "painless"
                     ],
-                    "lines_max" : 2,
-                    "lines_total" : 9,
-                    "chars_max" : 68,
-                    "chars_total" : 264,
+                    "lines_max" : 1,
+                    "lines_total" : 6,
+                    "chars_max" : 66,
+                    "chars_total" : 258,
                     "source_max" : 3,
                     "source_total" : 12,
                     "doc_max" : 0,
@@ -335,10 +335,10 @@ public void testToXContentWithSomeSharedMappings() {
                     "lang" : [
                       "painless"
                     ],
-                    "lines_max" : 2,
-                    "lines_total" : 9,
-                    "chars_max" : 68,
-                    "chars_total" : 264,
+                    "lines_max" : 1,
+                    "lines_total" : 6,
+                    "chars_max" : 66,
+                    "chars_total" : 258,
                     "source_max" : 3,
                     "source_total" : 12,
                     "doc_max" : 0,

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -923,6 +923,15 @@ protected boolean preserveSearchableSnapshotsIndicesUponCompletion() {
         return false;
     }
 
+    /**
+     * Invoke {@code POST /_features/_reset?error_trace} with the given {@link RestClient}.
+     */
+    public static void performPostFeaturesReset(RestClient restClient) throws IOException {
+        final var request = new Request(HttpPost.METHOD_NAME, "/_features/_reset");
+        request.addParameter("error_trace", "true");
+        assertOK(restClient.performRequest(request));
+    }
+
     private void wipeCluster() throws Exception {
         waitForClusterUpdates();
 
@@ -949,8 +958,7 @@ private void wipeCluster() throws Exception {
         wipeSnapshots();
 
         if (resetFeatureStates()) {
-            final Request postRequest = new Request("POST", "/_features/_reset");
-            cleanupClient().performRequest(postRequest);
+            performPostFeaturesReset(cleanupClient());
         }
 
         // wipe data streams before indices so that the backing indices for data streams are handled properly

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/integration/MlRestTestStateCleaner.java

@@ -17,6 +17,7 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.test.rest.ESRestTestCase.performPostFeaturesReset;
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.hasSize;
 
@@ -33,7 +34,7 @@ public MlRestTestStateCleaner(Logger logger, RestClient adminClient) {
     public void resetFeatures() throws IOException {
         deletePipelinesWithInferenceProcessors();
         // This resets all features, not just ML, but they should have been getting reset between tests anyway so it shouldn't matter
-        adminClient.performRequest(new Request("POST", "/_features/_reset"));
+        performPostFeaturesReset(adminClient);
     }
 
     @SuppressWarnings("unchecked")

x-pack/plugin/deprecation/qa/src/javaRestTest/java/org/elasticsearch/xpack/deprecation/MlDeprecationIT.java

@@ -59,8 +59,7 @@ protected String getTestRestCluster() {
 
     @After
     public void resetFeatures() throws IOException {
-        Response response = adminClient().performRequest(new Request("POST", "/_features/_reset"));
-        assertOK(response);
+        performPostFeaturesReset(adminClient());
     }
 
     @Override

x-pack/plugin/ml/qa/disabled/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlPluginDisabledIT.java

@@ -75,8 +75,7 @@ public void testActionsFail() throws Exception {
     }
 
     public void testMlFeatureReset() throws IOException {
-        Request request = new Request("POST", "/_features/_reset");
-        assertOK(client().performRequest(request));
+        performPostFeaturesReset(client());
     }
 
     @SuppressWarnings("unchecked")