Merge remote-tracking branch 'es/main' into mergeSortedNumericField_5

Author: martijnvg

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DockerBase.java

@@ -26,8 +26,7 @@ public enum DockerBase {
         "-wolfi",
         "apk"
     ),
-
-    FIPS("docker.elastic.co/wolfi/chainguard-base-fips:sha256-feb7aeb1bbcb331afa089388f2fa1e81997fc24642ca4fa06b7e502ff599a4cf", "-fips", "apk"),
+    FIPS("docker.elastic.co/wolfi/chainguard-base-fips:sha256-ebfc3f1d7dba992231747a2e05ad1b859843e81b5e676ad342859d7cf9e425a7", "-fips", "apk"),
     // spotless:on
     // Based on WOLFI above, with more extras. We don't set a base image because
     // we programmatically extend from the wolfi image.

distribution/docker/fips-docker-aarch64-export/build.gradle

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+

modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamsPlugin.java

@@ -211,7 +211,8 @@ public Collection<?> createComponents(PluginServices services) {
                 errorStoreInitialisationService.get(),
                 services.allocationService(),
                 dataStreamLifecycleErrorsPublisher.get(),
-                services.dataStreamGlobalRetentionSettings()
+                services.dataStreamGlobalRetentionSettings(),
+                services.projectResolver()
             )
         );
         dataLifecycleInitialisationService.get().init();

modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java

@@ -17,7 +17,6 @@
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
-import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.metadata.ResettableValue;
 import org.elasticsearch.cluster.metadata.Template;
@@ -49,22 +48,6 @@
  */
 public class DataStreamLifecycleFixtures {
 
-    public static DataStream createDataStream(
-        Metadata.Builder builder,
-        String dataStreamName,
-        int backingIndicesCount,
-        Settings.Builder backingIndicesSettings,
-        @Nullable DataStreamLifecycle lifecycle,
-        Long now
-    ) {
-        var projectBuilder = builder.getProject(Metadata.DEFAULT_PROJECT_ID);
-        if (projectBuilder == null) {
-            projectBuilder = ProjectMetadata.builder(Metadata.DEFAULT_PROJECT_ID);
-            builder.put(projectBuilder);
-        }
-        return createDataStream(projectBuilder, dataStreamName, backingIndicesCount, 0, backingIndicesSettings, lifecycle, now);
-    }
-
     public static DataStream createDataStream(
         ProjectMetadata.Builder builder,
         String dataStreamName,
@@ -76,31 +59,6 @@ public static DataStream createDataStream(
         return createDataStream(builder, dataStreamName, backingIndicesCount, 0, backingIndicesSettings, lifecycle, now);
     }
 
-    public static DataStream createDataStream(
-        Metadata.Builder builder,
-        String dataStreamName,
-        int backingIndicesCount,
-        int failureIndicesCount,
-        Settings.Builder backingIndicesSettings,
-        @Nullable DataStreamLifecycle lifecycle,
-        Long now
-    ) {
-        var projectBuilder = builder.getProject(Metadata.DEFAULT_PROJECT_ID);
-        if (projectBuilder == null) {
-            projectBuilder = ProjectMetadata.builder(Metadata.DEFAULT_PROJECT_ID);
-            builder.put(projectBuilder);
-        }
-        return createDataStream(
-            projectBuilder,
-            dataStreamName,
-            backingIndicesCount,
-            failureIndicesCount,
-            backingIndicesSettings,
-            lifecycle,
-            now
-        );
-    }
-
     public static DataStream createDataStream(
         ProjectMetadata.Builder builder,
         String dataStreamName,

modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java

@@ -35,6 +35,7 @@
 import org.elasticsearch.reindex.ReindexPlugin;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.test.ESIntegTestCase;
+import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.transport.RemoteTransportException;
 import org.elasticsearch.xcontent.XContentType;
 import org.junit.ClassRule;
@@ -87,6 +88,10 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
     }
 
     @SuppressWarnings("unchecked")
+    @TestLogging(
+        reason = "understanding why ipinfo asn database sometimes is not loaded",
+        value = "org.elasticsearch.ingest.geoip.DatabaseNodeService:TRACE"
+    )
     public void testEnterpriseDownloaderTask() throws Exception {
         /*
          * This test starts the enterprise geoip downloader task, and creates a database configuration. Then it creates an ingest
@@ -121,17 +126,17 @@ public void testEnterpriseDownloaderTask() throws Exception {
             assertNotNull(returnedSource);
             Object targetFieldValue = returnedSource.get(targetField);
             assertNotNull(targetFieldValue);
-            assertThat(((Map<String, Object>) targetFieldValue).get("organization_name"), equalTo("Bredband2 AB"));
+            assertThat(((Map<String, Object>) targetFieldValue).get("city_name"), equalTo("Linköping"));
         });
         assertBusy(() -> {
             logger.info("Ingesting another test document");
-            String documentId = ingestDocument(indexName, iplocationPipelineName, sourceField, "12.10.66.1");
+            String documentId = ingestDocument(indexName, iplocationPipelineName, sourceField, "103.134.48.0");
             GetResponse getResponse = client().get(new GetRequest(indexName, documentId)).actionGet();
             Map<String, Object> returnedSource = getResponse.getSource();
             assertNotNull(returnedSource);
             Object targetFieldValue = returnedSource.get(targetField);
             assertNotNull(targetFieldValue);
-            assertThat(((Map<String, Object>) targetFieldValue).get("organization_name"), equalTo("OAKLAWN JOCKEY CLUB, INC."));
+            assertThat(((Map<String, Object>) targetFieldValue).get("organization_name"), equalTo("PT Nevigate Telekomunikasi Indonesia"));
         });
     }
 

modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java

@@ -285,7 +285,7 @@ public void testMetrics() throws Exception {
             final BlobStore blobStore = blobStoreRepository.blobStore();
             final BlobStore delegateBlobStore = ((BlobStoreWrapper) blobStore).delegate();
             final S3BlobStore s3BlobStore = (S3BlobStore) delegateBlobStore;
-            final Map<S3BlobStore.StatsKey, S3BlobStore.IgnoreNoResponseMetricsCollector> statsCollectors = s3BlobStore
+            final Map<S3BlobStore.StatsKey, S3BlobStore.ElasticsearchS3MetricsCollector> statsCollectors = s3BlobStore
                 .getStatsCollectors().collectors;
 
             final var plugins = internalCluster().getInstance(PluginsService.class, nodeName)

modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/EnterpriseGeoIpDownloaderIT.java

@@ -143,16 +143,17 @@ public TimeValue getCompareAndExchangeAntiContentionDelay() {
         return service.compareAndExchangeAntiContentionDelay;
     }
 
-    // metrics collector that ignores null responses that we interpret as the request not reaching the S3 endpoint due to a network
-    // issue
-    class IgnoreNoResponseMetricsCollector extends RequestMetricCollector {
+    /**
+     * A {@link RequestMetricCollector} that processes the metrics related to each API invocation attempt according to Elasticsearch's needs
+     */
+    class ElasticsearchS3MetricsCollector extends RequestMetricCollector {
 
         final LongAdder requests = new LongAdder();
         final LongAdder operations = new LongAdder();
         private final Operation operation;
         private final Map<String, Object> attributes;
 
-        private IgnoreNoResponseMetricsCollector(Operation operation, OperationPurpose purpose) {
+        private ElasticsearchS3MetricsCollector(Operation operation, OperationPurpose purpose) {
             this.operation = operation;
             this.attributes = RepositoriesMetrics.createAttributesMap(repositoryMetadata, purpose, operation.getKey());
         }
@@ -582,7 +583,7 @@ public String toString() {
     }
 
     class StatsCollectors {
-        final Map<StatsKey, IgnoreNoResponseMetricsCollector> collectors = new ConcurrentHashMap<>();
+        final Map<StatsKey, ElasticsearchS3MetricsCollector> collectors = new ConcurrentHashMap<>();
 
         RequestMetricCollector getMetricCollector(Operation operation, OperationPurpose purpose) {
             return collectors.computeIfAbsent(new StatsKey(operation, purpose), k -> buildMetricCollector(k.operation(), k.purpose()));
@@ -605,8 +606,8 @@ Map<String, BlobStoreActionStats> statsMap(boolean isStateless) {
             }
         }
 
-        IgnoreNoResponseMetricsCollector buildMetricCollector(Operation operation, OperationPurpose purpose) {
-            return new IgnoreNoResponseMetricsCollector(operation, purpose);
+        ElasticsearchS3MetricsCollector buildMetricCollector(Operation operation, OperationPurpose purpose) {
+            return new ElasticsearchS3MetricsCollector(operation, purpose);
         }
     }
 

modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java

@@ -86,9 +86,6 @@ tests:
 - class: org.elasticsearch.xpack.apmdata.APMYamlTestSuiteIT
   method: test {yaml=/10_apm/Test template reinstallation}
   issue: https://github.com/elastic/elasticsearch/issues/116445
-- class: org.elasticsearch.ingest.geoip.EnterpriseGeoIpDownloaderIT
-  method: testEnterpriseDownloaderTask
-  issue: https://github.com/elastic/elasticsearch/issues/115163
 - class: org.elasticsearch.versioning.ConcurrentSeqNoVersioningIT
   method: testSeqNoCASLinearizability
   issue: https://github.com/elastic/elasticsearch/issues/117249
@@ -233,9 +230,6 @@ tests:
 - class: org.elasticsearch.xpack.ilm.TimeSeriesLifecycleActionsIT
   method: testHistoryIsWrittenWithFailure
   issue: https://github.com/elastic/elasticsearch/issues/123203
-- class: org.elasticsearch.action.admin.indices.diskusage.IndexDiskUsageAnalyzerTests
-  method: testCompletionField
-  issue: https://github.com/elastic/elasticsearch/issues/123269
 - class: org.elasticsearch.packaging.test.DockerTests
   method: test151MachineDependentHeapWithSizeOverride
   issue: https://github.com/elastic/elasticsearch/issues/123437
@@ -377,9 +371,6 @@ tests:
 - class: org.elasticsearch.index.engine.ThreadPoolMergeExecutorServiceTests
   method: testIORateIsAdjustedForRunningMergeTasks
   issue: https://github.com/elastic/elasticsearch/issues/125842
-- class: org.elasticsearch.test.rest.yaml.RcsCcsCommonYamlTestSuiteIT
-  method: test {p0=search.vectors/41_knn_search_bbq_hnsw/Test index configured rescore vector score consistency}
-  issue: https://github.com/elastic/elasticsearch/issues/125851
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=transform/transforms_start_stop/Verify start transform creates destination index with appropriate mapping}
   issue: https://github.com/elastic/elasticsearch/issues/125854
@@ -389,9 +380,24 @@ tests:
 - class: org.elasticsearch.xpack.ilm.TimeSeriesDataStreamsIT
   method: testSearchableSnapshotAction
   issue: https://github.com/elastic/elasticsearch/issues/125867
-- class: org.elasticsearch.xpack.inference.registry.ModelRegistryMetadataTests
-  method: testUpgrade
-  issue: https://github.com/elastic/elasticsearch/issues/125554
+- class: org.elasticsearch.xpack.downsample.DataStreamLifecycleDownsampleDisruptionIT
+  method: testDataStreamLifecycleDownsampleRollingRestart
+  issue: https://github.com/elastic/elasticsearch/issues/123769
+- class: org.elasticsearch.multiproject.test.CoreWithMultipleProjectsClientYamlTestSuiteIT
+  method: test {yaml=search.vectors/41_knn_search_bbq_hnsw/Test index configured rescore vector score consistency}
+  issue: https://github.com/elastic/elasticsearch/issues/125902
+- class: org.elasticsearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT
+  method: test {yaml=cat.allocation/10_basic/Node forecasts}
+  issue: https://github.com/elastic/elasticsearch/issues/125848
+- class: org.elasticsearch.xpack.test.rest.XPackRestIT
+  method: test {p0=ml/start_data_frame_analytics/Test start given dest index is not empty}
+  issue: https://github.com/elastic/elasticsearch/issues/125909
+- class: org.elasticsearch.indices.stats.IndexStatsIT
+  method: testThrottleStats
+  issue: https://github.com/elastic/elasticsearch/issues/125910
+- class: org.elasticsearch.xpack.esql.action.ManyShardsIT
+  method: testCancelUnnecessaryRequests
+  issue: https://github.com/elastic/elasticsearch/issues/125947
 
 # Examples:
 #