Merge remote-tracking branch 'origin/main' into lucene_snapshot

Author: elasticsearchmachine

BUILDING.md

@@ -82,7 +82,7 @@ In case of updating a dependency, ensure to remove the unused entry of the outda
 
 You can also automate the generation of this entry by running your build using the `--write-verification-metadata` commandline option:
 ```
->./gradlew --write-verification-metadata sha256 precommit
+./gradlew --write-verification-metadata sha256 precommit
 ```
 
 The `--write-verification-metadata` Gradle option is generally able to resolve reachable configurations,
@@ -92,10 +92,10 @@ uses the changed dependencies. In most cases, `precommit` or `check` are good ca
 We prefer sha256 checksums as md5 and sha1 are not considered safe anymore these days. The generated entry
 will have the `origin` attribute been set to `Generated by Gradle`.
 
->A manual confirmation of the Gradle generated checksums is currently not mandatory.
->If you want to add a level of verification you can manually confirm the checksum (e.g. by looking it up on the website of the library)
->Please replace the content of the `origin` attribute by `official site` in that case.
->
+> [!Tip]  
+> A manual confirmation of the Gradle generated checksums is currently not mandatory.
+> If you want to add a level of verification you can manually confirm the checksum (e.g. by looking it up on the website of the library)
+> Please replace the content of the `origin` attribute by `official site` in that case.
 
 #### Custom plugin and task implementations
 
@@ -229,13 +229,9 @@ In addition to snapshot builds JitPack supports building Pull Requests. Simply u
 3. Run the Gradle build as needed. Keep in mind the initial resolution might take a bit longer as this needs to be built
 by JitPack in the background before we can resolve the adhoc built dependency.
 
----
-
-**NOTE**
-
-You should only use that approach locally or on a developer branch for production dependencies as we do
+> [!Note]  
+> You should only use that approach locally or on a developer branch for production dependencies as we do
 not want to ship unreleased libraries into our releases.
----
 
 #### How to use a custom third party artifact?
 
@@ -265,12 +261,9 @@ allprojects {
   ```
 4. Run the Gradle build as needed with `--write-verification-metadata` to ensure the Gradle dependency verification does not fail on your custom dependency.
 
----
-**NOTE**
-
-As Gradle prefers to use modules whose descriptor has been created from real meta-data rather than being generated,
+> [!Note]  
+> As Gradle prefers to use modules whose descriptor has been created from real meta-data rather than being generated,
 flat directory repositories cannot be used to override artifacts with real meta-data from other repositories declared in the build.
-For example, if Gradle finds only `jmxri-1.2.1.jar` in a flat directory repository, but `jmxri-1.2.1.pom` in another repository
+> For example, if Gradle finds only `jmxri-1.2.1.jar` in a flat directory repository, but `jmxri-1.2.1.pom` in another repository
 that supports meta-data, it will use the second repository to provide the module.
-Therefore, it is recommended to declare a version that is not resolvable from public repositories we use (e.g. Maven Central)
----
+> Therefore, it is recommended to declare a version that is not resolvable from public repositories we use (e.g. Maven Central)

CONTRIBUTING.md

@@ -401,7 +401,8 @@ It is important that the only code covered by the Elastic licence is contained
 within the top-level `x-pack` directory. The build will fail its pre-commit
 checks if contributed code does not have the appropriate license headers.
 
-> **NOTE:** If you have imported the project into IntelliJ IDEA the project will
+> [!NOTE]
+> If you have imported the project into IntelliJ IDEA the project will
 > be automatically configured to add the correct license header to new source
 > files based on the source location.
 

modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java

@@ -254,7 +254,21 @@ protected S3ClientBuilder buildClientBuilder(S3ClientSettings clientSettings, Sd
         }
 
         if (Strings.hasLength(clientSettings.endpoint)) {
-            s3clientBuilder.endpointOverride(URI.create(clientSettings.endpoint));
+            String endpoint = clientSettings.endpoint;
+            if ((endpoint.startsWith("http://") || endpoint.startsWith("https://")) == false) {
+                // The SDK does not know how to interpret endpoints without a scheme prefix and will error. Therefore, when the scheme is
+                // absent, we'll supply HTTPS as a default to avoid errors.
+                // See https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/client-configuration.html#client-config-other-diffs
+                endpoint = "https://" + endpoint;
+                LOGGER.warn(
+                    """
+                        found S3 client with endpoint [{}] that is missing a scheme, guessing it should use 'https://'; \
+                        to suppress this warning, add a scheme prefix to the [{}] setting on this node""",
+                    clientSettings.endpoint,
+                    S3ClientSettings.ENDPOINT_SETTING.getConcreteSettingForNamespace("CLIENT_NAME").getKey()
+                );
+            }
+            s3clientBuilder.endpointOverride(URI.create(endpoint));
         }
 
         return s3clientBuilder;

modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3ServiceTests.java

@@ -12,7 +12,9 @@
 import software.amazon.awssdk.awscore.exception.AwsServiceException;
 import software.amazon.awssdk.core.retry.RetryPolicyContext;
 import software.amazon.awssdk.core.retry.conditions.RetryCondition;
+import software.amazon.awssdk.http.SdkHttpClient;
 import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
 import software.amazon.awssdk.services.s3.endpoints.S3EndpointParams;
 import software.amazon.awssdk.services.s3.endpoints.internal.DefaultS3EndpointProvider;
 import software.amazon.awssdk.services.s3.model.S3Exception;
@@ -29,8 +31,10 @@
 import org.elasticsearch.watcher.ResourceWatcherService;
 
 import java.io.IOException;
+import java.net.URI;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.Mockito.mock;
 
 public class S3ServiceTests extends ESTestCase {
@@ -217,4 +221,22 @@ public void testGetClientRegionFallbackToUsEast1() {
             );
         }
     }
+
+    public void testEndpointOverrideSchemeDefaultsToHttpsWhenNotSpecified() {
+        final S3Service s3Service = new S3Service(
+            mock(Environment.class),
+            Settings.EMPTY,
+            mock(ResourceWatcherService.class),
+            () -> Region.of("es-test-region")
+        );
+        final String endpointWithoutScheme = randomIdentifier() + ".ignore";
+        S3Client s3Client = s3Service.buildClient(
+            S3ClientSettings.getClientSettings(
+                Settings.builder().put("s3.client.test-client.endpoint", endpointWithoutScheme).build(),
+                "test-client"
+            ),
+            mock(SdkHttpClient.class)
+        );
+        assertThat(s3Client.serviceClientConfiguration().endpointOverride().get(), equalTo(URI.create("https://" + endpointWithoutScheme)));
+    }
 }

muted-tests.yml

@@ -450,6 +450,15 @@ tests:
 - class: org.elasticsearch.xpack.ccr.action.ShardFollowTaskReplicationTests
   method: testChangeFollowerHistoryUUID
   issue: https://github.com/elastic/elasticsearch/issues/127680
+- class: org.elasticsearch.action.admin.indices.diskusage.IndexDiskUsageAnalyzerTests
+  method: testKnnVectors
+  issue: https://github.com/elastic/elasticsearch/issues/127689
+- class: org.elasticsearch.snapshots.SnapshotShutdownIT
+  method: testSnapshotShutdownProgressTracker
+  issue: https://github.com/elastic/elasticsearch/issues/127690
+- class: org.elasticsearch.xpack.core.template.IndexTemplateRegistryRolloverIT
+  method: testRollover
+  issue: https://github.com/elastic/elasticsearch/issues/127692
 
 # Examples:
 #

rest-api-spec/README.markdown

@@ -58,8 +58,8 @@ The specification contains:
 * Request parameters
 * Request body specification
 
-**NOTE**
-If an API is stable but it response should be treated as an arbitrary map of key values please notate this as followed
+> [!Note]
+> If an API is stable but it response should be treated as an arbitrary map of key values please notate this as followed
 
 ```json
 {

server/src/main/java/org/elasticsearch/common/metrics/ExponentialBucketHistogram.java

@@ -0,0 +1,131 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.common.metrics;
+
+import java.util.Arrays;
+import java.util.concurrent.atomic.LongAdder;
+
+/**
+ * A histogram with a fixed number of buckets of exponentially increasing width.
+ * <p>
+ * The bucket boundaries are defined by increasing powers of two, e.g.
+ * <code>
+ *     (-&infin;, 1), [1, 2), [2, 4), [4, 8), ..., [2^({@link #bucketCount}-2), &infin;)
+ * </code>
+ * There are {@link #bucketCount} buckets.
+ */
+public class ExponentialBucketHistogram {
+
+    private final int bucketCount;
+    private final long lastBucketLowerBound;
+
+    public static int[] getBucketUpperBounds(int bucketCount) {
+        int[] bounds = new int[bucketCount - 1];
+        for (int i = 0; i < bounds.length; i++) {
+            bounds[i] = 1 << i;
+        }
+        return bounds;
+    }
+
+    private int getBucket(long observedValue) {
+        if (observedValue <= 0) {
+            return 0;
+        } else if (lastBucketLowerBound <= observedValue) {
+            return bucketCount - 1;
+        } else {
+            return Long.SIZE - Long.numberOfLeadingZeros(observedValue);
+        }
+    }
+
+    private final LongAdder[] buckets;
+
+    public ExponentialBucketHistogram(int bucketCount) {
+        if (bucketCount < 2 || bucketCount > Integer.SIZE) {
+            throw new IllegalArgumentException("Bucket count must be in [2, " + Integer.SIZE + "], got " + bucketCount);
+        }
+        this.bucketCount = bucketCount;
+        this.lastBucketLowerBound = getBucketUpperBounds(bucketCount)[bucketCount - 2];
+        buckets = new LongAdder[bucketCount];
+        for (int i = 0; i < bucketCount; i++) {
+            buckets[i] = new LongAdder();
+        }
+    }
+
+    public int[] calculateBucketUpperBounds() {
+        return getBucketUpperBounds(bucketCount);
+    }
+
+    public void addObservation(long observedValue) {
+        buckets[getBucket(observedValue)].increment();
+    }
+
+    /**
+     * @return An array of frequencies of handling times in buckets with upper bounds as returned by {@link #calculateBucketUpperBounds()},
+     *         plus an extra bucket for handling times longer than the longest upper bound.
+     */
+    public long[] getSnapshot() {
+        final long[] histogram = new long[bucketCount];
+        for (int i = 0; i < bucketCount; i++) {
+            histogram[i] = buckets[i].longValue();
+        }
+        return histogram;
+    }
+
+    /**
+     * Calculate the Nth percentile value
+     *
+     * @param percentile The percentile as a fraction (in [0, 1.0])
+     * @return A value greater than the specified fraction of values in the histogram
+     * @throws IllegalArgumentException if the requested percentile is invalid
+     */
+    public long getPercentile(float percentile) {
+        return getPercentile(percentile, getSnapshot(), calculateBucketUpperBounds());
+    }
+
+    /**
+     * Calculate the Nth percentile value
+     *
+     * @param percentile The percentile as a fraction (in [0, 1.0])
+     * @param snapshot An array of frequencies of handling times in buckets with upper bounds as per {@link #calculateBucketUpperBounds()}
+     * @param bucketUpperBounds The upper bounds of the buckets in the histogram, as per {@link #calculateBucketUpperBounds()}
+     * @return A value greater than the specified fraction of values in the histogram
+     * @throws IllegalArgumentException if the requested percentile is invalid
+     */
+    public long getPercentile(float percentile, long[] snapshot, int[] bucketUpperBounds) {
+        assert snapshot.length == bucketCount && bucketUpperBounds.length == bucketCount - 1;
+        if (percentile < 0 || percentile > 1) {
+            throw new IllegalArgumentException("Requested percentile must be in [0, 1.0], percentile=" + percentile);
+        }
+        final long totalCount = Arrays.stream(snapshot).sum();
+        long percentileIndex = (long) Math.ceil(totalCount * percentile);
+        // Find which bucket has the Nth percentile value and return the upper bound value.
+        for (int i = 0; i < bucketCount; i++) {
+            percentileIndex -= snapshot[i];
+            if (percentileIndex <= 0) {
+                if (i == snapshot.length - 1) {
+                    return Long.MAX_VALUE;
+                } else {
+                    return bucketUpperBounds[i];
+                }
+            }
+        }
+        assert false : "We shouldn't ever get here";
+        return Long.MAX_VALUE;
+    }
+
+    /**
+     * Clear all values in the histogram (non-atomic)
+     */
+    public void clear() {
+        for (int i = 0; i < bucketCount; i++) {
+            buckets[i].reset();
+        }
+    }
+}

server/src/main/java/org/elasticsearch/common/network/HandlingTimeTracker.java

@@ -9,58 +9,20 @@
 
 package org.elasticsearch.common.network;
 
-import java.util.concurrent.atomic.LongAdder;
+import org.elasticsearch.common.metrics.ExponentialBucketHistogram;
 
 /**
  * Tracks how long message handling takes on a transport thread as a histogram with fixed buckets.
  */
-public class HandlingTimeTracker {
+public class HandlingTimeTracker extends ExponentialBucketHistogram {
 
-    public static int[] getBucketUpperBounds() {
-        int[] bounds = new int[17];
-        for (int i = 0; i < bounds.length; i++) {
-            bounds[i] = 1 << i;
-        }
-        return bounds;
-    }
+    public static final int BUCKET_COUNT = 18;
 
-    private static int getBucket(long handlingTimeMillis) {
-        if (handlingTimeMillis <= 0) {
-            return 0;
-        } else if (LAST_BUCKET_LOWER_BOUND <= handlingTimeMillis) {
-            return BUCKET_COUNT - 1;
-        } else {
-            return Long.SIZE - Long.numberOfLeadingZeros(handlingTimeMillis);
-        }
+    public static int[] getBucketUpperBounds() {
+        return ExponentialBucketHistogram.getBucketUpperBounds(BUCKET_COUNT);
     }
 
-    public static final int BUCKET_COUNT = getBucketUpperBounds().length + 1;
-
-    private static final long LAST_BUCKET_LOWER_BOUND = getBucketUpperBounds()[BUCKET_COUNT - 2];
-
-    private final LongAdder[] buckets;
-
     public HandlingTimeTracker() {
-        buckets = new LongAdder[BUCKET_COUNT];
-        for (int i = 0; i < BUCKET_COUNT; i++) {
-            buckets[i] = new LongAdder();
-        }
+        super(BUCKET_COUNT);
     }
-
-    public void addHandlingTime(long handlingTimeMillis) {
-        buckets[getBucket(handlingTimeMillis)].increment();
-    }
-
-    /**
-     * @return An array of frequencies of handling times in buckets with upper bounds as returned by {@link #getBucketUpperBounds()}, plus
-     *         an extra bucket for handling times longer than the longest upper bound.
-     */
-    public long[] getHistogram() {
-        final long[] histogram = new long[BUCKET_COUNT];
-        for (int i = 0; i < BUCKET_COUNT; i++) {
-            histogram[i] = buckets[i].longValue();
-        }
-        return histogram;
-    }
-
 }

server/src/main/java/org/elasticsearch/http/AbstractHttpServerTransport.java

@@ -460,7 +460,7 @@ public void incomingRequest(final HttpRequest httpRequest, final HttpChannel htt
             handleIncomingRequest(httpRequest, trackingChannel, httpRequest.getInboundException());
         } finally {
             final long took = threadPool.rawRelativeTimeInMillis() - startTime;
-            networkService.getHandlingTimeTracker().addHandlingTime(took);
+            networkService.getHandlingTimeTracker().addObservation(took);
             final long logThreshold = slowLogThresholdMs;
             if (logThreshold > 0 && took > logThreshold) {
                 logger.warn(

server/src/main/java/org/elasticsearch/http/HttpRouteStatsTracker.java

@@ -91,7 +91,7 @@ public void addResponseStats(long contentLength) {
     }
 
     public void addResponseTime(long timeMillis) {
-        responseTimeTracker.addHandlingTime(timeMillis);
+        responseTimeTracker.addObservation(timeMillis);
     }
 
     public HttpRouteStats getStats() {
@@ -102,7 +102,7 @@ public HttpRouteStats getStats() {
             responseStats.count().longValue(),
             responseStats.totalSize().longValue(),
             responseStats.getHistogram(),
-            responseTimeTracker.getHistogram()
+            responseTimeTracker.getSnapshot()
         );
     }
 }