Merge branch 'main' into pkar/msearch-flatworld-search-project-routing

Author: pawankartik-elastic

docs/changelog/138282.yaml

@@ -0,0 +1,5 @@
+pr: 138282
+summary: Allow fast blob-cache introspection by shard-id
+area: Searchable Snapshots
+type: enhancement
+issues: []

docs/changelog/138756.yaml

@@ -0,0 +1,5 @@
+pr: 138756
+summary: Add project routing support to JDBC
+area: SQL
+type: enhancement
+issues: []

docs/changelog/138876.yaml

@@ -0,0 +1,5 @@
+pr: 138876
+summary: "Use doc values skipper for @timestamp in synthetic `_id` postings #138568"
+area: TSDB
+type: enhancement
+issues: []

libs/exponential-histogram/src/main/java/org/elasticsearch/exponentialhistogram/ExponentialHistogramMerger.java

@@ -36,7 +36,8 @@
  * while keeping the bucket count in the result below a given limit.
  */
 public class ExponentialHistogramMerger implements Accountable, Releasable {
-
+    // OpenTelemetry SDK default, we might make this configurable later
+    private static final int MAX_HISTOGRAM_BUCKETS = 320;
     private static final long BASE_SIZE = RamUsageEstimator.shallowSizeOfInstance(ExponentialHistogramMerger.class) + DownscaleStats.SIZE;
 
     // Our algorithm is not in-place, therefore we use two histograms and ping-pong between them
@@ -53,6 +54,14 @@ public class ExponentialHistogramMerger implements Accountable, Releasable {
     private final ExponentialHistogramCircuitBreaker circuitBreaker;
     private boolean closed = false;
 
+    /**
+     * Creates a new instance with the OpenTelemetry SDK default bucket limit of {@link ExponentialHistogramMerger#MAX_HISTOGRAM_BUCKETS}
+     * @param circuitBreaker the circuit breaker to use to limit memory allocations
+     */
+    public static ExponentialHistogramMerger create(ExponentialHistogramCircuitBreaker circuitBreaker) {
+        return create(MAX_HISTOGRAM_BUCKETS, circuitBreaker);
+    }
+
     /**
      * Creates a new instance with the specified bucket limit.
      *

modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/rest/RestPutDataStreamLifecycleAction.java

@@ -11,6 +11,7 @@
 import org.elasticsearch.action.datastreams.lifecycle.PutDataStreamLifecycleAction;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.internal.node.NodeClient;
+import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
@@ -46,17 +47,12 @@ public List<Route> routes() {
     @Override
     protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
         try (XContentParser parser = request.contentParser()) {
-            PutDataStreamLifecycleAction.Request putLifecycleRequest = PutDataStreamLifecycleAction.Request.parseRequest(
-                parser,
-                (dataRetention, enabled, downsamplingRounds, downsamplingMethod) -> new PutDataStreamLifecycleAction.Request(
-                    getMasterNodeTimeout(request),
-                    getAckTimeout(request),
-                    Strings.splitStringByCommaToArray(request.param("name")),
-                    dataRetention,
-                    enabled,
-                    downsamplingRounds,
-                    downsamplingMethod
-                )
+            final var lifecycle = DataStreamLifecycle.dataLifecycleFromXContent(parser);
+            PutDataStreamLifecycleAction.Request putLifecycleRequest = new PutDataStreamLifecycleAction.Request(
+                getMasterNodeTimeout(request),
+                getAckTimeout(request),
+                Strings.splitStringByCommaToArray(request.param("name")),
+                lifecycle
             );
             putLifecycleRequest.indicesOptions(IndicesOptions.fromRequest(request, putLifecycleRequest.indicesOptions()));
             return channel -> client.execute(

modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataIndexTemplateServiceTests.java

@@ -151,12 +151,12 @@ public void testLifecycleComposition() {
         // One lifecycle results in this lifecycle as the final
         {
             ResettableValue<List<DataStreamLifecycle.DownsamplingRound>> downsamplingRounds = randomDownsampling();
-            DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.createDataLifecycleTemplate(
-                true,
-                randomRetention(),
-                downsamplingRounds,
-                randomResettable(() -> randomSamplingMethod(downsamplingRounds.get()))
-            );
+            DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder()
+                .enabled(true)
+                .dataRetention(randomRetention())
+                .downsamplingRounds(downsamplingRounds)
+                .downsamplingMethod(randomResettable(() -> randomSamplingMethod(downsamplingRounds.get())))
+                .buildTemplate();
             List<DataStreamLifecycle.Template> lifecycles = List.of(lifecycle);
             DataStreamLifecycle result = composeDataLifecycles(lifecycles).build();
             // Defaults to true
@@ -169,12 +169,12 @@ public void testLifecycleComposition() {
         // Enabled is always true unless it's explicitly set to false
         {
             List<DataStreamLifecycle.DownsamplingRound> downsamplingRounds = randomRounds();
-            DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.createDataLifecycleTemplate(
-                false,
-                randomPositiveTimeValue(),
-                downsamplingRounds,
-                randomSamplingMethod(downsamplingRounds)
-            );
+            DataStreamLifecycle.Template lifecycle = DataStreamLifecycle.dataLifecycleBuilder()
+                .enabled(false)
+                .dataRetention(randomPositiveTimeValue())
+                .downsamplingRounds(downsamplingRounds)
+                .downsamplingMethod(randomSamplingMethod(downsamplingRounds))
+                .buildTemplate();
             List<DataStreamLifecycle.Template> lifecycles = List.of(lifecycle, DataStreamLifecycle.Template.DATA_DEFAULT);
             DataStreamLifecycle result = composeDataLifecycles(lifecycles).build();
             assertThat(result.enabled(), equalTo(true));
@@ -185,20 +185,22 @@ public void testLifecycleComposition() {
         // If both lifecycles have all properties, then the latest one overwrites all the others
         {
             DownsampleConfig.SamplingMethod downsamplingMethod1 = randomFrom(DownsampleConfig.SamplingMethod.LAST_VALUE);
-            DataStreamLifecycle.Template lifecycle1 = DataStreamLifecycle.createDataLifecycleTemplate(
-                false,
-                randomPositiveTimeValue(),
-                randomRounds(),
-                downsamplingMethod1
-            );
-            DataStreamLifecycle.Template lifecycle2 = DataStreamLifecycle.createDataLifecycleTemplate(
-                true,
-                randomPositiveTimeValue(),
-                randomRounds(),
-                downsamplingMethod1 == DownsampleConfig.SamplingMethod.LAST_VALUE
-                    ? DownsampleConfig.SamplingMethod.AGGREGATE
-                    : DownsampleConfig.SamplingMethod.LAST_VALUE
-            );
+            DataStreamLifecycle.Template lifecycle1 = DataStreamLifecycle.dataLifecycleBuilder()
+                .enabled(false)
+                .dataRetention(randomPositiveTimeValue())
+                .downsamplingRounds(randomRounds())
+                .downsamplingMethod(downsamplingMethod1)
+                .buildTemplate();
+            DataStreamLifecycle.Template lifecycle2 = DataStreamLifecycle.dataLifecycleBuilder()
+                .enabled(true)
+                .dataRetention(randomPositiveTimeValue())
+                .downsamplingRounds(randomRounds())
+                .downsamplingMethod(
+                    downsamplingMethod1 == DownsampleConfig.SamplingMethod.LAST_VALUE
+                        ? DownsampleConfig.SamplingMethod.AGGREGATE
+                        : DownsampleConfig.SamplingMethod.LAST_VALUE
+                )
+                .buildTemplate();
             List<DataStreamLifecycle.Template> lifecycles = List.of(lifecycle1, lifecycle2);
             DataStreamLifecycle result = composeDataLifecycles(lifecycles).build();
             assertThat(result.enabled(), equalTo(lifecycle2.enabled()));

modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/GetDataStreamsResponseTests.java

@@ -154,7 +154,7 @@ public void testResponseIlmAndDataStreamLifecycleRepresentation() throws Excepti
                 .setGeneration(3)
                 .setAllowCustomRouting(true)
                 .setIndexMode(IndexMode.STANDARD)
-                .setLifecycle(DataStreamLifecycle.createDataLifecycle(false, null, null, null))
+                .setLifecycle(DataStreamLifecycle.dataLifecycleBuilder().enabled(false).build())
                 .setDataStreamOptions(DataStreamOptions.FAILURE_STORE_ENABLED)
                 .setFailureIndices(DataStream.DataStreamIndices.failureIndicesBuilder(failureStores).build())
                 .build();

modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java

@@ -148,12 +148,12 @@ static DataStreamLifecycle.Template randomDataLifecycleTemplate() {
         ResettableValue<List<DataStreamLifecycle.DownsamplingRound>> downsampling = randomResettable(
             DataStreamLifecycleFixtures::randomDownsamplingRounds
         );
-        return DataStreamLifecycle.createDataLifecycleTemplate(
-            frequently(),
-            randomResettable(ESTestCase::randomTimeValue),
-            downsampling,
-            randomResettable(() -> randomSamplingMethod(downsampling.get()))
-        );
+        return DataStreamLifecycle.dataLifecycleBuilder()
+            .enabled(frequently())
+            .dataRetention(randomResettable(ESTestCase::randomTimeValue))
+            .downsamplingRounds(downsampling)
+            .downsamplingMethod(randomResettable(() -> randomSamplingMethod(downsampling.get())))
+            .buildTemplate();
     }
 
     public static <T> ResettableValue<T> randomResettable(Supplier<T> supplier) {

server/src/main/java/module-info.java

@@ -396,10 +396,10 @@
         to
             org.elasticsearch.inference,
             org.elasticsearch.metering,
-            org.elasticsearch.stateless,
             org.elasticsearch.settings.secure,
             org.elasticsearch.serverless.constants,
             org.elasticsearch.serverless.apifiltering,
+            org.elasticsearch.serverless.stateless,
             org.elasticsearch.internal.security,
             org.elasticsearch.xpack.gpu;
 

server/src/main/java/org/elasticsearch/action/datastreams/lifecycle/PutDataStreamLifecycleAction.java

@@ -11,7 +11,6 @@
 
 import org.elasticsearch.action.ActionType;
 import org.elasticsearch.action.IndicesRequest;
-import org.elasticsearch.action.downsample.DownsampleConfig;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.master.AcknowledgedRequest;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
@@ -20,23 +19,13 @@
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.xcontent.AbstractObjectParser;
-import org.elasticsearch.xcontent.ConstructingObjectParser;
-import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
-import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.Arrays;
-import java.util.List;
 import java.util.Objects;
 
-import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.DATA_RETENTION_FIELD;
-import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.DOWNSAMPLING_FIELD;
-import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.DOWNSAMPLING_METHOD_FIELD;
-import static org.elasticsearch.cluster.metadata.DataStreamLifecycle.ENABLED_FIELD;
-
 /**
  * Sets the data stream lifecycle that was provided in the request to the requested data streams.
  */
@@ -48,53 +37,6 @@ private PutDataStreamLifecycleAction() {/* no instances */}
 
     public static final class Request extends AcknowledgedRequest<Request> implements IndicesRequest.Replaceable, ToXContentObject {
 
-        public interface Factory {
-            Request create(
-                @Nullable TimeValue dataRetention,
-                @Nullable Boolean enabled,
-                @Nullable List<DataStreamLifecycle.DownsamplingRound> downsampling,
-                @Nullable DownsampleConfig.SamplingMethod downsamplingMethod
-            );
-        }
-
-        @SuppressWarnings("unchecked")
-        public static final ConstructingObjectParser<Request, Factory> PARSER = new ConstructingObjectParser<>(
-            "put_data_stream_lifecycle_request",
-            false,
-            (args, factory) -> factory.create(
-                (TimeValue) args[0],
-                (Boolean) args[1],
-                (List<DataStreamLifecycle.DownsamplingRound>) args[2],
-                (DownsampleConfig.SamplingMethod) args[3]
-            )
-        );
-
-        static {
-            PARSER.declareField(
-                ConstructingObjectParser.optionalConstructorArg(),
-                (p, c) -> TimeValue.parseTimeValue(p.text(), DATA_RETENTION_FIELD.getPreferredName()),
-                DATA_RETENTION_FIELD,
-                ObjectParser.ValueType.STRING
-            );
-            PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), ENABLED_FIELD);
-            PARSER.declareField(
-                ConstructingObjectParser.optionalConstructorArg(),
-                (p, c) -> AbstractObjectParser.parseArray(p, null, DataStreamLifecycle.DownsamplingRound::fromXContent),
-                DOWNSAMPLING_FIELD,
-                ObjectParser.ValueType.OBJECT_ARRAY
-            );
-            PARSER.declareField(
-                ConstructingObjectParser.optionalConstructorArg(),
-                (p, c) -> DownsampleConfig.SamplingMethod.fromString(p.text()),
-                DOWNSAMPLING_METHOD_FIELD,
-                ObjectParser.ValueType.STRING
-            );
-        }
-
-        public static Request parseRequest(XContentParser parser, Factory factory) {
-            return PARSER.apply(parser, factory);
-        }
-
         private String[] names;
         private IndicesOptions indicesOptions = IndicesOptions.builder()
             .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ERROR_WHEN_UNAVAILABLE_TARGETS)
@@ -150,26 +92,12 @@ public Request(
             @Nullable TimeValue dataRetention,
             @Nullable Boolean enabled
         ) {
-            this(masterNodeTimeout, ackTimeout, names, dataRetention, enabled, null, null);
-        }
-
-        public Request(
-            TimeValue masterNodeTimeout,
-            TimeValue ackTimeout,
-            String[] names,
-            @Nullable TimeValue dataRetention,
-            @Nullable Boolean enabled,
-            @Nullable List<DataStreamLifecycle.DownsamplingRound> downsamplingRounds,
-            @Nullable DownsampleConfig.SamplingMethod downsamplingMethod
-        ) {
-            super(masterNodeTimeout, ackTimeout);
-            this.names = names;
-            this.lifecycle = DataStreamLifecycle.dataLifecycleBuilder()
-                .dataRetention(dataRetention)
-                .enabled(enabled == null || enabled)
-                .downsamplingRounds(downsamplingRounds)
-                .downsamplingMethod(downsamplingMethod)
-                .build();
+            this(
+                masterNodeTimeout,
+                ackTimeout,
+                names,
+                DataStreamLifecycle.dataLifecycleBuilder().enabled(enabled == null || enabled).dataRetention(dataRetention).build()
+            );
         }
 
         public String[] getNames() {