Merge remote-tracking branch 'es/main' into ds_rollover_tsdb_to_logsdb

Author: martijnvg

.buildkite/scripts/dra-workflow.sh

@@ -70,6 +70,7 @@ echo --- Building release artifacts
   $VERSION_QUALIFIER_ARG \
   buildReleaseArtifacts \
   exportCompressedDockerImages \
+  exportDockerContexts \
   :distribution:generateDependenciesReport
 
 PATH="$PATH:${JAVA_HOME}/bin" # Required by the following script

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java

@@ -14,7 +14,10 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.metadata.ProjectId;
+import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.routing.GlobalRoutingTable;
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.routing.ShardRouting;
 import org.elasticsearch.cluster.routing.allocation.AllocationService;
@@ -126,19 +129,20 @@ public void setUp() throws Exception {
             Settings.builder().put("cluster.routing.allocation.awareness.attributes", "tag").build()
         );
 
-        Metadata.Builder mb = Metadata.builder();
+        final ProjectId projectId = ProjectId.DEFAULT;
+        ProjectMetadata.Builder pmb = ProjectMetadata.builder(projectId);
         for (int i = 1; i <= numIndices; i++) {
-            mb.put(
+            pmb.put(
                 IndexMetadata.builder("test_" + i)
                     .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()))
                     .numberOfShards(numShards)
                     .numberOfReplicas(numReplicas)
             );
         }
-        Metadata metadata = mb.build();
+        Metadata metadata = Metadata.builder().put(pmb).build();
         RoutingTable.Builder rb = RoutingTable.builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY);
         for (int i = 1; i <= numIndices; i++) {
-            rb.addAsNew(metadata.getProject().index("test_" + i));
+            rb.addAsNew(metadata.getProject(projectId).index("test_" + i));
         }
         RoutingTable routingTable = rb.build();
         DiscoveryNodes.Builder nb = DiscoveryNodes.builder();
@@ -151,7 +155,7 @@ public void setUp() throws Exception {
         }
         initialClusterState = ClusterState.builder(ClusterName.DEFAULT)
             .metadata(metadata)
-            .routingTable(routingTable)
+            .routingTable(GlobalRoutingTable.builder().put(projectId, routingTable).build())
             .nodes(nb)
             .nodeIdsToCompatibilityVersions(compatibilityVersions)
             .build();

distribution/docker/build.gradle

@@ -132,7 +132,7 @@ dependencies {
   fips "org.bouncycastle:bctls-fips:1.0.19"
 }
 
-ext.expansions = { Architecture architecture, DockerBase base ->
+ext.expansions = { Architecture architecture, DockerBase base, String publicationContext = '' ->
   def (major, minor) = VersionProperties.elasticsearch.split("\\.")
 
   // We tag our Docker images with various pieces of information, including a timestamp
@@ -152,6 +152,7 @@ ext.expansions = { Architecture architecture, DockerBase base ->
     'license'            : base == DockerBase.IRON_BANK ? 'Elastic License 2.0' : 'Elastic-License-2.0',
     'package_manager'    : base.packageManager,
     'docker_base'        : base.name().toLowerCase(),
+    'docker_context'     : publicationContext,
     'version'            : VersionProperties.elasticsearch,
     'major_minor_version': "${major}.${minor}",
     'retry'              : ShellRetry
@@ -179,9 +180,9 @@ private static String taskName(String prefix, Architecture architecture, DockerB
     suffix
 }
 
-ext.dockerBuildContext = { Architecture architecture, DockerBase base ->
+ext.dockerBuildContext = { Architecture architecture, DockerBase base, String publicationContext = '' ->
   copySpec {
-    final Map<String, String> varExpansions = expansions(architecture, base)
+    final Map<String, String> varExpansions = expansions(architecture, base, publicationContext)
     final Path projectDir = project.projectDir.toPath()
 
     if (base == DockerBase.IRON_BANK) {
@@ -291,17 +292,22 @@ tasks.named("composeUp").configure {
   dependsOn tasks.named("preProcessFixture")
 }
 
-void addBuildDockerContextTask(Architecture architecture, DockerBase base) {
+
+def exportDockerImages = tasks.register("exportDockerImages")
+def exportCompressedDockerImages = tasks.register("exportCompressedDockerImages")
+def exportDockerContexts = tasks.register("exportDockerContexts")
+
+void addBuildDockerContextTask(Architecture architecture, DockerBase base, String taskSuffix = 'DockerContext', String classifier = "docker-build-context") {
   String configDirectory = base == DockerBase.IRON_BANK ? 'scripts' : 'config'
   String arch = architecture == Architecture.AARCH64 ? '-aarch64' : ''
 
   final TaskProvider<Tar> buildDockerContextTask =
-    tasks.register(taskName('build', architecture, base, 'DockerContext'), Tar) {
+    tasks.register(taskName('build', architecture, base, taskSuffix), Tar) {
       archiveExtension = 'tar.gz'
       compression = Compression.GZIP
-      archiveClassifier = "docker-build-context${arch}"
+      archiveClassifier = "${classifier}${arch}"
       archiveBaseName = "elasticsearch${base.suffix}"
-      with dockerBuildContext(architecture, base)
+      with dockerBuildContext(architecture, base, classifier)
 
       into(configDirectory) {
         from(configurations.log4jConfig) {
@@ -344,6 +350,10 @@ void addBuildDockerContextTask(Architecture architecture, DockerBase base) {
       onlyIf("$architecture supported") { serviceProvider.get().isArchitectureSupported(architecture) }
     }
 
+  exportDockerContexts.configure {
+    dependsOn buildDockerContextTask
+  }
+
   if (base == DockerBase.IRON_BANK) {
     tasks.named("assemble").configure {
       dependsOn(buildDockerContextTask)
@@ -578,12 +588,14 @@ for (final Architecture architecture : Architecture.values()) {
       addTransformDockerContextTask(architecture, base)
       addBuildDockerImageTask(architecture, base)
     }
+    if(base == DockerBase.DEFAULT) {
+      // Add additional docker hub specific context which we use solely for publishing to docker hub.
+      // At the moment it only differs in not labels added that we need for openshift certification
+      addBuildDockerContextTask(architecture, base, 'DockerHubContext', "docker-hub-build-context")
+    }
   }
 }
 
-def exportDockerImages = tasks.register("exportDockerImages")
-def exportCompressedDockerImages = tasks.register("exportCompressedDockerImages")
-
 /*
  * The export subprojects write out the generated Docker images to disk, so
  * that they can be easily reloaded, for example into a VM for distribution testing

distribution/docker/src/docker/Dockerfile.default

@@ -139,13 +139,15 @@ LABEL org.label-schema.build-date="${build_date}" \\
   org.opencontainers.image.vendor="Elastic" \\
   org.opencontainers.image.version="${version}"
 
+<% if (docker_context != 'docker-hub-build-context') { %>
 LABEL name="Elasticsearch" \\
   maintainer="[email protected]" \\
   vendor="Elastic" \\
   version="${version}" \\
   release="1" \\
   summary="Elasticsearch" \\
   description="You know, for search."
+<% } %>
 
 RUN mkdir /licenses && ln LICENSE.txt /licenses/LICENSE
 

docs/changelog/126637.yaml

@@ -0,0 +1,5 @@
+pr: 126637
+summary: Improve resiliency of `UpdateTimeSeriesRangeService`
+area: TSDB
+type: bug
+issues: []

docs/reference/query-languages/esql/functions-operators/grouping-functions.md

@@ -16,6 +16,10 @@ The [`STATS`](/reference/query-languages/esql/commands/processing-commands.md#es
 :::{include} ../_snippets/functions/layout/bucket.md
 :::
 
+:::{note} 
+The `CATEGORIZE` function requires a [platinum license](https://www.elastic.co/subscriptions).
+:::
+
 :::{include} ../_snippets/functions/layout/categorize.md
 :::
 

docs/release-notes/breaking-changes.md

@@ -20,12 +20,12 @@ Aggregations:
 Allocation:
 * Increase minimum threshold in shard balancer [#115831](https://github.com/elastic/elasticsearch/pull/115831)
 * Remove `cluster.routing.allocation.disk.watermark.enable_for_single_data_node` setting [#114207](https://github.com/elastic/elasticsearch/pull/114207)
-* Remove cluster state from `/_cluster/reroute` response [#114231](https://github.com/elastic/elasticsearch/pull/114231) (issue: {es-issue}88978[#88978])
+* Remove cluster state from `/_cluster/reroute` response [#114231](https://github.com/elastic/elasticsearch/pull/114231) (issue: https://github.com/elastic/elasticsearch/issues/88978[#88978])
 
 Analysis:
 * Snowball stemmers have been upgraded [#114146](https://github.com/elastic/elasticsearch/pull/114146)
 * The 'german2' stemmer is now an alias for the 'german' snowball stemmer [#113614](https://github.com/elastic/elasticsearch/pull/113614)
-* The 'persian' analyzer has stemmer by default [#113482](https://github.com/elastic/elasticsearch/pull/113482) (issue: {es-issue}113050[#113050])
+* The 'persian' analyzer has stemmer by default [#113482](https://github.com/elastic/elasticsearch/pull/113482) (issue: https://github.com/elastic/elasticsearch/issues/113050[#113050])
 * The Korean dictionary for Nori has been updated [#114124](https://github.com/elastic/elasticsearch/pull/114124)
 
 Authentication:
@@ -56,32 +56,32 @@ Indices APIs:
 Infra/Core:
 * Change Elasticsearch timeouts to 429 response instead of 5xx [#116026](https://github.com/elastic/elasticsearch/pull/116026)
 * Limit `ByteSizeUnit` to 2 decimals [#120142](https://github.com/elastic/elasticsearch/pull/120142)
-* Remove `client.type` setting [#118192](https://github.com/elastic/elasticsearch/pull/118192) (issue: {es-issue}104574[#104574])
+* Remove `client.type` setting [#118192](https://github.com/elastic/elasticsearch/pull/118192) (issue: https://github.com/elastic/elasticsearch/issues/104574[#104574])
 * Remove any references to org.elasticsearch.core.RestApiVersion#V_7 [#118103](https://github.com/elastic/elasticsearch/pull/118103)
 
 Infra/Logging:
-* Change `deprecation.elasticsearch` keyword to `elasticsearch.deprecation` [#117933](https://github.com/elastic/elasticsearch/pull/117933) (issue: {es-issue}83251[#83251])
-* Rename deprecation index template [#125606](https://github.com/elastic/elasticsearch/pull/125606) (issue: {es-issue}125445[#125445])
+* Change `deprecation.elasticsearch` keyword to `elasticsearch.deprecation` [#117933](https://github.com/elastic/elasticsearch/pull/117933) (issue: https://github.com/elastic/elasticsearch/issues/83251[#83251])
+* Rename deprecation index template [#125606](https://github.com/elastic/elasticsearch/pull/125606) (issue: https://github.com/elastic/elasticsearch/issues/125445[#125445])
 
 Infra/Metrics:
 * Deprecated tracing.apm.* settings got removed. [#119926](https://github.com/elastic/elasticsearch/pull/119926)
 
 Infra/REST API:
-* Output a consistent format when generating error json [#90529](https://github.com/elastic/elasticsearch/pull/90529) (issue: {es-issue}89387[#89387])
+* Output a consistent format when generating error json [#90529](https://github.com/elastic/elasticsearch/pull/90529) (issue: https://github.com/elastic/elasticsearch/issues/89387[#89387])
 
 Ingest Node:
 * Remove `ecs` option on `user_agent` processor [#116077](https://github.com/elastic/elasticsearch/pull/116077)
 * Remove ignored fallback option on GeoIP processor [#116112](https://github.com/elastic/elasticsearch/pull/116112)
 
 Logs:
-* Conditionally enable logsdb by default for data streams matching with logs-*-* pattern. [#121049](https://github.com/elastic/elasticsearch/pull/121049) (issue: {es-issue}106489[#106489])
+* Conditionally enable logsdb by default for data streams matching with logs-*-* pattern. [#121049](https://github.com/elastic/elasticsearch/pull/121049) (issue: https://github.com/elastic/elasticsearch/issues/106489[#106489])
 
 Machine Learning:
 * Disable machine learning on macOS x86_64 [#104125](https://github.com/elastic/elasticsearch/pull/104125)
 
 Mapping:
 * Remove support for type, fields, `copy_to` and boost in metadata field definition [#118825](https://github.com/elastic/elasticsearch/pull/118825)
-* Turn `_source` meta fieldmapper's mode attribute into a no-op [#119072](https://github.com/elastic/elasticsearch/pull/119072) (issue: {es-issue}118596[#118596])
+* Turn `_source` meta fieldmapper's mode attribute into a no-op [#119072](https://github.com/elastic/elasticsearch/pull/119072) (issue: https://github.com/elastic/elasticsearch/issues/118596[#118596])
 
 Search:
 * Adjust `random_score` default field to `_seq_no` field [#118671](https://github.com/elastic/elasticsearch/pull/118671)

modules/data-streams/src/main/java/org/elasticsearch/datastreams/UpdateTimeSeriesRangeService.java

@@ -127,14 +127,14 @@ private ProjectMetadata.Builder updateTimeSeriesTemporalRange(ProjectMetadata pr
 
             // getWriteIndex() selects the latest added index:
             Index head = dataStream.getWriteIndex();
-            IndexMetadata im = project.getIndexSafe(head);
-            Instant currentEnd = IndexSettings.TIME_SERIES_END_TIME.get(im.getSettings());
-            TimeValue lookAheadTime = DataStreamsPlugin.getLookAheadTime(im.getSettings());
-            Instant newEnd = DataStream.getCanonicalTimestampBound(
-                now.plus(lookAheadTime.getMillis(), ChronoUnit.MILLIS).plus(pollInterval.getMillis(), ChronoUnit.MILLIS)
-            );
-            if (newEnd.isAfter(currentEnd)) {
-                try {
+            try {
+                IndexMetadata im = project.getIndexSafe(head);
+                Instant currentEnd = IndexSettings.TIME_SERIES_END_TIME.get(im.getSettings());
+                TimeValue lookAheadTime = DataStreamsPlugin.getLookAheadTime(im.getSettings());
+                Instant newEnd = DataStream.getCanonicalTimestampBound(
+                    now.plus(lookAheadTime.getMillis(), ChronoUnit.MILLIS).plus(pollInterval.getMillis(), ChronoUnit.MILLIS)
+                );
+                if (newEnd.isAfter(currentEnd)) {
                     Settings settings = Settings.builder()
                         .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), DEFAULT_DATE_TIME_FORMATTER.format(newEnd))
                         .build();
@@ -151,17 +151,17 @@ private ProjectMetadata.Builder updateTimeSeriesTemporalRange(ProjectMetadata pr
                     mBuilder.updateSettings(settings, head.getName());
                     // Verify that all temporal ranges of each backing index is still valid:
                     dataStream.validate(mBuilder::get);
-                } catch (Exception e) {
-                    LOGGER.error(
-                        () -> format(
-                            "unable to update [%s] for data stream [%s] and backing index [%s]",
-                            IndexSettings.TIME_SERIES_END_TIME.getKey(),
-                            dataStream.getName(),
-                            head.getName()
-                        ),
-                        e
-                    );
                 }
+            } catch (Exception e) {
+                LOGGER.error(
+                    () -> format(
+                        "unable to update [%s] for data stream [%s] and backing index [%s]",
+                        IndexSettings.TIME_SERIES_END_TIME.getKey(),
+                        dataStream.getName(),
+                        head.getName()
+                    ),
+                    e
+                );
             }
         }
         return mBuilder;