Merge branch 'main' into fix-dlm-ilm-mixing-tests

Author: nielsbauman

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java

@@ -61,7 +61,6 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-ingest-with-all-dependencies");
         map.put(LegacyRestTestBasePlugin.class, ":qa:smoke-test-plugins");
         map.put(LegacyRestTestBasePlugin.class, ":qa:system-indices");
-        map.put(LegacyRestTestBasePlugin.class, ":qa:verify-version-constants");
         map.put(LegacyRestTestBasePlugin.class, ":test:external-modules:test-apm-integration");
         map.put(LegacyRestTestBasePlugin.class, ":test:external-modules:test-delayed-aggs");
         map.put(LegacyRestTestBasePlugin.class, ":test:external-modules:test-die-with-dignity");

build.gradle

@@ -295,7 +295,16 @@ allprojects {
   tasks.register('resolveAllDependencies', ResolveAllDependencies) {
     def ignoredPrefixes = [DistributionDownloadPlugin.ES_DISTRO_CONFIG_PREFIX, "jdbcDriver"]
     configs = project.configurations.matching { config -> ignoredPrefixes.any { config.name.startsWith(it) } == false }
-    resolveJavaToolChain = true
+    if(project.path == ':') {
+      resolveJavaToolChain = true
+
+      // ensure we have best possible caching of bwc builds
+      dependsOn ":distribution:bwc:bugfix:buildBwcLinuxTar"
+      dependsOn ":distribution:bwc:bugfix2:buildBwcLinuxTar"
+      dependsOn ":distribution:bwc:minor:buildBwcLinuxTar"
+      dependsOn ":distribution:bwc:staged:buildBwcLinuxTar"
+      dependsOn ":distribution:bwc:staged2:buildBwcLinuxTar"
+    }
     if (project.path.contains("fixture")) {
       dependsOn tasks.withType(ComposePull)
     }
@@ -306,6 +315,7 @@ allprojects {
       // ensure we resolve p2 dependencies for the spotless eclipse formatter
       dependsOn "spotlessJavaCheck"
     }
+
   }
 
   ext.withReleaseBuild = { Closure config ->

docs/build.gradle

@@ -1,4 +1,5 @@
 import org.elasticsearch.gradle.Version
+import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.internal.doc.DocSnippetTask
 import static org.elasticsearch.gradle.testclusters.TestDistribution.DEFAULT
 
@@ -2059,3 +2060,29 @@ setups['setup-snapshots'] = setups['setup-repository'] + '''
     setups['atomic_red_regsvr32'].replace('#atomic_red_data#', events)
   }
 }
+
+tasks.register('verifyDocsLuceneVersion') {
+  doFirst {
+    File docsVersionsFile = file('Versions.asciidoc')
+    List<String> versionLines = docsVersionsFile.readLines('UTF-8')
+    String docsLuceneVersion = null
+    for (String line : versionLines) {
+      if (line.startsWith(':lucene_version:')) {
+        docsLuceneVersion = line.split()[1]
+      }
+    }
+    if (docsLuceneVersion == null) {
+      throw new GradleException('Could not find lucene version in docs version file')
+    }
+    String expectedLuceneVersion = VersionProperties.lucene
+    // remove potential -snapshot-{gitrev} suffix
+    expectedLuceneVersion -= ~/-snapshot-[0-9a-f]+$/
+    if (docsLuceneVersion != expectedLuceneVersion) {
+      throw new GradleException("Lucene version in docs [${docsLuceneVersion}] does not match version.properties [${expectedLuceneVersion}]")
+    }
+  }
+}
+
+tasks.named('check') {
+  dependsOn 'verifyDocsLuceneVersion'
+}

docs/changelog/124062.yaml

@@ -0,0 +1,5 @@
+pr: 124062
+summary: Upgrade to repository-gcs to use com.google.cloud:google-cloud-storage-bom:2.50.0
+area: Snapshot/Restore
+type: upgrade
+issues: []

docs/changelog/124913.yaml

@@ -0,0 +1,5 @@
+pr: 124913
+summary: Report `original_types`
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/125595.yaml

@@ -0,0 +1,5 @@
+pr: 125595
+summary: Esql - Fix lucene push down behavior when a range contains nanos and millis
+area: ES|QL
+type: bug
+issues: []

docs/changelog/125606.yaml

@@ -0,0 +1,17 @@
+pr: 125606
+summary: Rename deprecation index template
+area: Infra/Logging
+type: breaking
+issues:
+ - 125445
+breaking:
+  title: Rename deprecation index template
+  area: Logging
+  details: The deprecation datastream contains log entries for deprecations that
+    occured while Elasticsearch is running. The deprecation log entries had a
+    mismatch in their dataset name. In order to avoid changing the dataset
+    name in the existing datastream, a new datastream now exists.
+  impact: If querying for deprecations previously using the
+    `.logs-deprecation.elasticsearch-default` datastream, you should now use the
+    `.logs-elasticsearch.deprecation-default` datastream.
+  notable: false

docs/internal/DistributedArchitectureGuide.md

@@ -110,6 +110,16 @@ to communicate with Elasticsearch.
 
 ### Cluster State
 
+[ClusterState]:https://github.com/elastic/elasticsearch/blob/main/server/src/main/java/org/elasticsearch/cluster/ClusterState.java
+[Metadata]:https://github.com/elastic/elasticsearch/blob/main/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java
+[ProjectMetadata]:https://github.com/elastic/elasticsearch/blob/main/server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java
+
+The [Metadata] of a [ClusterState] is persisted on disk and comprises information from two categories:
+1. Cluster scope information such as `clusterUUID`, `CoordinationMetadata`
+2. Project scope information ([ProjectMetadata]) such as indices and templates belong to each project.
+
+Some concepts are applicable to both cluster and project scopes, e.g. [persistent tasks](#persistent-tasks). The state of a persistent task is therefore stored accordingly depending on the task's scope.
+
 #### Master Service
 
 #### Cluster State Publication
@@ -306,7 +316,7 @@ policies.
 
 ### How cluster capacity is determined
 
-[AutoscalingMetadata][] implements [Metadata.Custom][] in order to persist autoscaling policies. Each
+[AutoscalingMetadata][] implements [Metadata.ClusterCustom][] in order to persist autoscaling policies. Each
 Decider is an implementation of [AutoscalingDeciderService][]. The [AutoscalingCalculateCapacityService][]
 is responsible for running the calculation.
 
@@ -322,7 +332,7 @@ calls [through the CapacityResponseCache][], into the `AutoscalingCalculateCapac
 concurrent callers.
 
 [AutoscalingMetadata]: https://github.com/elastic/elasticsearch/blob/v8.13.2/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/AutoscalingMetadata.java#L38
-[Metadata.Custom]: https://github.com/elastic/elasticsearch/blob/v8.13.2/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java#L141-L145
+[Metadata.ClusterCustom]: https://github.com/elastic/elasticsearch/blob/f461731a30a6fe55d7d7b343d38426ddca1ac873/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java#L147
 [AutoscalingDeciderService]: https://github.com/elastic/elasticsearch/blob/v8.13.2/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingDeciderService.java#L16-L19
 [AutoscalingCalculateCapacityService]: https://github.com/elastic/elasticsearch/blob/v8.13.2/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/capacity/AutoscalingCalculateCapacityService.java#L43
 
@@ -505,6 +515,8 @@ The [Task management API] also exposes an endpoint where a task ID can be specif
 
 [PersistentTaskPlugin]:https://github.com/elastic/elasticsearch/blob/main/server/src/main/java/org/elasticsearch/plugins/PersistentTaskPlugin.java
 [PersistentTasksExecutor]:https://github.com/elastic/elasticsearch/blob/main/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java
+[PersistentTasksExecutor.Scope.Cluster]:https://github.com/elastic/elasticsearch/blob/f461731a30a6fe55d7d7b343d38426ddca1ac873/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java#L52
+[PersistentTasksExecutor.Scope.Project]:https://github.com/elastic/elasticsearch/blob/f461731a30a6fe55d7d7b343d38426ddca1ac873/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutor.java#L48
 [PersistentTasksExecutorRegistry]:https://github.com/elastic/elasticsearch/blob/main/server/src/main/java/org/elasticsearch/persistent/PersistentTasksExecutorRegistry.java
 [PersistentTasksNodeService]:https://github.com/elastic/elasticsearch/blob/main/server/src/main/java/org/elasticsearch/persistent/PersistentTasksNodeService.java
 [PersistentTasksClusterService]:https://github.com/elastic/elasticsearch/blob/main/server/src/main/java/org/elasticsearch/persistent/PersistentTasksClusterService.java
@@ -513,13 +525,14 @@ The [Task management API] also exposes an endpoint where a task ID can be specif
 [HealthNodeTaskExecutor]:https://github.com/elastic/elasticsearch/blob/main/server/src/main/java/org/elasticsearch/health/node/selection/HealthNodeTaskExecutor.java
 [SystemIndexMigrationExecutor]:https://github.com/elastic/elasticsearch/blob/main/server/src/main/java/org/elasticsearch/upgrades/SystemIndexMigrationExecutor.java
 [PersistentTasksCustomMetadata]:https://github.com/elastic/elasticsearch/blob/main/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetadata.java
+[ClusterPersistentTasksCustomMetadata]:https://github.com/elastic/elasticsearch/blob/main/server/src/main/java/org/elasticsearch/persistent/ClusterPersistentTasksCustomMetadata.java
 [PersistentTasksCustomMetadata.PersistentTask]:https://github.com/elastic/elasticsearch/blob/d466ad1c3c4cedc7d5f6ab5794abe7bfd72aef4e/server/src/main/java/org/elasticsearch/persistent/PersistentTasksCustomMetadata.java#L305
 
 Up until now we have discussed only ephemeral tasks. If we want a task to survive node failures, it needs to be registered as a persistent task at the cluster level.
 
-Plugins can register persistent tasks definitions by implementing [PersistentTaskPlugin] and returning one or more [PersistentTasksExecutor] instances. These are collated into a [PersistentTasksExecutorRegistry] which is provided to [PersistentTasksNodeService] active on each node in the cluster, and a [PersistentTasksClusterService] active on the master.
+Plugins can register persistent tasks definitions by implementing [PersistentTaskPlugin] and returning one or more [PersistentTasksExecutor] instances. These are collated into a [PersistentTasksExecutorRegistry] which is provided to [PersistentTasksNodeService] active on each node in the cluster, and a [PersistentTasksClusterService] active on the master. A [PersistentTasksExecutor] can declare either [project][PersistentTasksExecutor.Scope.Project] or [cluster][PersistentTasksExecutor.Scope.Cluster] scope, but not both. A project scope task is not able to access data on a different project.
 
-The [PersistentTasksClusterService] runs on the master to manage the set of running persistent tasks. It periodically checks that all persistent tasks are assigned to live nodes and handles the creation, completion, removal and updates-to-the-state of persistent task instances in the cluster state (see [PersistentTasksCustomMetadata]).
+The [PersistentTasksClusterService] runs on the master to manage the set of running persistent tasks. It periodically checks that all persistent tasks are assigned to live nodes and handles the creation, completion, removal and updates-to-the-state of persistent task instances in the cluster state (see [PersistentTasksCustomMetadata] and [ClusterPersistentTasksCustomMetadata]).
 
 The [PersistentTasksNodeService] monitors the cluster state to:
  - Start any tasks allocated to it (tracked in the local [TaskManager] by an [AllocatedPersistentTask])
@@ -529,7 +542,7 @@ If a node leaves the cluster while it has a persistent task allocated to it, the
 
 Some examples of the use of persistent tasks include:
  - [ShardFollowTasksExecutor]: Defined by [cross-cluster replication](#cross-cluster-replication-ccr) to poll a remote cluster for updates
- - [HealthNodeTaskExecutor]: Used to schedule work related to monitoring cluster health
+ - [HealthNodeTaskExecutor]: Used to schedule work related to monitoring cluster health. This is currently the only example of a cluster scope persistent task.
  - [SystemIndexMigrationExecutor]: Manages the migration of system indices after an upgrade
 
 ### Integration with APM

docs/reference/query-languages/query-dsl/query-dsl-knn-query.md

@@ -206,7 +206,7 @@ POST my-image-index/_search
 
 A sample query can look like below:
 
-```js
+```json
 {
   "query" : {
     "nested" : {
@@ -226,6 +226,7 @@ A sample query can look like below:
 }
 ```
 
+Note that nested `knn` only supports `score_mode=max`.
 
 ## Knn query with aggregations [knn-query-aggregations]
 

docs/reference/query-languages/query-dsl/query-dsl-nested-query.md

@@ -71,7 +71,7 @@ See [Multi-level nested queries](#multi-level-nested-query-ex) for an example.
 
 
 `score_mode`
-:   (Optional, string) Indicates how scores for matching child objects affect the root parent document’s [relevance score](/reference/query-languages/query-dsl/query-filter-context.md#relevance-scores). Valid values are:
+:   (Optional, string) Indicates how scores for matching child objects affect the root parent document’s [relevance score](/reference/query-languages/query-dsl/query-filter-context.md#relevance-scores). Default is `avg`, but nested `knn` queries only support `score_mode=max`. Valid values are:
 
 `avg` (Default)
 :   Use the mean relevance score of all matching child objects.