Merge branch 'main' into port-qa-legacy-projects

# Conflicts:
#	build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java

Author: mark-vieira

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RestrictedBuildApiService.java

@@ -56,7 +56,6 @@ private static ListMultimap<Class<?>, String> createLegacyRestTestBasePluginUsag
         map.put(LegacyRestTestBasePlugin.class, ":qa:remote-clusters");
         map.put(LegacyRestTestBasePlugin.class, ":qa:repository-multi-version");
         map.put(LegacyRestTestBasePlugin.class, ":qa:rolling-upgrade-legacy");
-        map.put(LegacyRestTestBasePlugin.class, ":qa:verify-version-constants");
         map.put(LegacyRestTestBasePlugin.class, ":test:external-modules:test-apm-integration");
         map.put(LegacyRestTestBasePlugin.class, ":test:external-modules:test-delayed-aggs");
         map.put(LegacyRestTestBasePlugin.class, ":test:external-modules:test-die-with-dignity");

docs/build.gradle

@@ -1,4 +1,5 @@
 import org.elasticsearch.gradle.Version
+import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.internal.doc.DocSnippetTask
 import static org.elasticsearch.gradle.testclusters.TestDistribution.DEFAULT
 
@@ -2059,3 +2060,29 @@ setups['setup-snapshots'] = setups['setup-repository'] + '''
     setups['atomic_red_regsvr32'].replace('#atomic_red_data#', events)
   }
 }
+
+tasks.register('verifyDocsLuceneVersion') {
+  doFirst {
+    File docsVersionsFile = file('Versions.asciidoc')
+    List<String> versionLines = docsVersionsFile.readLines('UTF-8')
+    String docsLuceneVersion = null
+    for (String line : versionLines) {
+      if (line.startsWith(':lucene_version:')) {
+        docsLuceneVersion = line.split()[1]
+      }
+    }
+    if (docsLuceneVersion == null) {
+      throw new GradleException('Could not find lucene version in docs version file')
+    }
+    String expectedLuceneVersion = VersionProperties.lucene
+    // remove potential -snapshot-{gitrev} suffix
+    expectedLuceneVersion -= ~/-snapshot-[0-9a-f]+$/
+    if (docsLuceneVersion != expectedLuceneVersion) {
+      throw new GradleException("Lucene version in docs [${docsLuceneVersion}] does not match version.properties [${expectedLuceneVersion}]")
+    }
+  }
+}
+
+tasks.named('check') {
+  dependsOn 'verifyDocsLuceneVersion'
+}

docs/changelog/125239.yaml

@@ -0,0 +1,6 @@
+pr: 125239
+summary: Adding common rerank options to Perform Inference API
+area: Machine Learning
+type: enhancement
+issues:
+ - 111273

docs/changelog/125529.yaml

@@ -0,0 +1,5 @@
+pr: 125529
+summary: Store arrays offsets for boolean fields natively with synthetic source
+area: Mapping
+type: enhancement
+issues: []

docs/changelog/125595.yaml

@@ -0,0 +1,5 @@
+pr: 125595
+summary: Esql - Fix lucene push down behavior when a range contains nanos and millis
+area: ES|QL
+type: bug
+issues: []

docs/changelog/125606.yaml

@@ -0,0 +1,17 @@
+pr: 125606
+summary: Rename deprecation index template
+area: Infra/Logging
+type: breaking
+issues:
+ - 125445
+breaking:
+  title: Rename deprecation index template
+  area: Logging
+  details: The deprecation datastream contains log entries for deprecations that
+    occured while Elasticsearch is running. The deprecation log entries had a
+    mismatch in their dataset name. In order to avoid changing the dataset
+    name in the existing datastream, a new datastream now exists.
+  impact: If querying for deprecations previously using the
+    `.logs-deprecation.elasticsearch-default` datastream, you should now use the
+    `.logs-elasticsearch.deprecation-default` datastream.
+  notable: false

qa/smoke-test-ingest-with-all-dependencies/src/yamlRestTest/resources/rest-api-spec/test/ingest/80_ingest_simulate.yml

@@ -840,9 +840,9 @@ setup:
 
   - do:
       allowed_warnings:
-        - "index template [test-composable-1] has index patterns [foo*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [test-composable-1] will take precedence during new index creation"
+        - "index template [foo_index_template] has index patterns [foo*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [foo_index_template] will take precedence during new index creation"
       indices.put_index_template:
-        name: test-composable-1
+        name: foo_index_template
         body:
           index_patterns:
             - foo*

qa/verify-version-constants/build.gradle

@@ -7,57 +7,14 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
 
-apply plugin: 'elasticsearch.internal-testclusters'
-apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.internal-java-rest-test'
 apply plugin: 'elasticsearch.bwc-test'
 
-dependencies {
-  testImplementation project(':modules:rest-root')
-}
-
 buildParams.bwcVersions.withIndexCompatible { bwcVersion, baseName ->
-  def baseCluster = testClusters.register(baseName) {
-      version = bwcVersion.toString()
-      setting 'xpack.security.enabled', 'true'
-      user username: 'admin', password: 'admin-password', role: 'superuser'
-  }
-
-  tasks.register("${baseName}#integTest", StandaloneRestIntegTestTask) {
-    useCluster baseCluster
-    nonInputProperties.systemProperty('tests.rest.cluster', baseCluster.map(c -> c.allHttpSocketURI.join(",")))
-    nonInputProperties.systemProperty('tests.clustername', "${->baseCluster.get().getName()}")
-  }
-
-  tasks.register(bwcTaskName(bwcVersion)) {
-    dependsOn "${baseName}#integTest"
+  tasks.register(bwcTaskName(bwcVersion), StandaloneRestIntegTestTask) {
+    usesBwcDistribution(bwcVersion)
+    systemProperty 'tests.cluster_version', bwcVersion
   }
 }
-
-tasks.register("verifyDocsLuceneVersion") {
-  doFirst {
-    File docsVersionsFile = layout.settingsDirectory.file('docs/Versions.asciidoc').asFile
-    List<String> versionLines = docsVersionsFile.readLines('UTF-8')
-    String docsLuceneVersion = null
-    for (String line : versionLines) {
-      if (line.startsWith(':lucene_version:')) {
-        docsLuceneVersion = line.split()[1]
-      }
-    }
-    if (docsLuceneVersion == null) {
-      throw new GradleException('Could not find lucene version in docs version file')
-    }
-    String expectedLuceneVersion = VersionProperties.lucene
-    // remove potential -snapshot-{gitrev} suffix
-    expectedLuceneVersion -= ~/-snapshot-[0-9a-f]+$/
-    if (docsLuceneVersion != expectedLuceneVersion) {
-      throw new GradleException("Lucene version in docs [${docsLuceneVersion}] does not match version.properties [${expectedLuceneVersion}]")
-    }
-  }
-}
-
-tasks.named("check").configure {
-  dependsOn "verifyDocsLuceneVersion"
-}

qa/verify-version-constants/src/javaRestTest/java/org/elasticsearch/qa/verify_version_constants/VerifyVersionConstantsIT.java

@@ -0,0 +1,97 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.qa.verify_version_constants;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.rest.ESRestTestCase;
+import org.elasticsearch.test.rest.ObjectPath;
+import org.hamcrest.Matchers;
+import org.junit.ClassRule;
+
+import java.io.IOException;
+import java.text.ParseException;
+import java.util.Map;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+
+public class VerifyVersionConstantsIT extends ESRestTestCase {
+
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .version(System.getProperty("tests.cluster_version"))
+        .setting("xpack.security.enabled", "false")
+        .build();
+
+    public void testLuceneVersionConstant() throws IOException, ParseException {
+        Response response = client().performRequest(new Request("GET", "/"));
+        assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
+        ObjectPath objectPath = ObjectPath.createFromResponse(response);
+
+        String luceneVersionString = objectPath.evaluate("version.lucene_version").toString();
+        org.apache.lucene.util.Version luceneVersion = org.apache.lucene.util.Version.parse(luceneVersionString);
+
+        IndexVersion indexVersion = getIndexVersion();
+        assertThat(indexVersion.luceneVersion(), equalTo(luceneVersion));
+    }
+
+    private IndexVersion getIndexVersion() throws IOException {
+        IndexVersion indexVersion = null;
+
+        Request request = new Request("GET", "_nodes");
+        request.addParameter("filter_path", "nodes.*.index_version,nodes.*.name");
+        Response response = client().performRequest(request);
+        ObjectPath objectPath = ObjectPath.createFromResponse(response);
+        Map<String, Object> nodeMap = objectPath.evaluate("nodes");
+        for (String id : nodeMap.keySet()) {
+            Number ix = objectPath.evaluate("nodes." + id + ".index_version");
+            IndexVersion version;
+            if (ix != null) {
+                version = IndexVersion.fromId(ix.intValue());
+            } else {
+                // it doesn't have index version (pre 8.11) - just infer it from the release version
+                version = parseLegacyVersion(System.getProperty("tests.cluster_version")).map(x -> IndexVersion.fromId(x.id()))
+                    .orElse(IndexVersions.MINIMUM_COMPATIBLE);
+            }
+
+            if (indexVersion == null) {
+                indexVersion = version;
+            } else {
+                String name = objectPath.evaluate("nodes." + id + ".name");
+                assertThat("Node " + name + " has a different index version to other nodes", version, Matchers.equalTo(indexVersion));
+            }
+        }
+
+        assertThat("Index version could not be read", indexVersion, notNullValue());
+        return indexVersion;
+    }
+
+    @Override
+    public boolean preserveClusterUponCompletion() {
+        /*
+         * We don't perform any writes to the cluster so there won't be anything
+         * to clean up. Also, our cleanup code is really only compatible with
+         * *write* compatible versions but this runs with *index* compatible
+         * versions.
+         */
+        return true;
+    }
+
+    @Override
+    protected String getTestRestCluster() {
+        return cluster.getHttpAddresses();
+    }
+}