Merge remote-tracking branch 'upstream/main' into remove_xpack_rolling_restart_checks

Author: jakelandis

.ci/bwcVersions

@@ -84,5 +84,6 @@ BWC_VERSION:
   - "8.6.0"
   - "8.6.1"
   - "8.6.2"
+  - "8.6.3"
   - "8.7.0"
   - "8.8.0"

.ci/snapshotBwcVersions

@@ -1,5 +1,5 @@
 BWC_VERSION:
   - "7.17.10"
-  - "8.6.2"
+  - "8.6.3"
   - "8.7.0"
   - "8.8.0"

TESTING.asciidoc

@@ -70,7 +70,7 @@ When running Elasticsearch with `./gradlew run`, assertions are enabled by
 default. To disable them, add the following command line option:
 
 -------------------------
--Dtests.jvm.argline="-da"
+-Dtests.jvm.argline="-da -dsa"
 -------------------------
 
 ==== Distribution

benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java

@@ -82,7 +82,7 @@ public class ScriptScoreBenchmark {
     private final ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, pluginsService.filterPlugins(ScriptPlugin.class));
 
     private final Map<String, MappedFieldType> fieldTypes = Map.ofEntries(
-        Map.entry("n", new NumberFieldType("n", NumberType.LONG, false, false, true, true, null, Map.of(), null, false, null))
+        Map.entry("n", new NumberFieldType("n", NumberType.LONG, false, false, true, true, null, Map.of(), null, false, null, null))
     );
     private final IndexFieldDataCache fieldDataCache = new IndexFieldDataCache.None();
     private final Map<String, Set<String>> sourcePaths = Map.of("n", Set.of("n"));

build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle

@@ -10,13 +10,10 @@ import org.elasticsearch.gradle.Architecture
 import org.elasticsearch.gradle.OS
 import org.elasticsearch.gradle.internal.info.BuildParams
 import org.gradle.initialization.BuildRequestMetaData
-import org.elasticsearch.gradle.internal.test.rerun.TestRerunPlugin
-import java.util.concurrent.TimeUnit
-
-long startTime = project.gradle.services.get(BuildRequestMetaData).getStartTime()
 
 buildScan {
   URL jenkinsUrl = System.getenv('JENKINS_URL') ? new URL(System.getenv('JENKINS_URL')) : null
+  String buildKiteUrl = System.getenv('BUILDKITE_BUILD_URL') ? System.getenv('BUILDKITE_BUILD_URL') : null
 
   // Automatically publish scans from Elasticsearch CI
   if (jenkinsUrl?.host?.endsWith('elastic.co') || jenkinsUrl?.host?.endsWith('elastic.dev')) {
@@ -25,12 +22,6 @@ buildScan {
   }
 
   background {
-    String buildNumber = System.getenv('BUILD_NUMBER')
-    String buildUrl = System.getenv('BUILD_URL')
-    String jobName = System.getenv('JOB_NAME')
-    String nodeName = System.getenv('NODE_NAME')
-    String jobBranch = System.getenv('ghprbTargetBranch') ?: System.getenv('JOB_BRANCH')
-
     tag OS.current().name()
     tag Architecture.current().name()
 
@@ -39,23 +30,25 @@ buildScan {
       tag 'FIPS'
     }
 
-    // Link to Jenkins worker logs and system metrics
-    if (nodeName) {
-      link 'System logs', "https://ci-stats.elastic.co/app/infra#/logs?" +
-        "&logFilter=(expression:'host.name:${nodeName}',kind:kuery)"
-      buildFinished {
-        link 'System metrics', "https://ci-stats.elastic.co/app/metrics/detail/host/" +
-          "${nodeName}?_g=()&metricTime=(autoReload:!f,refreshInterval:5000," +
-          "time:(from:${startTime - TimeUnit.MILLISECONDS.convert(5, TimeUnit.MINUTES)},interval:%3E%3D1m," +
-          "to:${System.currentTimeMillis() + TimeUnit.MILLISECONDS.convert(5, TimeUnit.MINUTES)}))"
-      }
-    }
-
     // Jenkins-specific build scan metadata
     if (jenkinsUrl) {
       // Disable async upload in CI to ensure scan upload completes before CI agent is terminated
       uploadInBackground = false
 
+      String buildNumber = System.getenv('BUILD_NUMBER')
+      String buildUrl = System.getenv('BUILD_URL')
+      String jobName = System.getenv('JOB_NAME')
+      String nodeName = System.getenv('NODE_NAME')
+      String jobBranch = System.getenv('ghprbTargetBranch') ?: System.getenv('JOB_BRANCH')
+
+      // Link to Jenkins worker logs and system metrics
+      if (nodeName) {
+        link 'System logs', "https://ci-stats.elastic.co/app/infra#/logs?&logFilter=(expression:'host.name:${nodeName}',kind:kuery)"
+        buildFinished {
+          link 'System metrics', "https://ci-stats.elastic.co/app/metrics/detail/host/${nodeName}"
+        }
+      }
+
       // Parse job name in the case of matrix builds
       // Matrix job names come in the form of "base-job-name/matrix_param1=value1,matrix_param2=value2"
       def splitJobName = jobName.split('/')
@@ -98,26 +91,45 @@ buildScan {
         value 'Git Commit ID', BuildParams.gitRevision
         link 'Source', "https://github.com/elastic/elasticsearch/tree/${BuildParams.gitRevision}"
       }
+    } else if (buildKiteUrl) { //Buildkite-specific build scan metadata
+      // Disable async upload in CI to ensure scan upload completes before CI agent is terminated
+      uploadInBackground = false
+
+      def branch = System.getenv('BUILDKITE_BRANCH')
+      def repoMatcher = System.getenv('BUILDKITE_REPO') =~ /(https:\/\/github\.com\/|git@github\.com:)(\S+)\.git/
+      def repository = repoMatcher.matches() ? repoMatcher.group(2) : "<unknown>"
+      tag 'CI'
+      link 'CI Build', buildKiteUrl
+      value 'Job Number', System.getenv('BUILDKITE_BUILD_NUMBER')
+
+
+      // Add SCM information
+      def prId = System.getenv('BUILDKITE_PULL_REQUEST')
+      if (prId) {
+        def prBaseUrl = (System.getenv('BUILDKITE_PULL_REQUEST_REPO') - ".git")
+        value 'Git Commit ID', System.getenv('BUILDKITE_COMMIT')
+        tag "pr/${prId}"
+        tag 'pull-request'
+        link 'Source', "${prBaseUrl}/tree/${System.getenv('BUILDKITE_COMMIT')}"
+        link 'Pull Request', "https://github.com/${repository}/pull/${prId}"
+      } else {
+        value 'Git Commit ID', BuildParams.gitRevision
+        link 'Source', "https://github.com/${repository}/tree/${BuildParams.gitRevision}"
+        tag branch
+      }
+
+      buildScanPublished { scan ->
+        // Attach build scan link as build metadata
+        // See: https://buildkite.com/docs/pipelines/build-meta-data
+        ['buildkite-agent', 'meta-data', 'set', "build-scan-${System.getenv('BUILDKITE_JOB_ID')}", "${scan.buildScanUri}"].execute()
+
+        // Add a build annotation
+        // See: https://buildkite.com/docs/agent/v3/cli-annotate
+        def body = """<div class="mb3"><span class="p1 border rounded">${System.getenv('BUILDKITE_LABEL')}</span> :gradle: build ran: <a href="${scan.buildScanUri}"><code>gradle ${gradle.startParameter.taskNames.join(' ')}</code></a></div>"""
+        ['buildkite-agent', 'annotate', '--context', 'gradle-build-scans', '--append', '--style', 'info', body].execute()
+      }
     } else {
       tag 'LOCAL'
     }
   }
 }
-
-subprojects {
-  project.getPlugins().withType(TestRerunPlugin) {
-    tasks.withType(Test).configureEach(new Action<Test>() {
-      @Override
-      void execute(Test test) {
-        test.doLast(new Action<Test>() {
-          @Override
-          void execute(Test t) {
-            if(t.rerun.didRerun.get() == true) {
-              buildScan.tag 'unexpected-test-jvm-exit'
-            }
-          }
-        })
-      }
-    })
-  }
-}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java

@@ -112,7 +112,7 @@ public void apply(Project project) {
             params.setGitOrigin(gitInfo.getOrigin());
             params.setBuildDate(ZonedDateTime.now(ZoneOffset.UTC));
             params.setTestSeed(getTestSeed());
-            params.setIsCi(System.getenv("JENKINS_URL") != null);
+            params.setIsCi(System.getenv("JENKINS_URL") != null || System.getenv("BUILDKITE_BUILD_URL") != null);
             params.setDefaultParallel(ParallelDetector.findDefaultParallel(project));
             params.setInFipsJvm(Util.getBooleanProperty("tests.fips.enabled", false));
             params.setIsSnapshotBuild(Util.getBooleanProperty("build.snapshot", true));

build-tools/src/main/resources/plugin-descriptor.properties

@@ -1,5 +1,5 @@
 # Elasticsearch plugin descriptor file
-# This file must exist as 'plugin-descriptor.properties' inside a plugin.
+# This file must exist as 'plugin-descriptor.properties' or 'stable-plugin-descriptor.properties inside a plugin.
 #
 ### example plugin for "foo"
 #
@@ -31,7 +31,9 @@ name=${name}
 # separated by "."'s and may have leading zeros
 java.version=${javaVersion}
 #
-# 'elasticsearch.version': version of elasticsearch compiled against
+# 'elasticsearch.version': version of elasticsearch compiled against.
+# Plugins implementing plugin-api.jar this version only has to match a major version of the ES server
+# For all other plugins it has to be the same as ES server version
 elasticsearch.version=${elasticsearchVersion}
 ### optional elements for plugins:
 <% if (classname) { %>

docs/changelog/92810.yaml

@@ -0,0 +1,6 @@
+pr: 92810
+summary: Do not fail node if SAML HTTP metadata is unavailable
+area: Authentication
+type: enhancement
+issues: 
+ - 37608

docs/changelog/93177.yaml

@@ -0,0 +1,7 @@
+pr: 93239
+summary: Support ignore malformed in boolean fields
+area: Search
+type: bug
+issues:
+ - 89542
+