Merge branch 'main' into nit-empty-incremental-bulk

Author: mhl-b

.buildkite/pipelines/periodic-fwc.template.yml

@@ -1,6 +1,6 @@
 steps:
-  - label: $FWC_VERSION / fwc
-    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$FWC_VERSION#fwcTest -Dtests.bwc.snapshot=false
+  - label: "{{matrix.FWC_VERSION}} / fwc"
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$$FWC_VERSION#fwcTest -Dtests.bwc.snapshot=false
     timeout_in_minutes: 300
     agents:
       provider: gcp
@@ -11,4 +11,4 @@ steps:
       setup:
         FWC_VERSION: $FWC_LIST
     env:
-      FWC_VERSION: $FWC_VERSION
+      FWC_VERSION: "{{matrix.FWC_VERSION}}"

.buildkite/pipelines/periodic-fwc.yml

@@ -1,7 +1,7 @@
 # This file is auto-generated. See .buildkite/pipelines/periodic-fwc.template.yml
 steps:
-  - label: $FWC_VERSION / fwc
-    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$FWC_VERSION#fwcTest -Dtests.bwc.snapshot=false
+  - label: "{{matrix.FWC_VERSION}} / fwc"
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v$$FWC_VERSION#fwcTest -Dtests.bwc.snapshot=false
     timeout_in_minutes: 300
     agents:
       provider: gcp
@@ -12,4 +12,4 @@ steps:
       setup:
         FWC_VERSION: []
     env:
-      FWC_VERSION: $FWC_VERSION
+      FWC_VERSION: "{{matrix.FWC_VERSION}}"

.buildkite/scripts/fwc-branches.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Configure FwC test branches
+# We do not want 7.x branch and only to run for branches that:
+# - have released at least one minor version (not main)
+# - have previous minor unreleased (not the oldest development branch)
+FWC_BRANCHES=()
+for branch in "${BRANCHES[@]}"; do
+  if [[ ! "$branch" =~ ^7\..* ]]; then
+    FWC_BRANCHES+=("$branch")
+  fi
+done
+# Remove first and last element
+FWC_BRANCHES=("${FWC_BRANCHES[@]:1:${#FWC_BRANCHES[@]}-2}")
+
+shouldRunFwcFor() {
+  local branch=$1
+  for fwc_branch in "${FWC_BRANCHES[@]}"; do
+    if [[ "$fwc_branch" == "$branch" ]]; then
+      return 0
+    fi
+  done
+  return 1
+}

.buildkite/scripts/generate-pr-performance-benchmark.sh

@@ -2,39 +2,69 @@
 
 set -euo pipefail
 
+# uncomment for tests
+#function buildkite-agent {
+#  local command=$1
+#  echo "$@"
+#  if [ "$command" == "annotate" ]; then
+#    while read -r line; do
+#      echo "  read: $line";
+#    done
+#  fi
+#}
+
 env_id_baseline=$(python3 -c 'import uuid; print(uuid.uuid4())')
 env_id_contender=$(python3 -c 'import uuid; print(uuid.uuid4())')
 merge_base=$(git merge-base "${GITHUB_PR_TARGET_BRANCH}" HEAD)
 
+# PR comment
+buildkite-agent meta-data set pr_comment:early_comment_job_id "$BUILDKITE_JOB_ID"
 buildkite-agent meta-data set pr_comment:custom-body:body \
-  "This build attempted two ${GITHUB_PR_COMMENT_VAR_BENCHMARK} benchmarks to evaluate performance impact of this PR."
+  "This build attempts two ${GITHUB_PR_COMMENT_VAR_BENCHMARK} benchmarks to evaluate performance impact of this PR. \
+To estimate benchmark completion time inspect previous nightly runs [here](https://buildkite.com/elastic/elasticsearch-performance-esbench-nightly/builds?branch=master)."
 buildkite-agent meta-data set pr_comment:custom-baseline:head \
   "* Baseline: ${merge_base} (env ID ${env_id_baseline})"
 buildkite-agent meta-data set pr_comment:custom-contender:head \
   "* Contender: ${GITHUB_PR_TRIGGERED_SHA} (env ID ${env_id_contender})"
 
-cat << _EOF_
+# Buildkite annotation
+cat << _EOF1_ | buildkite-agent annotate --context "pr-benchmark-notification"
+  This build attempts two ${GITHUB_PR_COMMENT_VAR_BENCHMARK} benchmarks to evaluate performance impact of PR [${GITHUB_PR_NUMBER}](https://github.com/elastic/elasticsearch/pull/${GITHUB_PR_NUMBER}).
+  To estimate benchmark completion time inspect previous nightly runs [here](https://buildkite.com/elastic/elasticsearch-performance-esbench-nightly/builds?branch=master).
+  * Baseline: [${merge_base:0:7}](https://github.com/elastic/elasticsearch/commit/${merge_base}) (env ID ${env_id_baseline})
+  * Contender: [${GITHUB_PR_TRIGGERED_SHA:0:7}](https://github.com/elastic/elasticsearch/commit/${GITHUB_PR_TRIGGERED_SHA}) (env ID ${env_id_contender})
+_EOF1_
+
+cat << _EOF2_
 steps:
-  - label: Trigger baseline benchmark
+  - label: Trigger baseline benchmark with ${merge_base:0:7}
     trigger: elasticsearch-performance-esbench-pr
     build:
-      message: Baseline benchmark for PR${GITHUB_PR_NUMBER}
+      message: Baseline benchmark for PR ${GITHUB_PR_NUMBER} with ${merge_base:0:7}
       branch: master
       env:
         CONFIGURATION_NAME: ${GITHUB_PR_COMMENT_VAR_BENCHMARK}
         ENV_ID: ${env_id_baseline}
         REVISION: ${merge_base}
-  - label: Trigger contender benchmark
+  - label: Trigger contender benchmark with ${GITHUB_PR_TRIGGERED_SHA:0:7}
     trigger: elasticsearch-performance-esbench-pr
     build:
-      message: Contender benchmark for PR${GITHUB_PR_NUMBER}
+      message: Contender benchmark for PR ${GITHUB_PR_NUMBER} with ${GITHUB_PR_TRIGGERED_SHA:0:7}
       branch: master
       env:
         CONFIGURATION_NAME: ${GITHUB_PR_COMMENT_VAR_BENCHMARK}
         ENV_ID: ${env_id_contender}
         ES_REPO_URL: https://github.com/${GITHUB_PR_OWNER}/${GITHUB_PR_REPO}.git
         REVISION: ${GITHUB_PR_TRIGGERED_SHA}
   - wait: ~
-  - label: Modify PR comment
-    command: buildkite-agent meta-data set pr_comment:custom-comparison:head "* [Benchmark results](<https://esbench-metrics.kb.us-east-2.aws.elastic-cloud.com:9243/app/dashboards#/view/d9079962-5866-49ef-b9f5-145f2141cd31?_a=(query:(language:kuery,query:'user-tags.env-id:${env_id_baseline} or user-tags.env-id:${env_id_contender}'))>)"
-_EOF_
+  - label: Update PR comment and Buildkite annotation
+    command: |
+      buildkite-agent meta-data set pr_comment:custom-body:body "This build ran two ${GITHUB_PR_COMMENT_VAR_BENCHMARK} benchmarks to evaluate performance impact of this PR."
+      buildkite-agent meta-data set pr_comment:custom-comparison:head "* [Benchmark results](<https://esbench-metrics.kb.us-east-2.aws.elastic-cloud.com:9243/app/dashboards#/view/d9079962-5866-49ef-b9f5-145f2141cd31?_a=(query:(language:kuery,query:'user-tags.env-id:${env_id_baseline} or user-tags.env-id:${env_id_contender}'))>)"
+      cat << _EOF3_ | buildkite-agent annotate --context "pr-benchmark-notification"
+        This build ran two ${GITHUB_PR_COMMENT_VAR_BENCHMARK} benchmarks to evaluate performance impact of PR [${GITHUB_PR_NUMBER}](https://github.com/elastic/elasticsearch/pull/${GITHUB_PR_NUMBER}).
+        * Baseline: [${merge_base:0:7}](https://github.com/elastic/elasticsearch/commit/${merge_base}) (env ID ${env_id_baseline})
+        * Contender: [${GITHUB_PR_TRIGGERED_SHA:0:7}](https://github.com/elastic/elasticsearch/commit/${GITHUB_PR_TRIGGERED_SHA}) (env ID ${env_id_contender})
+        * [Benchmark results](<https://esbench-metrics.kb.us-east-2.aws.elastic-cloud.com:9243/app/dashboards#/view/d9079962-5866-49ef-b9f5-145f2141cd31?_a=(query:(language:kuery,query:'user-tags.env-id:${env_id_baseline} or user-tags.env-id:${env_id_contender}'))>)
+      _EOF3_
+_EOF2_

.buildkite/scripts/periodic.trigger.sh

@@ -5,6 +5,7 @@ set -euo pipefail
 echo "steps:"
 
 source .buildkite/scripts/branches.sh
+source .buildkite/scripts/fwc-branches.sh
 
 IS_FIRST=true
 SKIP_DELAY="${SKIP_DELAY:-false}"
@@ -46,8 +47,7 @@ EOF
       branch: "$BRANCH"
       commit: "$LAST_GOOD_COMMIT"
 EOF
-# Include forward compatibility tests only for the bugfix branch
-if [[ "${BRANCH}" == "${BRANCHES[2]}" ]]; then
+if shouldRunFwcFor "$BRANCH"; then
   cat <<EOF
   - trigger: elasticsearch-periodic-fwc
     label: Trigger periodic-fwc pipeline for $BRANCH

TESTING.asciidoc

@@ -681,12 +681,15 @@ There are multiple base classes for tests:
   directly by unit tests.
 * **`ESSingleNodeTestCase`**: This test case sets up a cluster that has a
   single node.
-* **`ESIntegTestCase`**: An integration test case that creates a cluster that
-  might have multiple nodes.
-* **`ESRestTestCase`**: An integration tests that interacts with an external
-  cluster via the REST API. This is used for Java based REST tests.
-* **`ESClientYamlSuiteTestCase` **: A subclass of `ESRestTestCase` used to run
-  YAML based REST tests.
+* **`ESIntegTestCase`**: An internal integration test that starts nodes within the same JVM as the test.
+    These allow you to test functionality that is not exposed via the REST API, or for verifying a certain internal state.
+    Additionally, you can easily simulate tricky distributed setups that are difficult to do in REST tests.
+    If you only need to start one node, use `ESSingleNodeTestCase` instead, which is a much lighter test setup.
+* **`ESRestTestCase`**: An integration test that interacts with an external
+  cluster via the REST API. This is used for Java based REST tests. This should
+  be the first choice for writing integration tests as these tests run in a much more
+  realistic setup.
+* **`ESClientYamlSuiteTestCase` **: A subclass of `ESRestTestCase` used to run YAML based REST tests.
 
 === Good practices
 

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/ElasticsearchTestBasePluginFuncTest.groovy

@@ -108,4 +108,65 @@ class ElasticsearchTestBasePluginFuncTest extends AbstractGradleFuncTest {
         then:
         result.task(':test').outcome == TaskOutcome.UP_TO_DATE
     }
+
+    def "uses new test seed for every invocation"() {
+        given:
+        file("src/test/java/acme/SomeTests.java").text = """
+
+        public class SomeTests {
+            @org.junit.Test
+            public void printTestSeed() {
+                System.out.println("TESTSEED=[" + System.getProperty("tests.seed") + "]");
+            }
+        }
+
+        """
+        buildFile.text = """
+            plugins {
+             id 'java'
+             id 'elasticsearch.test-base'
+            }
+
+            tasks.named('test').configure {
+                testLogging {
+                    showStandardStreams = true
+                }
+            }
+
+            tasks.register('test2', Test) {
+                classpath = sourceSets.test.runtimeClasspath
+                testClassesDirs = sourceSets.test.output.classesDirs
+                testLogging {
+                    showStandardStreams = true
+                }
+            }
+
+            repositories {
+                mavenCentral()
+            }
+
+            dependencies {
+                testImplementation 'junit:junit:4.12'
+            }
+
+        """
+
+        when:
+        def result1 = gradleRunner("cleanTest", "cleanTest2", "test", "test2").build()
+        def result2 = gradleRunner("cleanTest", "cleanTest2", "test", "test2").build()
+
+        then:
+        def seeds1 = result1.output.findAll(/(?m)TESTSEED=\[([^\]]+)\]/) { it[1] }
+        def seeds2 = result2.output.findAll(/(?m)TESTSEED=\[([^\]]+)\]/) { it[1] }
+
+        seeds1.unique().size() == 1
+        seeds2.unique().size() == 1
+
+        verifyAll {
+            seeds1[0] != null
+            seeds2[0] != null
+            seeds1[0] != seeds2[0]
+        }
+        result2.output.contains("Configuration cache entry reused.")
+    }
 }

build-tools-internal/src/main/groovy/elasticsearch.fwc-test.gradle

@@ -7,19 +7,23 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-import org.elasticsearch.gradle.VersionProperties
+import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask
 
-def fwcVersions = buildParams.bwcVersions.released.findAll { it.major == VersionProperties.elasticsearchVersion.major &&  it.minor == VersionProperties.elasticsearchVersion.minor }
-def previousMinorSnapshot = buildParams.bwcVersions.unreleased.find { it.major == VersionProperties.elasticsearchVersion.major && it.minor == VersionProperties.elasticsearchVersion.minor - 1 }
-
-fwcVersions.each { fwcVersion ->
-  tasks.register("v${fwcVersion}#fwcTest", StandaloneRestIntegTestTask) {
-    usesBwcDistribution(previousMinorSnapshot)
-    usesBwcDistribution(fwcVersion)
-    systemProperty("tests.old_cluster_version", previousMinorSnapshot)
-    systemProperty("tests.new_cluster_version", fwcVersion)
-    nonInputProperties.systemProperty 'tests.fwc', 'true'
+Version elasticsearchVersion = Version.fromString(versions.get("elasticsearch"))
+def fwcVersions = buildParams.bwcVersions.released.findAll { it.major == elasticsearchVersion.major &&  it.minor == elasticsearchVersion.minor }
+def targetMajor = elasticsearchVersion.minor > 0 ? elasticsearchVersion.major : elasticsearchVersion.major - 1
+def targetMinor = elasticsearchVersion.minor > 0 ? elasticsearchVersion.minor - 1 : buildParams.bwcVersions.unreleased.findAll { it.major == targetMajor }*.minor.max()
+def previousMinorSnapshot = buildParams.bwcVersions.unreleased.find { it.major == targetMajor && it.minor == targetMinor }
+if (previousMinorSnapshot != null) {
+  fwcVersions.each { fwcVersion ->
+    tasks.register("v${fwcVersion}#fwcTest", StandaloneRestIntegTestTask) {
+      usesBwcDistribution(previousMinorSnapshot)
+      usesBwcDistribution(fwcVersion)
+      systemProperty("tests.old_cluster_version", previousMinorSnapshot)
+      systemProperty("tests.new_cluster_version", fwcVersion)
+      nonInputProperties.systemProperty 'tests.fwc', 'true'
+    }
   }
 }
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java

@@ -24,6 +24,7 @@
 import org.gradle.api.Project;
 import org.gradle.api.Task;
 import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.configuration.BuildFeatures;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.plugins.JavaPlugin;
 import org.gradle.api.provider.ProviderFactory;
@@ -56,6 +57,9 @@ public abstract class ElasticsearchTestBasePlugin implements Plugin<Project> {
     @Inject
     protected abstract ProviderFactory getProviderFactory();
 
+    @Inject
+    protected abstract BuildFeatures getBuildFeatures();
+
     @Override
     public void apply(Project project) {
         project.getRootProject().getPlugins().apply(GlobalBuildInfoPlugin.class);
@@ -164,9 +168,11 @@ public void execute(Task t) {
             );
             test.systemProperties(sysprops);
 
-            // ignore changing test seed when build is passed -Dignore.tests.seed for cacheability experimentation
-            if (System.getProperty("ignore.tests.seed") != null) {
-                nonInputProperties.systemProperty("tests.seed", buildParams.get().getTestSeed());
+            // ignore changing test seed when build is passed -Dignore.tests.seed for cacheability
+            // also ignore when configuration cache is on since the test seed as task input would break
+            // configuration cache reuse.
+            if (System.getProperty("ignore.tests.seed") != null || getBuildFeatures().getConfigurationCache().getActive().get()) {
+                nonInputProperties.systemProperty("tests.seed", buildParams.get().getTestSeedProvider());
             } else {
                 test.systemProperty("tests.seed", buildParams.get().getTestSeed());
             }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParameterExtension.java

@@ -56,6 +56,8 @@ public interface BuildParameterExtension {
 
     String getTestSeed();
 
+    Provider<String> getTestSeedProvider();
+
     Boolean getCi();
 
     Integer getDefaultParallel();
@@ -66,7 +68,7 @@ public interface BuildParameterExtension {
 
     Provider<BwcVersions> getBwcVersionsProvider();
 
-    Random getRandom();
+    Provider<Random> getRandom();
 
     Boolean getGraalVmRuntime();
 }