Add java-matrix and java-fips-matrix Buildkite pipelines (#97253) (#98040)

Author: brianseeders

.buildkite/hooks/pre-command

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# On some distros, this directory ends up not readable by the `elasticsearch` user that gets created during tests
+# This fixes that
+chmod 755 ~
+
+WORKSPACE="$(pwd)"
+export WORKSPACE
+
+BUILD_NUMBER="$BUILDKITE_BUILD_NUMBER"
+export BUILD_NUMBER
+
+COMPOSE_HTTP_TIMEOUT="120"
+export COMPOSE_HTTP_TIMEOUT
+
+JOB_BRANCH="$BUILDKITE_BRANCH"
+export JOB_BRANCH
+
+GRADLEW="./gradlew --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/"
+export GRADLEW
+
+GRADLEW_BAT="./gradlew.bat --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/"
+export GRADLEW_BAT
+
+export $(cat .ci/java-versions.properties | grep '=' | xargs)
+
+JAVA_HOME="$HOME/.java/$ES_BUILD_JAVA"
+export JAVA_HOME
+
+JAVA11_HOME="$HOME/.java/java11"
+export JAVA11_HOME
+
+JAVA16_HOME="$HOME/.java/openjdk16"
+export JAVA16_HOME
+
+if [[ "${ES_RUNTIME_JAVA:-}" ]]; then
+  RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+  export RUNTIME_JAVA_HOME
+fi
+
+GRADLE_BUILD_CACHE_USERNAME=$(vault read -field=username secret/ci/elastic-elasticsearch/migrated/gradle-build-cache)
+export GRADLE_BUILD_CACHE_USERNAME
+
+GRADLE_BUILD_CACHE_PASSWORD=$(vault read -field=password secret/ci/elastic-elasticsearch/migrated/gradle-build-cache)
+export GRADLE_BUILD_CACHE_PASSWORD

.buildkite/pipelines/periodic.yml

@@ -0,0 +1,47 @@
+steps:
+  - group: java-fips-matrix
+    steps:
+      - label: "{{matrix.ES_RUNTIME_JAVA}} / java-fips-matrix"
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.fips.enabled=true check
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            ES_RUNTIME_JAVA:
+              - java11
+              - openjdk17
+              - openjdk18
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: custom-32-98304
+          buildDirectory: /dev/shm/bk
+        env:
+          ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}"
+  - group: java-matrix
+    steps:
+      - label: "{{matrix.ES_RUNTIME_JAVA}} / java-matrix"
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true check
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            ES_RUNTIME_JAVA:
+              - java8
+              - java11
+              - openjdk16
+              - zulu8
+              - zulu11
+              - corretto11
+              - corretto8
+              - adoptopenjdk11
+              - adoptopenjdk8
+              - openjdk17
+              - openjdk18
+              - openjdk19
+              - openjdk20
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: custom-32-98304
+          buildDirectory: /dev/shm/bk
+        env:
+          ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}"

.ci/init.gradle

@@ -13,14 +13,26 @@ initscript {
 boolean USE_ARTIFACTORY = false
 
 if (System.getenv('VAULT_ADDR') == null) {
+  // When trying to reproduce errors outside of CI, it can be useful to allow this to just return rather than blowing up
+  if (System.getenv('CI') == null) {
+    return
+  }
+
   throw new GradleException("You must set the VAULT_ADDR environment variable to use this init script.")
 }
 
 if (System.getenv('VAULT_ROLE_ID') == null && System.getenv('VAULT_SECRET_ID') == null && System.getenv('VAULT_TOKEN') == null) {
+    // When trying to reproduce errors outside of CI, it can be useful to allow this to just return rather than blowing up
+  if (System.getenv('CI') == null) {
+    return
+  }
+
   throw new GradleException("You must set either the VAULT_ROLE_ID and VAULT_SECRET_ID environment variables, " +
     "or the VAULT_TOKEN environment variable to use this init script.")
 }
 
+final String vaultPathPrefix = System.getenv('VAULT_ADDR') ==~ /.+vault-ci.+\.dev.*/ ? "secret/ci/elastic-elasticsearch/migrated" : "secret/elasticsearch-ci"
+
 final String vaultToken = System.getenv('VAULT_TOKEN') ?: new Vault(
   new VaultConfig()
     .address(System.env.VAULT_ADDR)
@@ -44,8 +56,8 @@ final Vault vault = new Vault(
 
 if (USE_ARTIFACTORY) {
   final Map<String, String> artifactoryCredentials = vault.logical()
-    .read("secret/elasticsearch-ci/artifactory.elstc.co")
-    .getData();
+    .read("${vaultPathPrefix}/artifactory.elstc.co")
+    .getData()
   logger.info("Using elastic artifactory repos")
   Closure configCache = {
     return {
@@ -78,10 +90,10 @@ if (USE_ARTIFACTORY) {
   }
 }
 
-projectsLoaded {
-  rootProject {
-    project.pluginManager.withPlugin('com.gradle.build-scan') {
-      buildScan.server = 'https://gradle-enterprise.elastic.co'
+gradle.settingsEvaluated { settings ->
+  settings.pluginManager.withPlugin("com.gradle.enterprise") {
+    settings.gradleEnterprise {
+      server = 'https://gradle-enterprise.elastic.co'
     }
   }
 }
@@ -91,9 +103,9 @@ final String buildCacheUrl = System.getProperty('org.elasticsearch.build.cache.u
 final boolean buildCachePush = Boolean.valueOf(System.getProperty('org.elasticsearch.build.cache.push', 'false'))
 
 if (buildCacheUrl) {
-  final Map<String, String> buildCacheCredentials = vault.logical()
-    .read("secret/elasticsearch-ci/gradle-build-cache")
-    .getData();
+  final Map<String, String> buildCacheCredentials = System.getenv("GRADLE_BUILD_CACHE_USERNAME") ? [:] : vault.logical()
+    .read("${vaultPathPrefix}/gradle-build-cache")
+    .getData()
   gradle.settingsEvaluated { settings ->
     settings.buildCache {
       local {
@@ -104,8 +116,8 @@ if (buildCacheUrl) {
         url = buildCacheUrl
         push = buildCachePush
         credentials {
-          username = buildCacheCredentials.get("username")
-          password = buildCacheCredentials.get("password")
+          username = System.getenv("GRADLE_BUILD_CACHE_USERNAME") ?: buildCacheCredentials.get("username")
+          password = System.getenv("GRADLE_BUILD_CACHE_PASSWORD") ?: buildCacheCredentials.get("password")
         }
       }
     }

.ci/scripts/packaging-test.sh

@@ -40,6 +40,7 @@ if [ -f "/etc/os-release" ] ; then
         # Work around incorrect lintian version
         #  https://github.com/elastic/elasticsearch/issues/48573
         if [ $VERSION_ID == 10 ] ; then
+            sudo apt-get update -y
             sudo apt-get install -y --allow-downgrades lintian=2.15.0
         fi
     fi

.ci/scripts/run-gradle.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 # drop page cache and kernel slab objects on linux
 [[ -x /usr/local/sbin/drop-caches ]] && sudo /usr/local/sbin/drop-caches
+
 rm -Rfv ~/.gradle/init.d
 mkdir -p ~/.gradle/init.d && cp -v $WORKSPACE/.ci/init.gradle ~/.gradle/init.d
 if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then

build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle

@@ -19,7 +19,7 @@ buildScan {
   URL jenkinsUrl = System.getenv('JENKINS_URL') ? new URL(System.getenv('JENKINS_URL')) : null
 
   // Automatically publish scans from Elasticsearch CI
-  if (jenkinsUrl?.host?.endsWith('elastic.co') || jenkinsUrl?.host?.endsWith('elastic.dev')) {
+  if (jenkinsUrl?.host?.endsWith('elastic.co') || jenkinsUrl?.host?.endsWith('elastic.dev') || System.getenv('BUILDKITE') == 'true') {
     publishAlways()
     buildScan.server = 'https://gradle-enterprise.elastic.co'
   }

distribution/packages/build.gradle

@@ -335,6 +335,7 @@ Closure commonDebConfig(boolean oss, boolean jdk, String architecture) {
       if (oss) {
         rename('elasticsearch', 'elasticsearch-oss')
       }
+      fileMode 0644
     }
   }
 }