Separete branches in ci (#184)

* Use target branch in build CI. Properly name the CI jobs.

* Add unit test and integration tests on LS 8.x snapshot ES main branch.

* Add the target branch to the E2E tests for LS 8.x

* Add a step to E2E to cover LS snapshot vs ES main.

* Use multi-jdk BK agent and use JDK version properly in E2E pipeline.

* Add explanation comments for each jobs of each CI.

Author: mashhurs

.buildkite/build-pipeline.yml

@@ -6,25 +6,41 @@ agents:
   image: family/core-ubuntu-2204
 
 steps:
-  # ------------- Build with ES released versions ---------------------
-  # 8.x + (SNAPSHOT=FALSE) -> treeish: v8.11.1  artifact: 8.11.1
-  - label: ":hammer: Build plugin with released Elasticsearch version :elasticsearch:"
+  - label: ":hammer: Build plugin with LS & ES 8.x :elasticsearch:"
+    # Builds with LS and ES last 8.x released version
+    # Runs integration tests on 8.x released versions
     command:
       - .buildkite/scripts/run_tests.sh
     env:
       ELASTIC_STACK_VERSION: "8.x"
       SNAPSHOT: false
       INTEGRATION: true
       SECURE_INTEGRATION: true
+      TARGET_BRANCH: "8.x"
+      # temporary definition to cover PR-170
+      # TODO: remove once 8.16 released
+      ELASTICSEARCH_TREEISH: 8.16
 
-  # ------------- Build with ES snapshot versions and main branch ---------------------
-  # main + (SNAPSHOT=TRUE) -> treeish: main     artifact:8.12.0-SNAPSHOT
-  - label: ":hammer: Build plugin with Elasticsearch snapshot versions and `main` branch :elasticsearch:"
+  - label: ":hammer: Build plugin with LS 8.x-SNAPSHOT & ES `main` branch :elasticsearch:"
+    # Builds with LS last 8.x released version and ES main
+    # Runs integration tests on 8.x released versions
     command:
       - .buildkite/scripts/run_tests.sh
     env:
       ELASTIC_STACK_VERSION: "8.x"
       ELASTICSEARCH_TREEISH: "main"
       SNAPSHOT: true
       INTEGRATION: true
+      SECURE_INTEGRATION: true
+
+  - label: ":hammer: Build plugin with LS & ES `main` branch :elasticsearch:"
+    # Builds with LS last 8.x released version and ES main
+    # Runs integration tests on 8.x released versions
+    command:
+      - .buildkite/scripts/run_tests.sh
+    env:
+      ELASTIC_STACK_VERSION: "main"
+      ELASTICSEARCH_TREEISH: "main"
+      SNAPSHOT: true
+      INTEGRATION: true
       SECURE_INTEGRATION: true

.buildkite/e2e-pipeline.yml

@@ -3,14 +3,36 @@
 agents:
   provider: gcp
   imageProject: elastic-images-prod
-  image: family/platform-ingest-logstash-ubuntu-2204
+  image: family/platform-ingest-logstash-multi-jdk-ubuntu-2204
   machineType: "n2-standard-4"
   diskSizeGb: 120
 
 steps:
   # ------------- Run E2E tests ---------------------
-  - label: ":test_tube: Build plugin and run E2E tests :rocket:"
+  - label: ":test_tube: Run E2E tests with LS 8.x :rocket:"
+    # uses the LS main & plugin 8.x branch when building the plugin
+    # Runs integration tests against 8.x release version
     command:
       - .buildkite/scripts/run_e2e_tests.sh
     env:
-      ELASTIC_STACK_VERSION: "8.x"
+      ELASTIC_STACK_VERSION: "8.x"
+      TARGET_BRANCH: "8.x"
+
+  - label: ":test_tube: Run E2E tests with LS 8.x-SNAPSHOT :rocket:"
+    # uses the LS & plugin main branch when building the plugin
+    # Runs integration tests against 8.x-SNAPSHOT version
+    command:
+      - .buildkite/scripts/run_e2e_tests.sh
+    env:
+      ELASTIC_STACK_VERSION: "8.x"
+      SNAPSHOT: true
+
+  - label: ":test_tube: Run E2E tests with LS `main` :rocket:"
+    # uses the LS & plugin main branch when building the plugin
+    # Runs integration tests against snapshot.main of https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json
+    command:
+      - .buildkite/scripts/run_e2e_tests.sh
+    env:
+      ELASTIC_STACK_VERSION: main
+      SNAPSHOT: true
+      ELASTICSEARCH_TREEISH: main

.buildkite/pull-request-pipeline.yml

@@ -6,16 +6,31 @@ agents:
 
 steps:
   # ------------- Unit tests ---------------------
-  - label: ":hammer: CI setup and unit tests run :docker:"
+  - label: ":hammer: Unit tests with LS & ES 8.x-SNAPSHOT :docker:"
+    # Builds the plugin (with current changes) against LS 8.x-SNAPSHOT and ES version defined in gradle.properties
+    # Runs unit tests on LS & ES 8.x-SNAPSHOT docker
     command:
       - .buildkite/scripts/run_tests.sh
     env:
       ELASTIC_STACK_VERSION: "8.x"
+      SNAPSHOT: true
+      INTEGRATION: false
+
+  - label: ":hammer: Unit tests with LS 8.x-SNAPSHOT & ES main :docker:"
+    # Builds the plugin (with current changes) against LS 8.x-SNAPSHOT and ES main
+    # Runs unit tests on LS & ES 8.x-SNAPSHOT docker
+    command:
+      - .buildkite/scripts/run_tests.sh
+    env:
+      ELASTIC_STACK_VERSION: "8.x"
+      ELASTICSEARCH_TREEISH: main
       INTEGRATION: false
       SNAPSHOT: true
 
   # ------------- Integration tests ---------------------
-  - label: ":hammer: CI setup and integration tests with ES 8.x run on :docker:"
+  - label: ":hammer: Integration tests with LS & ES 8.x :docker:"
+    # Builds the plugin (with current changes) against LS 8.x and ES version defined in gradle.properties
+    # Runs integration tests on LS & ES 8.x docker
     command:
       - .buildkite/scripts/run_tests.sh
     env:
@@ -26,12 +41,40 @@ steps:
       # TODO: remove once 8.16 released
       ELASTICSEARCH_TREEISH: 8.16
 
-  - label: ":hammer: CI setup and integration tests with ES 8.x-SNAPSHOT run on :docker:"
+  - label: ":hammer: Integration tests with LS & ES 8.x-SNAPSHOT :docker:"
+    # Builds the plugin (with current changes) against LS 8.x-SNAPSHOT and ES version defined in gradle.properties
+    # Runs integration tests on LS & ES 8.x-SNAPSHOT docker
+    command:
+      - .buildkite/scripts/run_tests.sh
+    env:
+      ELASTIC_STACK_VERSION: "8.x"
+      SNAPSHOT: true
+      INTEGRATION: true
+      SECURE_INTEGRATION: true
+      LOG_LEVEL: "info"
+
+  - label: ":hammer: Integration tests with LS 8.x-SNAPSHOT & ES main :docker:"
+    # Builds the plugin (with current changes) against LS 8.x-SNAPSHOT and ES main
+    # Runs integration tests on LS & ES 8.x-SNAPSHOT docker
     command:
       - .buildkite/scripts/run_tests.sh
     env:
       ELASTIC_STACK_VERSION: "8.x"
+      ELASTICSEARCH_TREEISH: "main"
+      SNAPSHOT: true
       INTEGRATION: true
+      SECURE_INTEGRATION: true
       LOG_LEVEL: "info"
+
+  - label: ":hammer: Integration tests with LS & ES main :docker:"
+    # Builds the plugin (with current changes) against LS and ES main
+    # Runs integration tests on snapshot.main of https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json
+    command:
+      - .buildkite/scripts/run_tests.sh
+    env:
+      ELASTIC_STACK_VERSION: main
+      ELASTICSEARCH_TREEISH: main
+      SNAPSHOT: true
+      INTEGRATION: true
       SECURE_INTEGRATION: true
-      SNAPSHOT: true
+      LOG_LEVEL: "info"

.buildkite/scripts/run_e2e_tests.sh

@@ -2,26 +2,58 @@
 
 set -euo pipefail
 
-export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:/opt/buildkite-agent/.java/bin:$PATH"
-export JAVA_HOME="/opt/buildkite-agent/.java"
+export PATH="/opt/buildkite-agent/.rbenv/bin:/opt/buildkite-agent/.pyenv/bin:$PATH"
 eval "$(rbenv init -)"
 eval "$(pyenv init -)"
 
-VERSION_URL="https://storage.googleapis.com/artifacts-api/releases/current"
+VERSION_URL="https://raw.githubusercontent.com/elastic/logstash/main/ci/logstash_releases.json"
 
 ###
 # Resolve stack version and export
 resolve_current_stack_version() {
   set +o nounset
+  echo "Fetching versions from $VERSION_URL"
+  VERSIONS=$(curl --retry 5 --retry-delay 5 -fsSL $VERSION_URL)
 
-  local major_version="${ELASTIC_STACK_VERSION%%.*}"
-  local version=$(curl --retry 5 --retry-delay 5 -fsSL "$VERSION_URL/$major_version")
+  if [[ "$SNAPSHOT" == "true" ]]; then
+    key=$(echo "$VERSIONS" | jq -r '.snapshots."'"$ELASTIC_STACK_VERSION"'"')
+    echo "resolved key: $key"
+  else
+    key=$(echo "$VERSIONS" | jq -r '.releases."'"$ELASTIC_STACK_VERSION"'"')
+  fi
 
-  echo "Resolved version: $version"
-  export STACK_VERSION="$version"
+  echo "Resolved version: $key"
+  export STACK_VERSION="$key"
 }
 
-resolve_current_stack_version
+###
+# Checkout the target branch if defined
+checkout_target_branch() {
+  set +o nounset
+  if [ -z "$TARGET_BRANCH" ]; then
+    echo "Target branch is not specified, using default branch: main or BK defined"
+  else
+    echo "Changing the branch for ${TARGET_BRANCH}"
+    git checkout "$TARGET_BRANCH"
+  fi
+}
+
+set_required_jdk() {
+  set +o nounset
+  java_version="$(cat .java-version)"
+  echo "Required JDK version: $java_version"
+  if [[ "$java_version" == "17.0" ]]; then
+    jdk_home="/opt/buildkite-agent/.java/adoptiumjdk_17"
+  elif [[ "$java_version" == "21.0" ]]; then
+    jdk_home="/opt/buildkite-agent/.java/adoptiumjdk_21"
+  else
+    echo "Unsupported JDK."
+    exit 1
+  fi
+
+  export JAVA_HOME=$jdk_home
+  export PATH="$jdk_home:$PATH"
+}
 
 ###
 # Build the plugin, to do so we need Logstash source
@@ -41,6 +73,9 @@ build_plugin() {
   ./gradlew clean vendor localGem
 }
 
+resolve_current_stack_version
+checkout_target_branch
+set_required_jdk
 build_logstash
 build_plugin
 

.buildkite/scripts/run_tests.sh

@@ -5,4 +5,11 @@ else
   echo "Using ELASTICSEARCH_TREEISH ${ELASTICSEARCH_TREEISH} defined in the ENV."
 fi
 
+if [ -z "$TARGET_BRANCH" ]; then
+  echo "Target branch is not specified, using default branch: main or BK defined"
+else
+  echo "Changing the branch for ${TARGET_BRANCH}"
+  git checkout "$TARGET_BRANCH"
+fi
+
 mkdir -p .ci && curl -sL --retry 5 --retry-delay 5 https://github.com/logstash-plugins/.ci/archive/buildkite-1.x.tar.gz | tar zxvf - --skip-old-files --strip-components=1 -C .ci --wildcards '*Dockerfile*' '*docker*' '*.sh' && .ci/docker-setup.sh && .ci/docker-run.sh