Jakelandis align jackson versions

.ci/bwcVersions

@@ -57,6 +57,8 @@ BWC_VERSION:
   - "7.17.6"
   - "7.17.7"
   - "7.17.8"
+  - "7.17.9"
+  - "7.17.10"
   - "8.0.0"
   - "8.0.1"
   - "8.1.0"
@@ -78,5 +80,8 @@ BWC_VERSION:
   - "8.5.0"
   - "8.5.1"
   - "8.5.2"
+  - "8.5.3"
   - "8.6.0"
+  - "8.6.1"
+  - "8.6.2"
   - "8.7.0"

.ci/jobs.t/elastic+elasticsearch+dra-snapshot.yml

@@ -0,0 +1,67 @@
+---
+- job:
+    name: elastic+elasticsearch+%BRANCH%+dra-snapshot
+    workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+dra-snapshot
+    display-name: "elastic / elasticsearch # %BRANCH% - DRA snapshot"
+    description: "Publishing Daily Releasable Artifacts (DRAs) of Elasticsearch %BRANCH% snapshots.\n"
+    node: "ubuntu-20.04"
+    builders:
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+            RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+      - shell: |
+            #!/usr/local/bin/runbld --redirect-stderr
+            WORKFLOW="snapshot"
+            RM_BRANCH="%BRANCH%" && [[ "%BRANCH%" == "main" ]] && RM_BRANCH=master
+            ES_VERSION=$(cat build-tools-internal/version.properties \
+                  | grep elasticsearch \
+                  | sed "s/elasticsearch *= *//g")
+            VERSION_SUFFIX="" && [[ "$WORKFLOW" == "snapshot" ]] && VERSION_SUFFIX="-SNAPSHOT"
+            BEATS_BUILD_ID="$(./.ci/scripts/resolve-dra-manifest.sh beats $RM_BRANCH $ES_VERSION $WORKFLOW)"
+            ML_CPP_BUILD_ID="$(./.ci/scripts/resolve-dra-manifest.sh ml-cpp $RM_BRANCH $ES_VERSION $WORKFLOW)"
+            set -euo pipefail
+            set +x
+            VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id=$VAULT_ROLE_ID secret_id=$VAULT_SECRET_ID)
+            export VAULT_TOKEN
+            $WORKSPACE/.ci/scripts/run-gradle.sh -Ddra.artifacts=true \
+                -Ddra.artifacts.dependency.beats=${BEATS_BUILD_ID} \
+                -Ddra.artifacts.dependency.ml-cpp=${ML_CPP_BUILD_ID} \
+                -Ddra.workflow=$WORKFLOW \
+                -Dcsv=$WORKSPACE/build/distributions/dependencies-${ES_VERSION}${VERSION_SUFFIX}.csv \
+                buildReleaseArtifacts \
+                exportCompressedDockerImages \
+                :distribution:generateDependenciesReport
+
+            unset VAULT_TOKEN
+            set -x
+            $WORKSPACE/x-pack/plugin/sql/connectors/tableau/package.sh asm qualifier="$VERSION_SUFFIX"
+
+            # we regenerate this file as part of the release manager invocation
+            rm $WORKSPACE/build/distributions/elasticsearch-jdbc-${ES_VERSION}${VERSION_SUFFIX}.taco.sha512
+
+            # Allow other users access to read the artifacts so they are readable in the
+            # container
+            find $WORKSPACE -type f -path "*/build/distributions/*" -exec chmod a+r {} \;
+
+            # Allow other users write access to create checksum files
+            find $WORKSPACE -type d -path "*/build/distributions" -exec chmod a+w {} \;
+
+            # Artifacts should be generated
+            docker run --rm \
+                   --name release-manager \
+                   -e VAULT_ADDR \
+                   -e VAULT_ROLE_ID \
+                   -e VAULT_SECRET_ID \
+                   --mount type=bind,readonly=false,src="$PWD",target=/artifacts \
+                   docker.elastic.co/infra/release-manager:latest \
+                   cli collect \
+                   --project elasticsearch \
+                   --branch "$RM_BRANCH" \
+                   --commit "$GIT_COMMIT" \
+                   --workflow "$WORKFLOW" \
+                   --version "$ES_VERSION" \
+                   --artifact-set main \
+                   --dependency beats:https://artifacts-${WORKFLOW}.elastic.co/beats/${BEATS_BUILD_ID}/manifest-${ES_VERSION}${VERSION_SUFFIX}.json \
+                   --dependency ml-cpp:https://artifacts-${WORKFLOW}.elastic.co/ml-cpp/${ML_CPP_BUILD_ID}/manifest-${ES_VERSION}${VERSION_SUFFIX}.json

.ci/jobs.t/elastic+elasticsearch+dra-staging-trigger.yml

@@ -0,0 +1,6 @@
+---
+jjbb-template: periodic-trigger-lgc.yml
+vars:
+  - periodic-job: elastic+elasticsearch+%BRANCH%+dra-staging
+  - lgc-job: elastic+elasticsearch+%BRANCH%+intake
+  - cron: "H H/12 * * *"

.ci/jobs.t/elastic+elasticsearch+dra-staging.yml

@@ -1,6 +1,6 @@
 ---
 - job:
-    name: elastic+elasticsearch+%BRANCH%+dra-snapshot
+    name: elastic+elasticsearch+%BRANCH%+dra-staging
     workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+dra-staging
     display-name: "elastic / elasticsearch # %BRANCH% - DRA staging"
     description: "Publishing Daily Releasable Artifacts (DRAs) of Elasticsearch %BRANCH% staging.\n"
@@ -13,6 +13,12 @@
             RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
       - shell: |
             #!/usr/local/bin/runbld --redirect-stderr
+
+            # Don't publish main branch to staging
+            if [ "%BRANCH%" == "main" ]; then
+              exit 0
+            fi
+
             WORKFLOW="staging"
             RM_BRANCH="%BRANCH%" && [[ "%BRANCH%" == "main" ]] && RM_BRANCH=master
             ES_VERSION=$(cat build-tools-internal/version.properties \

.ci/jobs.t/elastic+elasticsearch+intake.yml

@@ -54,6 +54,13 @@
               kill-phase-on: NEVER
               current-parameters: true
               git-revision: true
+      - multijob:
+          name: Publish snapshot artifacts
+          projects:
+            - name: elastic+elasticsearch+%BRANCH%+dra-snapshot
+              kill-phase-on: NEVER
+              current-parameters: true
+              git-revision: true
       - multijob:
           name: Update last good commit
           projects:

.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-arm.yml

@@ -3,8 +3,7 @@
     name: elastic+elasticsearch+%BRANCH%+multijob+platform-support-arm
     display-name: "elastic / elasticsearch # %BRANCH% - arm compatibility"
     description: "Elasticsearch %BRANCH% ARM (aarch64) compatibility testing.\n"
-    # Don't use ramdisk for now as we are exhausting memory and causing oomkiller to trigger
-    # child-workspace: "/dev/shm/elastic+elasticsearch+%BRANCH%+multijob+platform-support-arm"
+    child-workspace: "/dev/shm/elastic+elasticsearch+%BRANCH%+multijob+platform-support-arm"
     project-type: matrix
     node: master
     scm:
@@ -17,6 +16,15 @@
           values:
             - "almalinux-8-aarch64&&immutable"
             - "ubuntu-1804-aarch64&&immutable"
+      - axis:
+          type: user-defined
+          name: GRADLE_TASK
+          values:
+            - 'checkPart1'
+            - 'checkPart2'
+            - 'checkPart3'
+            - 'bwcTestSnapshots'
+            - 'checkRestCompat'
     builders:
       - inject:
           properties-file: '.ci/java-versions-aarch64.properties'
@@ -28,4 +36,4 @@
             JAVA16_HOME=$HOME/.java/jdk16
       - shell: |
           #!/usr/local/bin/runbld --redirect-stderr
-          $WORKSPACE/.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true check
+          $WORKSPACE/.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true $GRADLE_TASK

.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-unix.yml

@@ -5,6 +5,7 @@
     description: "Elasticsearch %BRANCH% unix compatibility testing.\n"
     project-type: matrix
     node: master
+    child-workspace: "/var/lib/jenkins/workspace/elastic+elasticsearch+%BRANCH%+multijob+platform-support-unix"
     scm:
       - git:
           wipe-workspace: false

.ci/jobs.t/elastic+elasticsearch+periodic+ear.yml

@@ -16,7 +16,7 @@
           #!/bin/bash
           # Configure a dm-crypt volume backed by a file
           set -e
-          dd if=/dev/zero of=dm-crypt.img bs=1 count=0 seek=60GB
+          dd if=/dev/zero of=dm-crypt.img bs=1 count=0 seek=80GB
           dd if=/dev/urandom of=key.secret bs=2k count=1
           LOOP=$(losetup -f)
           sudo losetup $LOOP dm-crypt.img

.ci/jobs.t/elastic+elasticsearch+periodic+release-tests.yml

@@ -23,7 +23,9 @@
 
           mkdir -p ${BEATS_DIR}
           curl --fail -o "${BEATS_DIR}/metricbeat-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/metricbeat/metricbeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
-          curl --fail -o "${BEATS_DIR}/filebeat-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/filebeat/filebeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
+          curl --fail -o "${BEATS_DIR}/metricbeat-${ES_VERSION}-linux-arm64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/metricbeat/metricbeat-${ES_VERSION}-SNAPSHOT-linux-arm64.tar.gz
+          curl --fail -o "${BEATS_DIR}/filebeat-${ES_VERSION}-linux-x86_64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/metricbeat/metricbeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
+          curl --fail -o "${BEATS_DIR}/filebeat-${ES_VERSION}-linux-arm64.tar.gz" https://artifacts-snapshot.elastic.co/beats/${ES_VERSION}-SNAPSHOT/downloads/beats/filebeat/filebeat-${ES_VERSION}-SNAPSHOT-linux-arm64.tar.gz
 
           # Fetch ML artifacts
           export ML_IVY_REPO=$(mktemp -d)

.ci/jobs.t/elastic+elasticsearch+pull-request+bwc-snapshots-windows.yml

@@ -0,0 +1,55 @@
+---
+- job:
+    name: "elastic+elasticsearch+pull-request+bwc-snapshots-windows"
+    display-name: "elastic / elasticsearch - pull request bwc windows"
+    description: "Testing of Elasticsearch pull requests - bwc windows"
+    project-type: matrix
+    node: master
+    child-workspace: "C:\\Users\\jenkins\\workspace\\bwc-snapshots\\${BUILD_NUMBER}"
+    scm:
+      - git:
+          refspec: "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*"
+          branches:
+            - "${ghprbActualCommit}"
+    triggers:
+      - github-pull-request:
+          org-list:
+            - elastic
+          allow-whitelist-orgs-as-admins: true
+          trigger-phrase: '.*run\W+elasticsearch-ci/bwc-snapshots-windows.*'
+          github-hooks: true
+          status-context: elasticsearch-ci/bwc-snapshots-windows
+          cancel-builds-on-update: true
+          black-list-target-branches:
+            - 6.8
+          excluded-regions:
+            - ^docs/.*
+          white-list-labels:
+            - 'test-windows'
+          black-list-labels:
+            - '>test-mute'
+    axes:
+      - axis:
+          type: slave
+          name: nodes
+          values:
+            - "windows-immutable"
+      - axis:
+          type: yaml
+          filename: ".ci/snapshotBwcVersions"
+          name: "BWC_VERSION"
+    builders:
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$USERPROFILE\\.java\\$ES_BUILD_JAVA
+            RUNTIME_JAVA_HOME=$USERPROFILE\\.java\\$ES_RUNTIME_JAVA
+            JAVA11_HOME=$USERPROFILE\\.java\\java11
+            JAVA16_HOME=$USERPROFILE\\.java\\openjdk16
+      - batch: |
+          del /f /s /q %USERPROFILE%\.gradle\init.d\*.*
+          mkdir %USERPROFILE%\.gradle\init.d
+          copy .ci\init.gradle %USERPROFILE%\.gradle\init.d\
+          (
+            echo call %GRADLEW_BAT% --max-workers=4 -Dbwc.checkout.align=true v%BWC_VERSION%#bwcTest ^|^| exit /b 1
+          ) | java -jar "C:\Program Files\infra\bin\runbld" --redirect-stderr -

.ci/jobs.t/elastic+elasticsearch+pull-request+part-1-windows.yml

@@ -33,6 +33,7 @@
           properties-content: |
             JAVA_HOME=$USERPROFILE\\.java\\$ES_BUILD_JAVA
             RUNTIME_JAVA_HOME=$USERPROFILE\\.java\\$ES_RUNTIME_JAVA
+            JAVA11_HOME=$USERPROFILE\\.java\\java11
             JAVA16_HOME=$USERPROFILE\\.java\\openjdk16
             GRADLE_TASK=checkPart1
       - batch: |

.ci/jobs.t/elastic+elasticsearch+pull-request+part-2-windows.yml

@@ -33,6 +33,7 @@
           properties-content: |
             JAVA_HOME=$USERPROFILE\\.java\\$ES_BUILD_JAVA
             RUNTIME_JAVA_HOME=$USERPROFILE\\.java\\$ES_RUNTIME_JAVA
+            JAVA11_HOME=$USERPROFILE\\.java\\java11
             JAVA16_HOME=$USERPROFILE\\.java\\openjdk16
             GRADLE_TASK=checkPart2
       - batch: |

.ci/jobs.t/elastic+elasticsearch+pull-request+part-3-windows.yml

@@ -34,6 +34,7 @@
           properties-content: |
             JAVA_HOME=$USERPROFILE\\.java\\$ES_BUILD_JAVA
             RUNTIME_JAVA_HOME=$USERPROFILE\\.java\\$ES_RUNTIME_JAVA
+            JAVA11_HOME=$USERPROFILE\\.java\\java11
             JAVA16_HOME=$USERPROFILE\\.java\\openjdk16
             GRADLE_TASK=checkPart3
       - batch: |