Add scheduled CI job to update staging artifact when dependencies change

Author: mark-vieira

.ci/jobs.t/elastic+elasticsearch+dra-staging-update.yml

@@ -0,0 +1,42 @@
+---
+- job:
+    name: elastic+elasticsearch+%BRANCH%+dra-staging-update
+    display-name: "elastic / elasticsearch # %BRANCH% - DRA staging update"
+    description: "Scheduled job to check for staging dependency updates"
+    node: master
+    triggers:
+      - timed: "*/30 * * * *"
+    builders:
+      - conditional-step:
+          condition-kind: shell
+          condition-command: |
+            #!/usr/bin/env bash
+            set -o pipefail
+            RM_BRANCH="%BRANCH%" && [[ "%BRANCH%" == "main" ]] && RM_BRANCH=master
+            BEATS_MANIFEST=$(curl -sS https://artifacts-staging.elastic.co/beats/latest/${RM_BRANCH}.json | jq -r '.manifest_url')
+            ML_MANIFEST=$(curl -sS https://artifacts-staging.elastic.co/ml-cpp/latest/${RM_BRANCH}.json | jq -r '.manifest_url')
+            ES_MANIFEST=$(curl -sS https://artifacts-staging.elastic.co/elasticsearch/latest/${RM_BRANCH}.json | jq -r '.manifest_url')
+            ES_BEATS_DEPENDENCY=$(curl -sS $ES_MANIFEST | jq -r '.projects.elasticsearch.dependencies[] | select(.prefix == "beats") | .build_uri')
+            ES_ML_DEPENDENCY=$(curl -sS $ES_MANIFEST | jq -r '.projects.elasticsearch.dependencies[] | select(.prefix == "ml-cpp") | .build_uri')
+
+            if [ "$BEATS_MANIFEST" = "$ES_BEATS_DEPENDENCY" ]; then
+              echo "ES has the latest beats"
+            else
+              echo "Need to trigger a build, $BEATS_MANIFEST available but ES has $ES_BEATS_DEPENDENCY"
+              exit 0
+            fi
+
+            if [ "$ML_MANIFEST" = "$ES_ML_DEPENDENCY" ]; then
+              echo "ES has the latest ml-cpp"
+            else
+              echo "Need to trigger a build, $ML_MANIFEST available but ES has $ES_ML_DEPENDENCY"
+              exit 0
+            fi
+
+            # A non-zero exit value means the trigger step will be skipped
+            exit 1
+          steps:
+            - trigger-builds:
+                - project: "elastic+elasticsearch+%BRANCH%+triggers/elastic+elasticsearch+%BRANCH%+dra-staging-trigger"
+                  current-parameters: false
+                  git-revision: false