Skip to content

Commit 2c8e6e8

Browse files
brianseedersbrianseeders
authored
[buildkite] Migrate DRA workflows (#99132) (#99256)
(cherry picked from commit 9aa17a5)
1 parent 3182536 commit 2c8e6e8

File tree

5 files changed

+189
-0
lines changed

5 files changed

+189
-0
lines changed

‎.buildkite/hooks/pre-command‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,3 +58,14 @@ if [[ "${USE_LUCENE_SNAPSHOT_CREDS:-}" == "true" ]]; then
5858

5959
unset data
6060
fi
61+
62+
if [[ "${USE_DRA_CREDENTIALS:-}" == "true" ]]; then
63+
DRA_VAULT_ROLE_ID_SECRET=$(vault read -field=role-id secret/ci/elastic-elasticsearch/legacy-vault-credentials)
64+
export DRA_VAULT_ROLE_ID_SECRET
65+
66+
DRA_VAULT_SECRET_ID_SECRET=$(vault read -field=secret-id secret/ci/elastic-elasticsearch/legacy-vault-credentials)
67+
export DRA_VAULT_SECRET_ID_SECRET
68+
69+
DRA_VAULT_ADDR=https://secrets.elastic.co:8200
70+
export DRA_VAULT_ADDR
71+
fi

‎.buildkite/pipelines/dra-workflow.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
steps:
2+
- command: .buildkite/scripts/dra-workflow.sh
3+
env:
4+
USE_DRA_CREDENTIALS: "true"
5+
agents:
6+
provider: gcp
7+
image: family/elasticsearch-ubuntu-2204
8+
machineType: custom-32-98304
9+
buildDirectory: /dev/shm/bk

‎.buildkite/scripts/dra-update-staging.sh‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
5+
source .buildkite/scripts/branches.sh
6+
7+
for BRANCH in "${BRANCHES[@]}"; do
8+
# Don't publish main branch to staging
9+
if [[ "$BRANCH" == "main" ]]; then
10+
continue
11+
fi
12+
13+
echo "--- Checking $BRANCH"
14+
15+
BEATS_MANIFEST=$(curl -sS "https://artifacts-staging.elastic.co/beats/latest/${BRANCH}.json" | jq -r '.manifest_url')
16+
ML_MANIFEST=$(curl -sS "https://artifacts-staging.elastic.co/ml-cpp/latest/${BRANCH}.json" | jq -r '.manifest_url')
17+
ES_MANIFEST=$(curl -sS "https://artifacts-staging.elastic.co/elasticsearch/latest/${BRANCH}.json" | jq -r '.manifest_url')
18+
19+
ES_BEATS_DEPENDENCY=$(curl -sS "$ES_MANIFEST" | jq -r '.projects.elasticsearch.dependencies[] | select(.prefix == "beats") | .build_uri')
20+
ES_ML_DEPENDENCY=$(curl -sS "$ES_MANIFEST" | jq -r '.projects.elasticsearch.dependencies[] | select(.prefix == "ml-cpp") | .build_uri')
21+
22+
SHOULD_TRIGGER=""
23+
24+
if [ "$BEATS_MANIFEST" = "$ES_BEATS_DEPENDENCY" ]; then
25+
echo "ES has the latest beats"
26+
else
27+
echo "Need to trigger a build, $BEATS_MANIFEST available but ES has $ES_BEATS_DEPENDENCY"
28+
SHOULD_TRIGGER=true
29+
fi
30+
31+
if [ "$ML_MANIFEST" = "$ES_ML_DEPENDENCY" ]; then
32+
echo "ES has the latest ml-cpp"
33+
else
34+
echo "Need to trigger a build, $ML_MANIFEST available but ES has $ES_ML_DEPENDENCY"
35+
SHOULD_TRIGGER=true
36+
fi
37+
38+
if [[ "$SHOULD_TRIGGER" == "true" ]]; then
39+
echo "Triggering DRA staging workflow for $BRANCH"
40+
cat << EOF | buildkite-agent pipeline upload
41+
steps:
42+
- trigger: elasticsearch-dra-workflow
43+
label: Trigger DRA staging workflow for $BRANCH
44+
async: true
45+
build:
46+
branch: "$BRANCH"
47+
env:
48+
DRA_WORKFLOW: staging
49+
EOF
50+
fi
51+
done

‎.buildkite/scripts/dra-workflow.sh‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
5+
WORKFLOW="${DRA_WORKFLOW:-snapshot}"
6+
BRANCH="${BUILDKITE_BRANCH:-}"
7+
8+
# Don't publish main branch to staging
9+
if [[ "$BRANCH" == "main" && "$WORKFLOW" == "staging" ]]; then
10+
exit 0
11+
fi
12+
13+
echo --- Preparing
14+
15+
# TODO move this to image
16+
sudo apt-get update -y
17+
sudo apt-get install -y libxml2-utils python3.10-venv
18+
19+
RM_BRANCH="$BRANCH"
20+
if [[ "$BRANCH" == "main" ]]; then
21+
RM_BRANCH=master
22+
fi
23+
24+
ES_VERSION=$(grep elasticsearch build-tools-internal/version.properties | sed "s/elasticsearch *= *//g")
25+
26+
VERSION_SUFFIX=""
27+
if [[ "$WORKFLOW" == "snapshot" ]]; then
28+
VERSION_SUFFIX="-SNAPSHOT"
29+
fi
30+
31+
BEATS_BUILD_ID="$(./.ci/scripts/resolve-dra-manifest.sh beats "$RM_BRANCH" "$ES_VERSION" "$WORKFLOW")"
32+
ML_CPP_BUILD_ID="$(./.ci/scripts/resolve-dra-manifest.sh ml-cpp "$RM_BRANCH" "$ES_VERSION" "$WORKFLOW")"
33+
34+
LICENSE_KEY_ARG=""
35+
BUILD_SNAPSHOT_ARG=""
36+
37+
if [[ "$WORKFLOW" == "staging" ]]; then
38+
LICENSE_KEY=$(mktemp -d)/license.key
39+
# Notice that only the public key is being read here, which isn't really secret
40+
vault read -field pubkey secret/ci/elastic-elasticsearch/migrated/license | base64 --decode > "$LICENSE_KEY"
41+
LICENSE_KEY_ARG="-Dlicense.key=$LICENSE_KEY"
42+
43+
BUILD_SNAPSHOT_ARG="-Dbuild.snapshot=false"
44+
fi
45+
46+
echo --- Building release artifacts
47+
48+
.ci/scripts/run-gradle.sh -Ddra.artifacts=true \
49+
-Ddra.artifacts.dependency.beats="${BEATS_BUILD_ID}" \
50+
-Ddra.artifacts.dependency.ml-cpp="${ML_CPP_BUILD_ID}" \
51+
-Ddra.workflow="$WORKFLOW" \
52+
-Dcsv="$WORKSPACE/build/distributions/dependencies-${ES_VERSION}${VERSION_SUFFIX}.csv" \
53+
$LICENSE_KEY_ARG \
54+
$BUILD_SNAPSHOT_ARG \
55+
buildReleaseArtifacts \
56+
exportCompressedDockerImages \
57+
:distribution:generateDependenciesReport
58+
59+
PATH="$PATH:${JAVA_HOME}/bin" # Required by the following script
60+
x-pack/plugin/sql/connectors/tableau/package.sh asm qualifier="$VERSION_SUFFIX"
61+
62+
# we regenerate this file as part of the release manager invocation
63+
rm "build/distributions/elasticsearch-jdbc-${ES_VERSION}${VERSION_SUFFIX}.taco.sha512"
64+
65+
# Allow other users access to read the artifacts so they are readable in the
66+
# container
67+
find "$WORKSPACE" -type f -path "*/build/distributions/*" -exec chmod a+r {} \;
68+
69+
# Allow other users write access to create checksum files
70+
find "$WORKSPACE" -type d -path "*/build/distributions" -exec chmod a+w {} \;
71+
72+
echo --- Running release-manager
73+
74+
# Artifacts should be generated
75+
docker run --rm \
76+
--name release-manager \
77+
-e VAULT_ADDR="$DRA_VAULT_ADDR" \
78+
-e VAULT_ROLE_ID="$DRA_VAULT_ROLE_ID_SECRET" \
79+
-e VAULT_SECRET_ID="$DRA_VAULT_SECRET_ID_SECRET" \
80+
--mount type=bind,readonly=false,src="$PWD",target=/artifacts \
81+
docker.elastic.co/infra/release-manager:latest \
82+
cli collect \
83+
--project elasticsearch \
84+
--branch "$RM_BRANCH" \
85+
--commit "$BUILDKITE_COMMIT" \
86+
--workflow "$WORKFLOW" \
87+
--version "$ES_VERSION" \
88+
--artifact-set main \
89+
--dependency "beats:https://artifacts-${WORKFLOW}.elastic.co/beats/${BEATS_BUILD_ID}/manifest-${ES_VERSION}${VERSION_SUFFIX}.json" \
90+
--dependency "ml-cpp:https://artifacts-${WORKFLOW}.elastic.co/ml-cpp/${ML_CPP_BUILD_ID}/manifest-${ES_VERSION}${VERSION_SUFFIX}.json"

‎.buildkite/scripts/dra-workflow.trigger.sh‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
5+
echo "steps:"
6+
7+
source .buildkite/scripts/branches.sh
8+
9+
for BRANCH in "${BRANCHES[@]}"; do
10+
if [[ "$BRANCH" == "main" ]]; then
11+
continue
12+
fi
13+
14+
INTAKE_PIPELINE_SLUG="elasticsearch-intake"
15+
BUILD_JSON=$(curl -sH "Authorization: Bearer ${BUILDKITE_API_TOKEN}" "https://api.buildkite.com/v2/organizations/elastic/pipelines/${INTAKE_PIPELINE_SLUG}/builds?branch=${BRANCH}&state=passed&per_page=1" | jq '.[0] | {commit: .commit, url: .web_url}')
16+
LAST_GOOD_COMMIT=$(echo "${BUILD_JSON}" | jq -r '.commit')
17+
18+
cat <<EOF
19+
- trigger: elasticsearch-dra-workflow
20+
label: Trigger DRA staging workflow for $BRANCH
21+
async: true
22+
build:
23+
branch: "$BRANCH"
24+
commit: "$LAST_GOOD_COMMIT"
25+
env:
26+
DRA_WORKFLOW: staging
27+
EOF
28+
done

0 commit comments

Comments
 (0)