Skip to content

Commit e2878ba

Browse files
oakrizanoakrizan
authored
LLVM Apple & FPM pipelines migration (#330)
1 parent d3e2bdb commit e2878ba

File tree

13 files changed

+164
-25
lines changed

13 files changed

+164
-25
lines changed

‎.buildkite/fpm-pipeline.yml

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,27 @@
1+
# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
2+
3+
env:
4+
SETUP_GVM_VERSION: "v0.5.1"
5+
IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
6+
DOCKER_REGISTRY: "docker.elastic.co"
7+
STAGING_IMAGE: "${DOCKER_REGISTRY}/observability-ci"
8+
MAKEFILE: "fpm"
9+
CHANGESET_FILE: ".buildkite/scripts/changeset/fpm"
10+
DOCKER_FILTER_REF: "docker.elastic.co/beats-dev"
11+
BUILDX: "0"
12+
113
steps:
2-
- label: "Example test"
3-
command: echo "Hello!"
14+
- label: ":linux: Build FPM / Ubuntu X86_64"
15+
key: "build-ubuntu-x86"
16+
if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.source == "ui" || build.branch == "main" || build.branch =~ /^[0-9]+\.[0-9]+$$/
17+
command:
18+
- ".buildkite/scripts/llvm-fpm/build.sh ${MAKEFILE} ${CHANGESET_FILE} ${DOCKER_FILTER_REF}"
19+
- ".buildkite/scripts/llvm-fpm/publish.sh ${MAKEFILE} ${CHANGESET_FILE}"
20+
notify:
21+
- github_commit_status:
22+
context: "Build FPM / Ubuntu X86_64"
23+
env:
24+
REPOSITORY: "${STAGING_IMAGE}"
25+
agents:
26+
provider: "gcp"
27+
image: "${IMAGE_UBUNTU_X86_64}"

‎.buildkite/hooks/pre-command

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,11 @@ GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token"
1111
# Secrets must be redacted
1212
# https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables
1313

14-
if [[ "$BUILDKITE_PIPELINE_SLUG" == "golang-crossbuild" && "$BUILDKITE_STEP_KEY" =~ ^build ]]; then
14+
if [[ "$BUILDKITE_PIPELINE_SLUG" == "golang-crossbuild" && "$BUILDKITE_STEP_KEY" == build* ]]; then
1515
export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
1616
fi
1717

18-
if [[ "$BUILDKITE_PIPELINE_SLUG" == "golang-crossbuild" && ("$BUILDKITE_STEP_KEY" =~ ^build || "$BUILDKITE_STEP_KEY" =~ ^publish)]]; then
18+
if [[ ("$BUILDKITE_PIPELINE_SLUG" == "golang-crossbuild" || "$BUILDKITE_PIPELINE_SLUG" == "llvm-apple" || "$BUILDKITE_PIPELINE_SLUG" == "fpm") && ( "$BUILDKITE_STEP_KEY" == build* ) ]]; then
1919
export DOCKER_USERNAME_SECRET=$(retry 5 vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
2020
export DOCKER_PASSWORD_SECRET=$(retry 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
2121
docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null

‎.buildkite/hooks/pre-exit

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,11 @@ source .buildkite/scripts/common.sh
66

77
unset_secrets
88

9-
if [[ "$BUILDKITE_PIPELINE_SLUG" == "golang-crossbuild" && "$BUILDKITE_STEP_KEY" =~ ^build ]]; then
9+
if [[ "$BUILDKITE_PIPELINE_SLUG" == "golang-crossbuild" && "$BUILDKITE_STEP_KEY" == build* ]]; then
1010
google_cloud_logout_active_account
1111
fi
1212

13-
if [[ "$BUILDKITE_PIPELINE_SLUG" == "golang-crossbuild" && ("$BUILDKITE_STEP_KEY" =~ ^build || "$BUILDKITE_STEP_KEY" =~ ^publish) ]]; then
13+
if [[ ( "$BUILDKITE_PIPELINE_SLUG" == "golang-crossbuild" || "$BUILDKITE_PIPELINE_SLUG" == "llvm-apple" || "$BUILDKITE_PIPELINE_SLUG" == "fpm") && ( "$BUILDKITE_STEP_KEY" == build* ) ]]; then
1414
docker logout "${DOCKER_REGISTRY}"
1515
fi
1616

‎.buildkite/llvm-apple-pipeline.yml

Lines changed: 58 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,59 @@
1+
# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
2+
3+
env:
4+
SETUP_GVM_VERSION: "v0.5.1"
5+
IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
6+
IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
7+
DOCKER_REGISTRY: "docker.elastic.co"
8+
STAGING_IMAGE: "${DOCKER_REGISTRY}/observability-ci"
9+
MAKEFILE: "go/llvm-apple"
10+
CHANGESET_FILE: ".buildkite/scripts/changeset/llvm-apple"
11+
DOCKER_FILTER_REF: "*/*/golang-crossbuild:llvm-apple*"
12+
BUILDX: "0"
113
steps:
2-
- label: "Example test"
3-
command: echo "Hello!"
14+
- label: ":linux: Build LLVM Apple / Ubuntu X86_64 - {{matrix.debianVersion}}"
15+
key: "build-ubuntu-x86"
16+
if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.source == "ui" || build.branch == "main" || build.branch =~ /^[0-9]+\.[0-9]+$$/
17+
command:
18+
- ".buildkite/scripts/llvm-fpm/build.sh ${MAKEFILE} ${CHANGESET_FILE} ${DOCKER_FILTER_REF}"
19+
- ".buildkite/scripts/llvm-fpm/publish.sh ${MAKEFILE} ${CHANGESET_FILE}"
20+
notify:
21+
- github_commit_status:
22+
context: "Build LLVM Apple / Ubuntu X86_64"
23+
env:
24+
TAG_EXTENSION: "-debian{{matrix.debianVersion}}-amd64"
25+
REPOSITORY: "${STAGING_IMAGE}"
26+
DEBIAN_VERSION: "{{matrix.debianVersion}}"
27+
agents:
28+
provider: "gcp"
29+
image: "${IMAGE_UBUNTU_X86_64}"
30+
matrix:
31+
setup:
32+
debianVersion:
33+
- "10"
34+
- "11"
35+
- "12"
36+
37+
- label: ":linux: Build LLVM Apple / Ubuntu ARM - {{matrix.debianVersion}}"
38+
key: "build-ubuntu-arm"
39+
if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.source == "ui" || build.branch == "main" || build.branch =~ /^[0-9]+\.[0-9]+$$/
40+
command:
41+
- ".buildkite/scripts/llvm-fpm/build.sh ${MAKEFILE} ${CHANGESET_FILE} ${DOCKER_FILTER_REF}"
42+
- ".buildkite/scripts/llvm-fpm/publish.sh ${MAKEFILE} ${CHANGESET_FILE}"
43+
notify:
44+
- github_commit_status:
45+
context: "Build LLVM Apple / Ubuntu ARM"
46+
env:
47+
TAG_EXTENSION: "-debian{{matrix.debianVersion}}-arm64"
48+
REPOSITORY: "${STAGING_IMAGE}"
49+
DEBIAN_VERSION: "{{matrix.debianVersion}}"
50+
agents:
51+
provider: "aws"
52+
imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
53+
instanceType: "t4g.large"
54+
matrix:
55+
setup:
56+
debianVersion:
57+
- "10"
58+
- "11"
59+
- "12"

‎.buildkite/pipeline.yml

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ env:
66
IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
77
DOCKER_REGISTRY: "docker.elastic.co"
88
STAGING_IMAGE: "${DOCKER_REGISTRY}/observability-ci"
9-
DOCKER_REGISTRY_SECRET_PATH: "secret/observability-team/ci/docker-registry/prod"
109
BUILDX: 1
1110

1211
steps:
@@ -55,17 +54,14 @@ steps:
5554

5655
- group: "Release"
5756
key: "release"
58-
notify:
59-
- github_commit_status:
60-
context: "Release / Ubuntu X86_64 && ARM"
6157

6258
steps:
6359
- label: ":linux: Release / Ubuntu X86_64 - {{matrix.makefile}}"
6460
key: "release-ubuntu-x86"
6561
command:
6662
- ".buildkite/scripts/build.sh {{matrix.makefile}}"
6763
- ".buildkite/scripts/publish.sh {{matrix.makefile}}"
68-
branches: "(main|\\d+\\.\\d+)"
64+
if: build.branch == "main" || build.branch =~ /^[0-9]+\.[0-9]+$$/
6965
agents:
7066
provider: "gcp"
7167
image: "${IMAGE_UBUNTU_X86_64}"
@@ -79,22 +75,28 @@ steps:
7975
- "Makefile.debian10"
8076
- "Makefile.debian11"
8177
- "Makefile.debian12"
78+
notify:
79+
- github_commit_status:
80+
context: "Release / Ubuntu X86_64"
8281

8382
- label: ":linux: Release / Ubuntu ARM - Makefile.debian9"
8483
key: "release-ubuntu-arm"
8584
command:
8685
- ".buildkite/scripts/build.sh Makefile.debian9"
8786
- ".buildkite/scripts/publish.sh Makefile.debian9"
88-
branches: "(main|\\d+\\.\\d+)"
87+
if: build.branch == "main" || build.branch =~ /^[0-9]+\.[0-9]+$$/
8988
agents:
9089
provider: "aws"
9190
imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
9291
instanceType: "t4g.large"
92+
notify:
93+
- github_commit_status:
94+
context: "Release / Ubuntu ARM"
9395

9496
- label: "Post-Release"
9597
key: "post-release"
9698
command: ".buildkite/scripts/post-release.sh ${GOLANG_VERSION}"
97-
branches: "(main|\\d+\\.\\d+)"
99+
if: build.branch == "main" || build.branch =~ /^[0-9]+\.[0-9]+$$/
98100
depends_on:
99101
- "release-ubuntu-x86"
100102
- "release-ubuntu-arm"
@@ -104,3 +106,4 @@ steps:
104106
agents:
105107
provider: "gcp"
106108
image: "${IMAGE_UBUNTU_X86_64}"
109+

‎.buildkite/scripts/changeset/fpm

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
^\.buildkite/fpm-pipeline.yml$
2+
^\.buildkite/scripts/llvm-fpm
3+
^\.buildkite/scripts/changeset/fpm
4+
^fpm/

‎.buildkite/scripts/changeset/llvm-apple

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
^\.buildkite/llvm-apple-pipeline.yml$
2+
^\.buildkite/scripts/llvm-apple
3+
^\.buildkite/scripts/changeset/llvm-fpm
4+
^go/llvm-apple/

‎.buildkite/scripts/common.sh

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,3 +146,15 @@ check_is_arm() {
146146
is_arm=""
147147
fi
148148
}
149+
150+
are_files_changed() {
151+
changeset=$1
152+
153+
if git diff --name-only HEAD@{1} HEAD | grep -qE "$changeset"; then
154+
return 0;
155+
else
156+
echo "WARN! No files changed in $changeset"
157+
return 1;
158+
fi
159+
}
160+

‎.buildkite/scripts/llvm-fpm/build.sh

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
source .buildkite/scripts/common.sh
6+
7+
makefile=${1}
8+
patterns=${2}
9+
docker_filter_ref=${3}
10+
11+
if ! are_files_changed "$patterns" ; then
12+
exit 0
13+
fi
14+
15+
add_bin_path
16+
with_go "${GOLANG_VERSION}"
17+
with_mage
18+
19+
retry 3 make -C "${makefile}" build GS_BUCKET_PATH=ingest-buildkite-ci
20+
docker images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" --filter=reference="${docker_filter_ref}"

‎.buildkite/scripts/llvm-fpm/publish.sh

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
source .buildkite/scripts/common.sh
6+
7+
makefile=${1}
8+
patterns=${2}
9+
10+
if ! are_files_changed "$patterns" ; then
11+
exit 0
12+
fi
13+
14+
add_bin_path
15+
retry 3 make -C ${makefile} push

0 commit comments

Comments
 (0)