Skip to content

Commit f041ab3

Browse files
v1vv1v
authored
bk: use GCP OIDC (#610)
1 parent 6155abe commit f041ab3

File tree

11 files changed

+59
-41
lines changed

11 files changed

+59
-41
lines changed

‎.buildkite/fpm-pipeline.yml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,17 @@ env:
88
MAKEFILE: "fpm"
99
BUILDX: "0"
1010

11+
# This section is used to define the plugins that will be used in the pipeline.
12+
# See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins
13+
common:
14+
- oblt_google_auth_plugin: &gcp_oidc_plugin
15+
# See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/golang-crossbuild/01-gcp-buildkite-oidc.tf
16+
# This plugin authenticates to Google Cloud using the OIDC token.
17+
elastic/oblt-google-auth#v1.3.0:
18+
lifetime: 10800 # seconds
19+
project-id: "elastic-observability-ci"
20+
project-number: "911195782929"
21+
1122
steps:
1223
- label: ":linux: multiarch Linux x86_64/arm64 FPM docker image"
1324
key: "build-and-publish-ubuntu-x86-fpm"
@@ -22,3 +33,5 @@ steps:
2233
agents:
2334
provider: "gcp"
2435
image: "${IMAGE_UBUNTU_X86_64}"
36+
plugins:
37+
- *gcp_oidc_plugin

‎.buildkite/hooks/pre-exit

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,6 @@ source .buildkite/scripts/common.sh
66

77
unset_secrets
88

9-
if [[ "$BUILDKITE_PIPELINE_SLUG" == "golang-crossbuild" && "$BUILDKITE_STEP_KEY" == build* ]]; then
10-
google_cloud_logout_active_account
11-
fi
12-
139
if [[ ( "$BUILDKITE_PIPELINE_SLUG" == "golang-crossbuild" || "$BUILDKITE_PIPELINE_SLUG" == "llvm-apple" || "$BUILDKITE_PIPELINE_SLUG" == "fpm") && ( "$BUILDKITE_STEP_KEY" == build* ) ]]; then
1410
docker logout "${DOCKER_REGISTRY}"
1511
fi

‎.buildkite/llvm-apple-pipeline.yml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,18 @@ env:
88
STAGING_IMAGE: "${DOCKER_REGISTRY}/observability-ci"
99
MAKEFILE: "go/llvm-apple"
1010
BUILDX: "0"
11+
12+
# This section is used to define the plugins that will be used in the pipeline.
13+
# See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins
14+
common:
15+
- oblt_google_auth_plugin: &gcp_oidc_plugin
16+
# See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/golang-crossbuild/01-gcp-buildkite-oidc.tf
17+
# This plugin authenticates to Google Cloud using the OIDC token.
18+
elastic/oblt-google-auth#v1.3.0:
19+
lifetime: 10800 # seconds
20+
project-id: "elastic-observability-ci"
21+
project-number: "911195782929"
22+
1123
steps:
1224
- label: ":linux: Build LLVM Apple / Ubuntu X86_64 - {{matrix.debianVersion}}"
1325
key: "build-ubuntu-x86-llvm-apple"
@@ -25,6 +37,8 @@ steps:
2537
agents:
2638
provider: "gcp"
2739
image: "${IMAGE_UBUNTU_X86_64}"
40+
plugins:
41+
- *gcp_oidc_plugin
2842
matrix:
2943
setup:
3044
debianVersion:
@@ -49,6 +63,8 @@ steps:
4963
provider: "aws"
5064
imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
5165
instanceType: "t4g.large"
66+
plugins:
67+
- *gcp_oidc_plugin
5268
matrix:
5369
setup:
5470
debianVersion:

‎.buildkite/pipeline.yml

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,17 @@ env:
99
STAGING_IMAGE: "${DOCKER_REGISTRY}/observability-ci"
1010
BUILDX: 1
1111

12+
# This section is used to define the plugins that will be used in the pipeline.
13+
# See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins
14+
common:
15+
- oblt_google_auth_plugin: &gcp_oidc_plugin
16+
# See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/golang-crossbuild/01-gcp-buildkite-oidc.tf
17+
# This plugin authenticates to Google Cloud using the OIDC token.
18+
elastic/oblt-google-auth#v1.3.0:
19+
lifetime: 10800 # seconds
20+
project-id: "elastic-observability-ci"
21+
project-number: "911195782929"
22+
1223
steps:
1324

1425
- group: "FPM"
@@ -97,6 +108,8 @@ steps:
97108
provider: "gcp"
98109
image: "${IMAGE_UBUNTU_X86_64}"
99110
instanceType: "${INSTANCE_TYPE_X86_64}"
111+
plugins:
112+
- *gcp_oidc_plugin
100113
retry:
101114
automatic:
102115
limit: 1
@@ -128,6 +141,8 @@ steps:
128141
provider: "aws"
129142
imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
130143
instanceType: "t4g.large"
144+
plugins:
145+
- *gcp_oidc_plugin
131146
retry:
132147
automatic:
133148
limit: 1
@@ -159,6 +174,8 @@ steps:
159174
provider: "gcp"
160175
image: "${IMAGE_UBUNTU_X86_64}"
161176
instanceType: "${INSTANCE_TYPE_X86_64}"
177+
plugins:
178+
- *gcp_oidc_plugin
162179
retry:
163180
automatic:
164181
limit: 1
@@ -190,6 +207,8 @@ steps:
190207
provider: "aws"
191208
imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
192209
instanceType: "t4g.large"
210+
plugins:
211+
- *gcp_oidc_plugin
193212
retry:
194213
automatic:
195214
limit: 1

‎.buildkite/scripts/build.sh

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,8 @@ check_is_arm
1111
add_bin_path
1212
with_go "${GOLANG_VERSION}"
1313
with_mage
14-
google_cloud_auth
1514

16-
make -C go -f "${MAKEFILE}" build"${is_arm}" GS_BUCKET_PATH=ingest-buildkite-ci
15+
make -C go -f "${MAKEFILE}" build"${is_arm}" GS_BUCKET_PATH=golang-crossbuild-ci-internal
1716

1817
echo "--- List Docker images staging"
1918
docker images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}" --filter=reference="${STAGING_IMAGE}/golang-crossbuild"

‎.buildkite/scripts/common.sh

Lines changed: 0 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -90,16 +90,6 @@ retry() {
9090
return 0
9191
}
9292

93-
google_cloud_auth() {
94-
echo "running google_cloud_auth"
95-
local gsUtilLocation=$(mktemp -d -p ${BIN} -t "${TMP_FOLDER}.XXXXXXXXX")
96-
GOOGLE_CREDENTIALS_FILENAME="google-cloud-credentials.json"
97-
local secretFileLocation=${gsUtilLocation}/${GOOGLE_CREDENTIALS_FILENAME}
98-
echo "${PRIVATE_CI_GCS_CREDENTIALS_SECRET}" > ${secretFileLocation}
99-
gcloud auth activate-service-account --key-file ${secretFileLocation} 2> /dev/null
100-
export GOOGLE_APPLICATION_CREDENTIALS=${secretFileLocation}
101-
}
102-
10393
unset_secrets () {
10494
for var in $(printenv | sed 's;=.*;;' | sort); do
10595
if [[ "$var" == *_SECRET || "$var" == *_TOKEN ]]; then
@@ -108,20 +98,6 @@ unset_secrets () {
10898
done
10999
}
110100

111-
google_cloud_logout_active_account() {
112-
local active_account=$(gcloud auth list --filter=status:ACTIVE --format="value(account)" 2>/dev/null)
113-
if [[ -n "$active_account" && -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]]; then
114-
echo "Logging out from GCP for active account"
115-
gcloud auth revoke $active_account > /dev/null 2>&1
116-
else
117-
echo "No active GCP accounts found."
118-
fi
119-
if [ -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]; then
120-
unset GOOGLE_APPLICATION_CREDENTIALS
121-
cleanup
122-
fi
123-
}
124-
125101
cleanup() {
126102
echo "Deleting temporary files..."
127103
rm -rf ${BIN}/${TMP_FOLDER}.*

‎.buildkite/scripts/llvm-apple/build.sh

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,8 @@ makefile=${1}
99
add_bin_path
1010
with_go "${GOLANG_VERSION}"
1111
with_mage
12-
google_cloud_auth
1312

14-
retry 3 make -C "${makefile}" build GS_BUCKET_PATH=ingest-buildkite-ci
13+
retry 3 make -C "${makefile}" build GS_BUCKET_PATH=golang-crossbuild-ci-internal
1514

1615
echo "--- List Docker images"
1716
docker images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}"

‎.buildkite/scripts/llvm-fpm/build_and_publish.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ add_bin_path
1010
with_go "${GOLANG_VERSION}"
1111
with_mage
1212

13-
retry 3 make -C "${makefile}" build GS_BUCKET_PATH=ingest-buildkite-ci
13+
retry 3 make -C "${makefile}" build GS_BUCKET_PATH=golang-crossbuild-ci-internal
1414

1515
echo "--- List Docker images"
1616
docker images --format "table {{.Repository}}:{{.Tag}}\t{{.Size}}"

‎Makefile.common

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,20 +7,20 @@ NPCAP_VERSION := 1.80
77
NPCAP_FILE := npcap-$(NPCAP_VERSION)-oem.exe
88
SUFFIX_NPCAP_VERSION := -npcap-$(NPCAP_VERSION)
99
NPCAP_REPOSITORY := docker.elastic.co/observability-ci
10-
GS_BUCKET_PATH ?= ingest-buildkite-ci
10+
GS_BUCKET_PATH ?= golang-crossbuild-ci-internal
1111

1212
# Requires login at google storage.
1313
copy-npcap:
1414
ifeq ($(CI),true)
15-
@gsutil cp gs://$(GS_BUCKET_PATH)/private/$(NPCAP_FILE) ../npcap/lib/$(NPCAP_FILE)
15+
@gcloud storage cp gs://$(GS_BUCKET_PATH)/private/$(NPCAP_FILE) ../npcap/lib/$(NPCAP_FILE)
1616
else
1717
@echo 'Only available if running in the CI'
1818
endif
1919

2020
# Requires login at google storage.
2121
copy-sdks:
2222
ifeq ($(CI),true)
23-
@gcloud storage cp gs://ingest-buildkite-ci/sdks . --recursive
23+
@gcloud storage cp gs://$(GS_BUCKET_PATH)/sdks . --recursive
2424
else
2525
@echo 'Only available if running in the CI'
2626
endif

‎NPCAP.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@ If you'd like to bump the npcap version please follow the below steps:
55
1) Update `NPCAP_VERSION` value in the `Makefile`.
66
* **NOTE**: Make sure the PR adding this is back-ported to the Go versions required by the Packetbeat CrossBuild target in [the mage file](https://github.com/elastic/beats/blob/main/x-pack/packetbeat/magefile.go). This is specified in the beats `.go-version` file.
77
2) Download the new artifact.
8-
3) Upload the artifact to `gs://ingest-buildkite-ci/private`.
9-
* **NOTE**: This particular Google Bucket can be accessible only by Elasticians who have got access to the Google project called `elastic-platform-ingest`.
8+
3) Upload the artifact to `gs://golang-crossbuild-ci-internal/private`.
9+
* **NOTE**: This particular Google Bucket can be accessible only by Elasticians who have got access to the Google project called `elastic-observability-ci`. It's managed thorugh some Terraform code.
1010

1111
Credentials to the artifact service can be found in the `APM-Shared` folder in the password management tool.
1212

0 commit comments

Comments
 (0)