Skip to content

Commit 173396b

Browse files
jcsfjcsf
committed
TUN-9800: Migrate cloudflared-ci pipelines to Gitlab CI
## Summary This commit migrates the cloduflared ci pipelines, that built, tested and component tested the linux binaries to gitlab ci. The only thing that is remaining to move from teamcity to gitlab are now the release pipelines that run on master. Relates to TUN-9800
1 parent d9e13ab commit 173396b

File tree

14 files changed

+209
-83
lines changed

14 files changed

+209
-83
lines changed

‎.ci/ci-image.gitlab-ci.yml‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,9 @@ include:
88
inputs:
99
stage: pre-build
1010
jobPrefix: ci-image
11-
# runOnChangesTo: [".ci/image/**"]
12-
# runOnMR: true
13-
# runOnBranches: '^master$'
14-
runOnBranches: "^.+$"
11+
runOnChangesTo: [".ci/image/**"]
12+
runOnMR: true
13+
runOnBranches: '^master$'
1514
commentImageRefs: false
1615
runner: vm-linux-x86-4cpu-8gb
1716
EXTRA_DIB_ARGS: "--manifest=.ci/image/.docker-images"
@@ -23,9 +22,8 @@ include:
2322
inputs:
2423
stage: pre-build
2524
jobPrefix: ci-image
26-
# runOnMR: true
27-
# runOnBranches: '^master$'
28-
runOnBranches: "^.+$"
25+
runOnMR: true
26+
runOnBranches: '^master$'
2927
IMAGE_PATH: "$REGISTRY_HOST/stash/tun/cloudflared/ci-image/master"
3028
VARIABLE_NAME: BUILD_IMAGE
3129
needs:

‎.ci/commons.gitlab-ci.yml‎

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,19 @@
55
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
66
when: always
77
- when: never
8-
# Rules to run the job only on branches that are not master. This is needed because for now
9-
# we need to keep a similar behavior due to the integration with teamcity, which requires us
10-
# to not trigger pipelines on tags and/or merge requests.
11-
run-on-branch:
8+
# Rules to run the job only on merge requests
9+
run-on-mr:
1210
- if: $CI_COMMIT_TAG
1311
when: never
14-
- if: $CI_PIPELINE_SOURCE != "merge_request_event" && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
12+
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
13+
when: always
14+
- when: never
15+
# Rules to run the job on merge_requests and master branch
16+
run-always:
17+
- if: $CI_COMMIT_TAG
18+
when: never
19+
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
20+
- if: $CI_COMMIT_BRANCH != null && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
1521
when: always
1622
- when: never
1723

@@ -28,4 +34,20 @@
2834
else
2935
echo "No tag present — skipping."
3036
exit 0
31-
fi
37+
fi
38+
39+
.component-tests:
40+
image: $BUILD_IMAGE
41+
rules:
42+
- !reference [.default-rules, run-always]
43+
variables:
44+
COMPONENT_TESTS_CONFIG: component-test-config.yaml
45+
COMPONENT_TESTS_CONFIG_CONTENT: Y2xvdWRmbGFyZWRfYmluYXJ5OiBjbG91ZGZsYXJlZC5leGUKY3JlZGVudGlhbHNfZmlsZTogY3JlZC5qc29uCm9yaWdpbmNlcnQ6IGNlcnQucGVtCnpvbmVfZG9tYWluOiBhcmdvdHVubmVsdGVzdC5jb20Kem9uZV90YWc6IDQ4Nzk2ZjFlNzBiYjc2NjljMjliYjUxYmEyODJiZjY1
46+
secrets:
47+
DNS_API_TOKEN:
48+
vault: gitlab/cloudflare/tun/cloudflared/_dev/_terraform_atlantis/component_tests_token/data@kv
49+
file: false
50+
COMPONENT_TESTS_ORIGINCERT:
51+
vault: gitlab/cloudflare/tun/cloudflared/_dev/component_tests_cert_pem/data@kv
52+
file: false
53+
cache: {}

‎.ci/image/Dockerfile‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,9 @@ RUN apt-get update && \
77
apt-get install --no-install-recommends --allow-downgrades -y \
88
build-essential \
99
git \
10-
go-boring=1.24.4-1 \
10+
go-boring=1.24.6-1 \
1111
libffi-dev \
12+
procps \
1213
python3-dev \
1314
python3-pip \
1415
python3-setuptools \

‎.ci/linux.gitlab-ci.yml‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
.golang-inputs: &golang_inputs
2+
runOnMR: true
3+
runOnBranches: '^master$'
4+
outputDir: artifacts
5+
runner: linux-x86-8cpu-16gb
6+
stage: build
7+
golangVersion: "boring-1.24"
8+
CGO_ENABLED: 1
9+
10+
include:
11+
###################
12+
### Linux Build ###
13+
###################
14+
- component: $CI_SERVER_FQDN/cloudflare/ci/golang/boring-make@~latest
15+
inputs:
16+
<<: *golang_inputs
17+
jobPrefix: linux-build
18+
GOLANG_MAKE_TARGET: ci-build
19+
20+
########################
21+
### Linux FIPS Build ###
22+
########################
23+
- component: $CI_SERVER_FQDN/cloudflare/ci/golang/boring-make@~latest
24+
inputs:
25+
<<: *golang_inputs
26+
jobPrefix: linux-fips-build
27+
GOLANG_MAKE_TARGET: ci-fips-build
28+
29+
#################
30+
### Unit Tests ##
31+
#################
32+
- component: $CI_SERVER_FQDN/cloudflare/ci/golang/boring-make@~latest
33+
inputs:
34+
<<: *golang_inputs
35+
stage: test
36+
jobPrefix: test
37+
GOLANG_MAKE_TARGET: ci-test
38+
39+
######################
40+
### Unit Tests FIPS ##
41+
######################
42+
- component: $CI_SERVER_FQDN/cloudflare/ci/golang/boring-make@~latest
43+
inputs:
44+
<<: *golang_inputs
45+
stage: test
46+
jobPrefix: test-fips
47+
GOLANG_MAKE_TARGET: ci-fips-test
48+
49+
#################
50+
### Vuln Check ##
51+
#################
52+
- component: $CI_SERVER_FQDN/cloudflare/ci/golang/boring-make@~latest
53+
inputs:
54+
<<: *golang_inputs
55+
runOnBranches: '^$'
56+
stage: validate
57+
jobPrefix: vulncheck
58+
GOLANG_MAKE_TARGET: vulncheck
59+
60+
#################################
61+
### Run Linux Component Tests ###
62+
#################################
63+
component-tests-linux: &component-tests-linux
64+
stage: test
65+
extends: .component-tests
66+
needs:
67+
- ci-image-get-image-ref
68+
- linux-build-boring-make
69+
script:
70+
- ./.ci/scripts/component-tests.sh
71+
variables: &component-tests-variables
72+
CI: 1
73+
COMPONENT_TESTS_CONFIG_CONTENT: Y2xvdWRmbGFyZWRfYmluYXJ5OiBjbG91ZGZsYXJlZApjcmVkZW50aWFsc19maWxlOiBjcmVkLmpzb24Kb3JpZ2luY2VydDogY2VydC5wZW0Kem9uZV9kb21haW46IGFyZ290dW5uZWx0ZXN0LmNvbQp6b25lX3RhZzogNDg3OTZmMWU3MGJiNzY2OWMyOWJiNTFiYTI4MmJmNjU=
74+
tags:
75+
- linux-x86-8cpu-16gb
76+
artifacts:
77+
reports:
78+
junit: report.xml
79+
80+
######################################
81+
### Run Linux FIPS Component Tests ###
82+
######################################
83+
component-tests-linux-fips:
84+
<<: *component-tests-linux
85+
needs:
86+
- ci-image-get-image-ref
87+
- linux-fips-build-boring-make
88+
variables:
89+
<<: *component-tests-variables
90+
COMPONENT_TESTS_FIPS: 1

‎.ci/mac.gitlab-ci.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ include:
66
###############################
77
.mac-build-defaults: &mac-build-defaults
88
rules:
9-
- !reference [.default-rules, run-on-branch]
9+
- !reference [.default-rules, run-on-mr]
1010
tags:
1111
- "macstadium-${RUNNER_ARCH}"
1212
parallel:

‎.ci/scripts/component-tests.sh‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#!/bin/bash
2+
set -e -o pipefail
3+
4+
# Fetch cloudflared from the artifacts folder
5+
mv ./artifacts/cloudflared ./cloudflared
6+
7+
python3 -m venv env
8+
. env/bin/activate
9+
10+
pip install --upgrade -r component-tests/requirements.txt
11+
12+
# Creates and routes a Named Tunnel for this build. Also constructs
13+
# config file from env vars.
14+
python3 component-tests/setup.py --type create
15+
16+
# Define the cleanup function
17+
cleanup() {
18+
# The Named Tunnel is deleted and its route unprovisioned here.
19+
python3 component-tests/setup.py --type cleanup
20+
}
21+
22+
# The trap will call the cleanup function on script exit
23+
trap cleanup EXIT
24+
25+
pytest component-tests -o log_cli=true --log-cli-level=INFO --junit-xml=report.xml

‎fmt-check.sh‎ renamed to ‎.ci/scripts/fmt-check.sh‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
11
#!/bin/bash
2-
32
set -e -o pipefail
43

5-
OUTPUT=$(goimports -l -d -local github.com/cloudflare/cloudflared $(go list -mod=vendor -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc))
4+
OUTPUT=$(go run -mod=readonly golang.org/x/tools/cmd/goimports@v0.30.0 -l -d -local github.com/cloudflare/cloudflared $(go list -mod=vendor -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc))
65

76
if [ -n "$OUTPUT" ] ; then
87
PAGER=$(which colordiff || echo cat)

‎.ci/scripts/windows/component-test.ps1‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ Write-Host "Running component tests"
3131
try {
3232
python -m pip --disable-pip-version-check install --upgrade -r component-tests/requirements.txt --use-pep517
3333
python component-tests/setup.py --type create
34-
python -m pytest component-tests -o log_cli=true --log-cli-level=INFO
34+
python -m pytest component-tests -o log_cli=true --log-cli-level=INFO --junit-xml=report.xml
3535
if ($LASTEXITCODE -ne 0) {
3636
throw "Failed component tests"
3737
}

‎.ci/scripts/windows/go-wrapper.ps1‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ Param(
33
[string]$ScriptToExecute
44
)
55

6-
# This script its a wrapper that downloads a specific version
6+
# The script is a wrapper that downloads a specific version
77
# of go, adds it to the PATH and executes a script with that go
88
# version in the path.
99

‎.ci/windows.gitlab-ci.yml‎

Lines changed: 9 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ include:
66
###################################
77
.windows-build-defaults: &windows-build-defaults
88
rules:
9-
- !reference [.default-rules, run-on-branch]
9+
- !reference [.default-rules, run-always]
1010
tags:
1111
- windows-x86
1212
cache: {}
@@ -27,27 +27,20 @@ build-cloudflared-windows:
2727
### Load Environment Variables for Component Tests ###
2828
######################################################
2929
load-windows-env-variables:
30-
rules:
31-
- !reference [.default-rules, run-on-branch]
3230
stage: pre-build
31+
extends: .component-tests
3332
script:
34-
- echo "COMPONENT_TESTS_CONFIG=component-test-config.yaml" >> windows.env
35-
- echo "COMPONENT_TESTS_CONFIG_CONTENT=Y2xvdWRmbGFyZWRfYmluYXJ5OiBjbG91ZGZsYXJlZC5leGUKY3JlZGVudGlhbHNfZmlsZTogY3JlZC5qc29uCm9yaWdpbmNlcnQ6IGNlcnQucGVtCnpvbmVfZG9tYWluOiBhcmdvdHVubmVsdGVzdC5jb20Kem9uZV90YWc6IDQ4Nzk2ZjFlNzBiYjc2NjljMjliYjUxYmEyODJiZjY1" >> windows.env
33+
- echo "COMPONENT_TESTS_CONFIG=$COMPONENT_TESTS_CONFIG" >> windows.env
34+
- echo "COMPONENT_TESTS_CONFIG_CONTENT=$COMPONENT_TESTS_CONFIG_CONTENT" >> windows.env
3635
- echo "DNS_API_TOKEN=$DNS_API_TOKEN" >> windows.env
3736
# We have to encode the `COMPONENT_TESTS_ORIGINCERT` secret, because it content is a file, otherwise we can't export it using gitlab
3837
- echo "COMPONENT_TESTS_ORIGINCERT=$(echo "$COMPONENT_TESTS_ORIGINCERT" | base64 -w0)" >> windows.env
39-
secrets:
40-
DNS_API_TOKEN:
41-
vault: gitlab/cloudflare/tun/cloudflared/_dev/_terraform_atlantis/component_tests_token/data@kv
42-
file: false
43-
COMPONENT_TESTS_ORIGINCERT:
44-
vault: gitlab/cloudflare/tun/cloudflared/_dev/component_tests_cert_pem/data@kv
45-
file: false
38+
variables:
39+
COMPONENT_TESTS_CONFIG_CONTENT: Y2xvdWRmbGFyZWRfYmluYXJ5OiBjbG91ZGZsYXJlZC5leGUKY3JlZGVudGlhbHNfZmlsZTogY3JlZC5qc29uCm9yaWdpbmNlcnQ6IGNlcnQucGVtCnpvbmVfZG9tYWluOiBhcmdvdHVubmVsdGVzdC5jb20Kem9uZV90YWc6IDQ4Nzk2ZjFlNzBiYjc2NjljMjliYjUxYmEyODJiZjY1
4640
artifacts:
4741
access: 'none'
4842
reports:
4943
dotenv: windows.env
50-
cache: {}
5144

5245
###################################
5346
### Run Windows Component Tests ###
@@ -60,6 +53,9 @@ component-tests-cloudflared-windows:
6053
# We have to decode the secret we encoded on the `load-windows-env-variables` job
6154
- $env:COMPONENT_TESTS_ORIGINCERT = [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($env:COMPONENT_TESTS_ORIGINCERT))
6255
- powershell -ExecutionPolicy Bypass -File ".\.ci\scripts\windows\go-wrapper.ps1" "${GO_VERSION}" ".\.ci\scripts\windows\component-test.ps1"
56+
artifacts:
57+
reports:
58+
junit: report.xml
6359

6460
################################
6561
### Package Windows Binaries ###

0 commit comments

Comments
 (0)