Skip to content

Commit 6dee3a5

Browse files
apiarian-datadogapiarian-datadog
committed
chore: fork and simplify the gitlab pipelines
1 parent 851fd84 commit 6dee3a5

File tree

5 files changed

+395
-6
lines changed

5 files changed

+395
-6
lines changed

.gitlab-ci.yml renamed to .gitlab-ci-old.yml

File renamed without changes.

‎.gitlab/config-old.yaml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# gomplate template generation pipeline
2+
3+
inputFiles:
4+
- .gitlab/templates/bottlecap.yaml.tpl
5+
- .gitlab/templates/go-agent.yaml.tpl
6+
- .gitlab/templates/lambda-extension.yaml.tpl
7+
8+
outputFiles:
9+
- .gitlab/pipeline-bottlecap.yaml
10+
- .gitlab/pipeline-go-agent.yaml
11+
- .gitlab/pipeline-lambda-extension.yaml
12+
13+
datasources:
14+
architectures:
15+
url: .gitlab/datasources/architectures.yaml
16+
17+
environments:
18+
url: .gitlab/datasources/environments.yaml
19+
20+
regions:
21+
url: .gitlab/datasources/regions.yaml

‎.gitlab/config.yaml‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,10 @@
11
# gomplate template generation pipeline
22

33
inputFiles:
4-
- .gitlab/templates/bottlecap.yaml.tpl
5-
- .gitlab/templates/go-agent.yaml.tpl
6-
- .gitlab/templates/lambda-extension.yaml.tpl
4+
- .gitlab/templates/pipeline.yaml.tpl
75

86
outputFiles:
9-
- .gitlab/pipeline-bottlecap.yaml
10-
- .gitlab/pipeline-go-agent.yaml
11-
- .gitlab/pipeline-lambda-extension.yaml
7+
- .gitlab/pipeline.yaml
128

139
datasources:
1410
architectures:

‎.gitlab/templates/pipeline.yaml.tpl‎

Lines changed: 300 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,300 @@
1+
stages:
2+
- build
3+
- test
4+
- sign
5+
- publish
6+
7+
default:
8+
retry:
9+
max: 1
10+
when:
11+
- runner_system_failure
12+
13+
variables:
14+
CI_DOCKER_TARGET_IMAGE: registry.ddbuild.io/ci/datadog-lambda-extension
15+
CI_DOCKER_TARGET_VERSION: latest
16+
17+
{{ range $architecture := (ds "architectures").architectures }}
18+
19+
build go agent ({{ $architecture.name }}):
20+
stage: build
21+
image: registry.ddbuild.io/images/docker:20.10
22+
tags: ["arch:amd64"]
23+
artifacts:
24+
expire_in: 1 hr
25+
paths:
26+
- .layers/datadog_extension-{{ $architecture.name }}.zip
27+
- .layers/datadog_extension-{{ $architecture.name }}/*
28+
variables:
29+
ARCHITECTURE: {{ $architecture.name }}
30+
script:
31+
- cd .. && git clone -b $AGENT_BRANCH --single-branch https://github.com/DataDog/datadog-agent.git && cd datadog-lambda-extension
32+
- .gitlab/scripts/build_go_agent.sh
33+
34+
build bottlecap ({{ $architecture.name }}):
35+
stage: build
36+
image: registry.ddbuild.io/images/docker:20.10
37+
tags: ["arch:amd64"]
38+
needs:
39+
- build go agent ({{ $architecture.name }})
40+
dependencies:
41+
- build go agent ({{ $architecture.name }})
42+
artifacts:
43+
expire_in: 1 hr
44+
paths:
45+
- .layers/datadog_bottlecap-{{ $architecture.name }}.zip
46+
- .layers/datadog_bottlecap-{{ $architecture.name }}/*
47+
variables:
48+
ARCHITECTURE: {{ $architecture.name }}
49+
script:
50+
- .gitlab/scripts/build_bottlecap.sh
51+
52+
# Alpine
53+
build go agent ({{ $architecture.name }}, alpine):
54+
stage: build
55+
image: registry.ddbuild.io/images/docker:20.10
56+
tags: ["arch:amd64"]
57+
artifacts:
58+
expire_in: 1 hr
59+
paths:
60+
- .layers/datadog_extension-{{ $architecture.name }}-alpine.zip
61+
- .layers/datadog_extension-{{ $architecture.name }}-alpine/*
62+
variables:
63+
ARCHITECTURE: {{ $architecture.name }}
64+
ALPINE: 1
65+
script:
66+
- cd .. && git clone -b $AGENT_BRANCH --single-branch https://github.com/DataDog/datadog-agent.git && cd datadog-lambda-extension
67+
- .gitlab/scripts/build_go_agent.sh
68+
69+
build bottlecap ({{ $architecture.name }}, alpine):
70+
stage: build
71+
image: registry.ddbuild.io/images/docker:20.10
72+
tags: ["arch:amd64"]
73+
needs:
74+
- build go agent ({{ $architecture.name }}, alpine)
75+
dependencies:
76+
- build go agent ({{ $architecture.name }}, alpine)
77+
artifacts:
78+
expire_in: 1 hr
79+
paths:
80+
- .layers/datadog_bottlecap-{{ $architecture.name }}-alpine.zip
81+
- .layers/datadog_bottlecap-{{ $architecture.name }}-alpine/*
82+
variables:
83+
ARCHITECTURE: {{ $architecture.name }}
84+
ALPINE: 1
85+
script:
86+
- .gitlab/scripts/build_bottlecap.sh
87+
# Alpine end
88+
89+
check layer size ({{ $architecture.name }}):
90+
stage: test
91+
image: registry.ddbuild.io/images/docker:20.10
92+
tags: ["arch:amd64"]
93+
needs:
94+
- build bottlecap ({{ $architecture.name }})
95+
dependencies:
96+
- build bottlecap ({{ $architecture.name }})
97+
variables:
98+
LAYER_FILE: datadog_bottlecap-{{ $architecture.name }}.zip
99+
script:
100+
- .gitlab/scripts/check_layer_size.sh
101+
102+
fmt ({{ $architecture.name }}):
103+
stage: test
104+
tags: ["arch:{{ $architecture.name }}"]
105+
image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION}
106+
needs: []
107+
script:
108+
- cd bottlecap && cargo fmt
109+
110+
check ({{ $architecture.name }}):
111+
stage: test
112+
tags: ["arch:{{ $architecture.name }}"]
113+
image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION}
114+
needs: []
115+
script:
116+
- cd bottlecap && cargo check
117+
118+
clippy ({{ $architecture.name }}):
119+
stage: test
120+
tags: ["arch:{{ $architecture.name }}"]
121+
image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION}
122+
needs: []
123+
script:
124+
- cd bottlecap && cargo clippy --all-features
125+
126+
{{ range $environment := (ds "environments").environments }}
127+
128+
{{ if or (eq $environment.name "prod") }}
129+
sign layer ({{ $architecture.name }}):
130+
stage: sign
131+
tags: ["arch:amd64"]
132+
image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION}
133+
rules:
134+
- if: '$CI_COMMIT_TAG =~ /^v.*/'
135+
when: manual
136+
needs:
137+
- build bottlecap ({{ $architecture.name }})
138+
- check layer size ({{ $architecture.name }})
139+
- fmt ({{ $architecture.name }})
140+
- check ({{ $architecture.name }})
141+
- clippy ({{ $architecture.name }})
142+
dependencies:
143+
- build bottlecap ({{ $architecture.name }})
144+
artifacts: # Re specify artifacts so the modified signed file is passed
145+
expire_in: 1 day # Signed layers should expire after 1 day
146+
paths:
147+
- .layers/datadog_bottlecap-{{ $architecture.name }}.zip
148+
variables:
149+
LAYER_FILE: datadog_bottlecap-{{ $architecture.name }}.zip
150+
before_script:
151+
- EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh
152+
script:
153+
- .gitlab/scripts/sign_layers.sh {{ $environment.name }}
154+
{{ end }}
155+
156+
publish layer {{ $environment.name }} ({{ $architecture.name }}):
157+
stage: publish
158+
tags: ["arch:amd64"]
159+
image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION}
160+
rules:
161+
- if: '"{{ $environment.name }}" =~ /^(sandbox|staging)/'
162+
when: manual
163+
allow_failure: true
164+
- if: '$CI_COMMIT_TAG =~ /^v.*/'
165+
needs:
166+
{{ if or (eq $environment.name "prod") }}
167+
- sign layer ({{ $architecture.name }})
168+
{{ else }}
169+
- build bottlecap ({{ $architecture.name }})
170+
- check layer size ({{ $architecture.name }})
171+
- fmt ({{ $architecture.name }})
172+
- check ({{ $architecture.name }})
173+
- clippy ({{ $architecture.name }})
174+
{{ end }}
175+
dependencies:
176+
{{ if or (eq $environment.name "prod") }}
177+
- sign layer ({{ $architecture.name }})
178+
{{ else }}
179+
- build bottlecap ({{ $architecture.name }})
180+
{{ end }}
181+
parallel:
182+
matrix:
183+
- REGION: {{ range (ds "regions").regions }}
184+
- {{ .code }}
185+
{{- end}}
186+
variables:
187+
ARCHITECTURE: {{ $architecture.name }}
188+
LAYER_FILE: datadog_bottlecap-{{ $architecture.name }}.zip
189+
STAGE: {{ $environment.name }}
190+
before_script:
191+
- EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh
192+
script:
193+
- .gitlab/scripts/publish_layers.sh
194+
195+
{{ if or (eq $environment.name "sandbox") }}
196+
publish private images:
197+
stage: publish
198+
tags: ["arch:amd64"]
199+
image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION}
200+
needs:
201+
- build bottlecap (arm64)
202+
- build bottlecap (amd64)
203+
when: manual
204+
dependencies:
205+
- build bottlecap (arm64)
206+
- build bottlecap (amd64)
207+
before_script:
208+
- EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh
209+
script:
210+
- .gitlab/scripts/build_private_image.sh
211+
212+
publish private images (alpine):
213+
stage: publish
214+
tags: ["arch:amd64"]
215+
image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION}
216+
needs:
217+
- build bottlecap (arm64, alpine)
218+
- build bottlecap (amd64, alpine)
219+
when: manual
220+
dependencies:
221+
- build bottlecap (arm64, alpine)
222+
- build bottlecap (amd64, alpine)
223+
variables:
224+
ALPINE: 1
225+
before_script:
226+
- EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh
227+
script:
228+
- .gitlab/scripts/build_private_image.sh
229+
{{ end }}
230+
231+
{{- end }} # environments end
232+
233+
{{- end }} # architectures end
234+
235+
build images:
236+
stage: build
237+
tags: ["arch:amd64"]
238+
image: registry.ddbuild.io/images/docker:20.10
239+
rules:
240+
- if: '$CI_COMMIT_TAG =~ /^v.*/'
241+
needs:
242+
- build bottlecap (arm64)
243+
- build bottlecap (amd64)
244+
dependencies:
245+
- build bottlecap (arm64)
246+
- build bottlecap (amd64)
247+
script:
248+
- .gitlab/scripts/build_image.sh
249+
250+
build images (alpine):
251+
stage: build
252+
tags: ["arch:amd64"]
253+
image: registry.ddbuild.io/images/docker:20.10
254+
rules:
255+
- if: '$CI_COMMIT_TAG =~ /^v.*/'
256+
needs:
257+
- build bottlecap (arm64, alpine)
258+
- build bottlecap (amd64, alpine)
259+
dependencies:
260+
- build bottlecap (arm64, alpine)
261+
- build bottlecap (amd64, alpine)
262+
variables:
263+
ALPINE: 1
264+
script:
265+
- .gitlab/scripts/build_image.sh
266+
267+
publish images:
268+
stage: publish
269+
rules:
270+
- if: '$CI_COMMIT_TAG =~ /^v.*/'
271+
needs:
272+
- build images
273+
when: manual
274+
trigger:
275+
project: DataDog/public-images
276+
branch: main
277+
strategy: depend
278+
variables:
279+
IMG_SOURCES: ${CI_DOCKER_TARGET_IMAGE}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}
280+
IMG_DESTINATIONS: lambda-extension:${VERSION},lambda-extension:latest
281+
IMG_REGISTRIES: dockerhub,ecr-public,gcr-datadoghq
282+
IMG_SIGNING: false
283+
284+
publish images (alpine):
285+
stage: publish
286+
rules:
287+
- if: '$CI_COMMIT_TAG =~ /^v.*/'
288+
needs:
289+
- build images (alpine)
290+
when: manual
291+
trigger:
292+
project: DataDog/public-images
293+
branch: main
294+
strategy: depend
295+
variables:
296+
IMG_SOURCES: ${CI_DOCKER_TARGET_IMAGE}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-alpine
297+
IMG_DESTINATIONS: lambda-extension:${VERSION}-alpine,lambda-extension:latest-alpine
298+
IMG_REGISTRIES: dockerhub,ecr-public,gcr-datadoghq
299+
IMG_SIGNING: false
300+

0 commit comments

Comments
 (0)