Skip to content

Commit b2e91c0

Browse files
apiarian-datadogapiarian-datadog
authored
chore: [SVLS-6279] refactor the build pipelines (#552)
Please note that this moves all of the scripts that gitlab runs into the `.gitlab/` directory. This means that the `scripts/` directory files are different from the gitlab ones.
1 parent 851fd84 commit b2e91c0

22 files changed

+726
-648
lines changed

‎.gitlab-ci.yml‎

Lines changed: 14 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,12 @@ variables:
1616
value: ""
1717

1818
stages:
19+
- prepare
1920
- generate
20-
- build
21+
- run
2122

2223
ci image:
23-
stage: build
24+
stage: prepare
2425
image: registry.ddbuild.io/images/docker:20.10
2526
tags: ["arch:arm64"]
2627
rules:
@@ -44,55 +45,29 @@ generator:
4445
cache: *go-cache
4546
artifacts:
4647
paths:
47-
- .gitlab/pipeline-bottlecap.yaml
48-
- .gitlab/pipeline-go-agent.yaml
49-
- .gitlab/pipeline-lambda-extension.yaml
48+
- .gitlab/pipeline.yaml
5049
reports:
5150
dotenv: .env
5251
script:
5352
- if [[ "$CI_COMMIT_TAG" =~ ^v[0-9]+$ ]]; then echo "VERSION=${CI_COMMIT_TAG//[!0-9]/}" >> .env; fi
5453
- apk add --no-cache gomplate
5554
- gomplate --config .gitlab/config.yaml
5655

57-
bottlecap-only:
58-
stage: build
56+
generated pipeline:
57+
stage: run
5958
trigger:
6059
include:
61-
- artifact: .gitlab/pipeline-bottlecap.yaml
62-
job: generator
63-
strategy: depend
64-
rules:
65-
- when: on_success
66-
67-
go-agent-only:
68-
stage: build
69-
trigger:
70-
include:
71-
- artifact: .gitlab/pipeline-go-agent.yaml
72-
job: generator
73-
strategy: depend
74-
rules:
75-
- if: $CI_PIPELINE_SOURCE == "web"
76-
when: manual
77-
variables:
78-
AGENT_BRANCH: $AGENT_BRANCH
79-
LAYER_SUFFIX: $LAYER_SUFFIX
80-
AGENT_VERSION: $AGENT_VERSION
81-
82-
lambda-extension:
83-
stage: build
84-
trigger:
85-
include:
86-
- artifact: .gitlab/pipeline-lambda-extension.yaml
60+
- artifact: .gitlab/pipeline.yaml
8761
job: generator
8862
strategy: depend
63+
forward:
64+
yaml_variables: true
65+
# Without this pipeline_variables: true bit the pipeline we trigger will
66+
# not get the variables that we set as part of our release process in our
67+
# manual pipeline runs.
68+
pipeline_variables: true
8969
needs:
9070
- job: generator
9171
artifacts: true
9272
rules:
93-
- if: $CI_PIPELINE_SOURCE == "web"
94-
variables:
95-
VERSION: $VERSION
96-
AGENT_BRANCH: $AGENT_BRANCH
97-
LAYER_SUFFIX: $LAYER_SUFFIX
98-
AGENT_VERSION: $AGENT_VERSION
73+
- when: on_success

‎.gitlab/config.yaml‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,14 @@
11
# gomplate template generation pipeline
22

33
inputFiles:
4-
- .gitlab/templates/bottlecap.yaml.tpl
5-
- .gitlab/templates/go-agent.yaml.tpl
6-
- .gitlab/templates/lambda-extension.yaml.tpl
4+
- .gitlab/templates/pipeline.yaml.tpl
75

86
outputFiles:
9-
- .gitlab/pipeline-bottlecap.yaml
10-
- .gitlab/pipeline-go-agent.yaml
11-
- .gitlab/pipeline-lambda-extension.yaml
7+
- .gitlab/pipeline.yaml
128

139
datasources:
14-
architectures:
15-
url: .gitlab/datasources/architectures.yaml
10+
flavors:
11+
url: .gitlab/datasources/flavors.yaml
1612

1713
environments:
1814
url: .gitlab/datasources/environments.yaml

‎.gitlab/datasources/architectures.yaml‎

Lines changed: 0 additions & 3 deletions
This file was deleted.

‎.gitlab/datasources/flavors.yaml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
flavors:
2+
- name: amd64
3+
arch: amd64
4+
alpine: 0
5+
needs_code_checks: true
6+
needs_layer_publish: true
7+
suffix: amd64
8+
9+
- name: arm64
10+
arch: arm64
11+
alpine: 0
12+
needs_code_checks: true
13+
needs_layer_publish: true
14+
suffix: arm64
15+
16+
- name: amd64, alpine
17+
arch: amd64
18+
alpine: 1
19+
needs_code_checks: false
20+
needs_layer_publish: false
21+
suffix: amd64-alpine
22+
23+
- name: arm64, alpine
24+
arch: arm64
25+
alpine: 1
26+
needs_code_checks: false
27+
needs_layer_publish: false
28+
suffix: arm64-alpine
29+
30+
# Unfortunately our mutli-arch images don't fit nicely into the flavors
31+
# structure above.
32+
multi_arch_image_flavors:
33+
- name: basic
34+
alpine: 0
35+
platform: linux/amd64,linux/arm64
36+
dependency_names: [amd64, arm64]
37+
suffix: ""
38+
39+
- name: alpine
40+
alpine: 1
41+
platform: linux/amd64,linux/arm64
42+
dependency_names: ["amd64, alpine", "arm64, alpine"]
43+
suffix: "-alpine"

‎.gitlab/scripts/Dockerfile.bottlecap.alpine.compile‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
FROM alpine:3.16 AS compiler
2+
ARG PLATFORM
3+
RUN apk add --no-cache curl gcc musl-dev make unzip bash autoconf automake libtool g++
4+
5+
SHELL ["/bin/bash", "-c"]
6+
7+
# Install Protocol Buffers, from package instead of manually
8+
RUN apk add --no-cache protoc
9+
10+
# Install Rust Toolchain
11+
RUN curl https://sh.rustup.rs -sSf | \
12+
sh -s -- --profile minimal --default-toolchain stable-$PLATFORM-unknown-linux-musl -y
13+
ENV PATH=/root/.cargo/bin:$PATH
14+
RUN rustup component add rust-src --toolchain stable-$PLATFORM-unknown-linux-musl
15+
16+
# Build Bottlecap
17+
RUN mkdir -p /tmp/dd
18+
COPY ./bottlecap/src /tmp/dd/bottlecap/src
19+
COPY ./bottlecap/Cargo.toml /tmp/dd/bottlecap/Cargo.toml
20+
COPY ./bottlecap/Cargo.lock /tmp/dd/bottlecap/Cargo.lock
21+
# Added `-C link-arg=-lgcc` for alpine.
22+
ENV RUSTFLAGS="-C panic=abort -C link-arg=-lgcc"
23+
WORKDIR /tmp/dd/bottlecap
24+
RUN --mount=type=cache,target=/root/.cargo/registry cargo +stable build --release --target $PLATFORM-unknown-linux-musl
25+
RUN cp /tmp/dd/bottlecap/target/$PLATFORM-unknown-linux-musl/release/bottlecap /tmp/dd/bottlecap/bottlecap
26+
27+
# keep the smallest possible docker image
28+
FROM scratch
29+
COPY --from=compiler /tmp/dd/bottlecap/bottlecap /
30+
ENTRYPOINT ["/bottlecap"]
31+

‎.gitlab/scripts/Dockerfile.bottlecap.compile‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
FROM public.ecr.aws/lambda/provided:al2 AS compiler
2+
ARG PLATFORM
3+
RUN yum install -y curl gcc gcc-c++ make unzip
4+
5+
# Install Protocol Buffers compiler by hand, since AL2 does not have a recent enough version.
6+
COPY ./scripts/install-protoc.sh /
7+
RUN chmod +x /install-protoc.sh && /install-protoc.sh
8+
RUN curl https://sh.rustup.rs -sSf | \
9+
sh -s -- --profile minimal --default-toolchain stable-$PLATFORM-unknown-linux-gnu -y
10+
ENV PATH=/root/.cargo/bin:$PATH
11+
RUN rustup component add rust-src --toolchain stable-$PLATFORM-unknown-linux-gnu
12+
RUN mkdir -p /tmp/dd
13+
COPY ./bottlecap/src /tmp/dd/bottlecap/src
14+
COPY ./bottlecap/Cargo.toml /tmp/dd/bottlecap/Cargo.toml
15+
COPY ./bottlecap/Cargo.lock /tmp/dd/bottlecap/Cargo.lock
16+
ENV RUSTFLAGS="-C panic=abort"
17+
WORKDIR /tmp/dd/bottlecap
18+
RUN --mount=type=cache,target=/usr/local/cargo/registry cargo +stable build --release --target $PLATFORM-unknown-linux-gnu
19+
RUN cp /tmp/dd/bottlecap/target/$PLATFORM-unknown-linux-gnu/release/bottlecap /tmp/dd/bottlecap/bottlecap
20+
21+
# keep the smallest possible docker image
22+
FROM scratch
23+
COPY --from=compiler /tmp/dd/bottlecap/bottlecap /
24+
ENTRYPOINT ["/bottlecap"]
25+

‎.gitlab/scripts/Dockerfile.build_layer‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
FROM ubuntu:22.04 as compresser
2+
ARG DATADOG_WRAPPER=datadog_wrapper
3+
ARG SUFFIX
4+
5+
RUN apt-get update
6+
RUN apt-get install -y zip binutils upx
7+
8+
COPY .binaries/datadog-agent-$SUFFIX /datadog-agent-go
9+
RUN strip /datadog-agent-go
10+
RUN upx -1 /datadog-agent-go
11+
12+
RUN mkdir /extensions
13+
WORKDIR /extensions
14+
15+
COPY .binaries/bottlecap-$SUFFIX /extensions/datadog-agent
16+
17+
COPY ./scripts/$DATADOG_WRAPPER /$DATADOG_WRAPPER
18+
RUN chmod +x /$DATADOG_WRAPPER
19+
20+
RUN zip -r datadog_extension.zip /extensions /$DATADOG_WRAPPER /datadog-agent-go
21+
22+
# keep the smallest possible docker image
23+
FROM scratch
24+
COPY --from=compresser /extensions/datadog_extension.zip /
25+
ENTRYPOINT ["/datadog_extension.zip"]
26+

‎.gitlab/scripts/Dockerfile.extension_image‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
FROM scratch
2+
ARG TARGETARCH
3+
COPY .layers/datadog_extension-$TARGETARCH/extensions/datadog-agent opt/extensions/datadog-agent
4+
COPY .layers/datadog_extension-$TARGETARCH/datadog-agent-go opt/datadog-agent-go
5+
COPY --chmod=0755 scripts/datadog_wrapper opt/datadog_wrapper

‎.gitlab/scripts/Dockerfile.extension_image.alpine‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
FROM scratch
2+
ARG TARGETARCH
3+
COPY .layers/datadog_extension-$TARGETARCH-alpine/extensions/datadog-agent opt/extensions/datadog-agent
4+
COPY .layers/datadog_extension-$TARGETARCH-alpine/datadog-agent-go opt/datadog-agent-go
5+
COPY --chmod=0755 scripts/datadog_wrapper opt/datadog_wrapper

‎.gitlab/scripts/Dockerfile.go_agent.alpine.compile‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
FROM alpine:3.16 AS compiler
2+
ARG EXTENSION_VERSION
3+
ARG AGENT_VERSION
4+
ARG BUILD_TAGS
5+
6+
RUN apk add --no-cache git make musl-dev gcc
7+
COPY --from=golang:1.23.6-alpine /usr/local/go/ /usr/lib/go
8+
9+
ENV GOROOT=/usr/lib/go
10+
ENV GOPATH=/go
11+
ENV PATH=/go/bin:$PATH
12+
13+
RUN mkdir -p ${GOPATH}/src ${GOPATH}/bin
14+
RUN mkdir -p /tmp/dd/datadog-agent
15+
16+
# cache dependencies
17+
COPY ./scripts/.cache/go.mod /tmp/dd/datadog-agent
18+
COPY ./scripts/.cache/go.sum /tmp/dd/datadog-agent
19+
WORKDIR /tmp/dd/datadog-agent
20+
21+
# copy source files (/tgz gets unzip automatically by Docker)
22+
ADD ./scripts/.src/datadog-agent.tgz /tmp/dd
23+
24+
# build the extension
25+
WORKDIR /tmp/dd/datadog-agent/cmd/serverless
26+
# add the current version number to the tags package before compilation
27+
28+
RUN --mount=type=cache,target=/go/pkg/mod \
29+
--mount=type=cache,target=/root/.cache/go-build \
30+
if [ -z "$AGENT_VERSION" ]; then \
31+
/usr/lib/go/bin/go build -ldflags="-w -extldflags '-static' \
32+
-X github.com/DataDog/datadog-agent/pkg/serverless/tags.currentExtensionVersion=$EXTENSION_VERSION" \
33+
-tags "${BUILD_TAGS}" -o datadog-agent; \
34+
else \
35+
/usr/lib/go/bin/go build -ldflags="-w -extldflags '-static' \
36+
-X github.com/DataDog/datadog-agent/pkg/serverless/tags.currentExtensionVersion=$EXTENSION_VERSION \
37+
-X github.com/DataDog/datadog-agent/pkg/version.agentVersionDefault=$AGENT_VERSION" \
38+
-tags "${BUILD_TAGS}" -o datadog-agent; \
39+
fi
40+
41+
RUN /usr/lib/go/bin/go tool nm datadog-agent | grep -w 'github.com/DataDog/datadog-agent/pkg/version.agentVersionDefault' || \
42+
(echo "agentVersionDefault variable doesn't exist" && exit 1)
43+
44+
RUN strip datadog-agent
45+
46+
# keep the smallest possible docker image
47+
FROM scratch
48+
COPY --from=compiler /tmp/dd/datadog-agent/cmd/serverless/datadog-agent /
49+
ENTRYPOINT ["/datadog-agent"]
50+

0 commit comments

Comments
 (0)