chore: [SVLS-6279] refactor the build pipelines

.gitlab-ci.yml

@@ -16,11 +16,12 @@ variables:
     value: ""
 
 stages:
+  - prepare
   - generate
-  - build
+  - run
 
 ci image:
-  stage: build
+  stage: prepare
   image: registry.ddbuild.io/images/docker:20.10
   tags: ["arch:arm64"]
   rules:
@@ -44,55 +45,29 @@ generator:
   cache: *go-cache
   artifacts:
     paths:
-      - .gitlab/pipeline-bottlecap.yaml
-      - .gitlab/pipeline-go-agent.yaml
-      - .gitlab/pipeline-lambda-extension.yaml
+      - .gitlab/pipeline.yaml
     reports:
       dotenv: .env
   script:
     - if [[ "$CI_COMMIT_TAG" =~ ^v[0-9]+$ ]]; then echo "VERSION=${CI_COMMIT_TAG//[!0-9]/}" >> .env; fi
     - apk add --no-cache gomplate
     - gomplate --config .gitlab/config.yaml
 
-bottlecap-only:
-  stage: build
+generated pipeline:
+  stage: run
   trigger:
     include:
-      - artifact: .gitlab/pipeline-bottlecap.yaml
-        job: generator
-    strategy: depend
-  rules:
-    - when: on_success
-
-go-agent-only:
-  stage: build
-  trigger:
-    include:
-      - artifact: .gitlab/pipeline-go-agent.yaml
-        job: generator
-    strategy: depend
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "web"
-      when: manual
-  variables:
-    AGENT_BRANCH: $AGENT_BRANCH
-    LAYER_SUFFIX: $LAYER_SUFFIX
-    AGENT_VERSION: $AGENT_VERSION
-
-lambda-extension:
-  stage: build
-  trigger:
-    include:
-      - artifact: .gitlab/pipeline-lambda-extension.yaml
+      - artifact: .gitlab/pipeline.yaml
         job: generator
     strategy: depend
+    forward:
+      yaml_variables: true
+      # Without this pipeline_variables: true bit the pipeline we trigger will
+      # not get the variables that we set as part of our release process in our
+      # manual pipeline runs.
+      pipeline_variables: true
   needs:
     - job: generator
       artifacts: true
   rules:
-    - if: $CI_PIPELINE_SOURCE == "web"
-  variables:
-    VERSION: $VERSION
-    AGENT_BRANCH: $AGENT_BRANCH
-    LAYER_SUFFIX: $LAYER_SUFFIX
-    AGENT_VERSION: $AGENT_VERSION
+    - when: on_success

.gitlab/config.yaml

@@ -1,18 +1,14 @@
 # gomplate template generation pipeline
 
 inputFiles:
-  - .gitlab/templates/bottlecap.yaml.tpl
-  - .gitlab/templates/go-agent.yaml.tpl
-  - .gitlab/templates/lambda-extension.yaml.tpl
+  - .gitlab/templates/pipeline.yaml.tpl
 
 outputFiles:
-  - .gitlab/pipeline-bottlecap.yaml
-  - .gitlab/pipeline-go-agent.yaml
-  - .gitlab/pipeline-lambda-extension.yaml
+  - .gitlab/pipeline.yaml
 
 datasources:
-  architectures:
-    url: .gitlab/datasources/architectures.yaml
+  flavors:
+    url: .gitlab/datasources/flavors.yaml
 
   environments:
     url: .gitlab/datasources/environments.yaml

.gitlab/datasources/flavors.yaml

@@ -0,0 +1,43 @@
+flavors:
+  - name: amd64
+    arch: amd64
+    alpine: 0
+    needs_code_checks: true
+    needs_layer_publish: true
+    suffix: amd64
+
+  - name: arm64
+    arch: arm64
+    alpine: 0
+    needs_code_checks: true
+    needs_layer_publish: true
+    suffix: arm64
+
+  - name: amd64, alpine
+    arch: amd64
+    alpine: 1
+    needs_code_checks: false
+    needs_layer_publish: false
+    suffix: amd64-alpine
+
+  - name: arm64, alpine
+    arch: arm64
+    alpine: 1
+    needs_code_checks: false
+    needs_layer_publish: false
+    suffix: arm64-alpine
+
+# Unfortunately our mutli-arch images don't fit nicely into the flavors
+# structure above.
+multi_arch_image_flavors:
+  - name: basic
+    alpine: 0
+    platform: linux/amd64,linux/arm64
+    dependency_names: [amd64, arm64]
+    suffix: ""
+
+  - name: alpine
+    alpine: 1
+    platform: linux/amd64,linux/arm64
+    dependency_names: ["amd64, alpine", "arm64, alpine"]
+    suffix: "-alpine"

.gitlab/scripts/Dockerfile.bottlecap.alpine.compile

@@ -0,0 +1,31 @@
+FROM alpine:3.16 AS compiler
+ARG PLATFORM
+RUN apk add --no-cache curl gcc musl-dev make unzip bash autoconf automake libtool g++
+
+SHELL ["/bin/bash", "-c"]
+
+# Install Protocol Buffers, from package instead of manually
+RUN apk add --no-cache protoc
+
+# Install Rust Toolchain
+RUN curl https://sh.rustup.rs -sSf | \
+    sh -s -- --profile minimal --default-toolchain stable-$PLATFORM-unknown-linux-musl -y
+ENV PATH=/root/.cargo/bin:$PATH
+RUN rustup component add rust-src --toolchain stable-$PLATFORM-unknown-linux-musl
+
+# Build Bottlecap
+RUN mkdir -p /tmp/dd
+COPY ./bottlecap/src /tmp/dd/bottlecap/src
+COPY ./bottlecap/Cargo.toml /tmp/dd/bottlecap/Cargo.toml
+COPY ./bottlecap/Cargo.lock /tmp/dd/bottlecap/Cargo.lock
+# Added `-C link-arg=-lgcc` for alpine.
+ENV RUSTFLAGS="-C panic=abort -C link-arg=-lgcc"
+WORKDIR /tmp/dd/bottlecap
+RUN --mount=type=cache,target=/root/.cargo/registry cargo +stable build --release --target $PLATFORM-unknown-linux-musl
+RUN cp /tmp/dd/bottlecap/target/$PLATFORM-unknown-linux-musl/release/bottlecap /tmp/dd/bottlecap/bottlecap
+
+# keep the smallest possible docker image
+FROM scratch
+COPY --from=compiler /tmp/dd/bottlecap/bottlecap /
+ENTRYPOINT ["/bottlecap"]
+

.gitlab/scripts/Dockerfile.bottlecap.compile

@@ -0,0 +1,25 @@
+FROM public.ecr.aws/lambda/provided:al2 AS compiler
+ARG PLATFORM
+RUN yum install -y curl gcc gcc-c++ make unzip
+
+# Install Protocol Buffers compiler by hand, since AL2 does not have a recent enough version.
+COPY ./scripts/install-protoc.sh /
+RUN chmod +x /install-protoc.sh && /install-protoc.sh
+RUN curl https://sh.rustup.rs -sSf | \
+    sh -s -- --profile minimal --default-toolchain stable-$PLATFORM-unknown-linux-gnu -y
+ENV PATH=/root/.cargo/bin:$PATH
+RUN rustup component add rust-src --toolchain stable-$PLATFORM-unknown-linux-gnu
+RUN mkdir -p /tmp/dd
+COPY ./bottlecap/src /tmp/dd/bottlecap/src
+COPY ./bottlecap/Cargo.toml /tmp/dd/bottlecap/Cargo.toml
+COPY ./bottlecap/Cargo.lock /tmp/dd/bottlecap/Cargo.lock
+ENV RUSTFLAGS="-C panic=abort"
+WORKDIR /tmp/dd/bottlecap
+RUN --mount=type=cache,target=/usr/local/cargo/registry cargo +stable build --release --target $PLATFORM-unknown-linux-gnu
+RUN cp /tmp/dd/bottlecap/target/$PLATFORM-unknown-linux-gnu/release/bottlecap /tmp/dd/bottlecap/bottlecap
+
+# keep the smallest possible docker image
+FROM scratch
+COPY --from=compiler /tmp/dd/bottlecap/bottlecap /
+ENTRYPOINT ["/bottlecap"]
+

.gitlab/scripts/Dockerfile.build_layer

@@ -0,0 +1,26 @@
+FROM ubuntu:22.04 as compresser
+ARG DATADOG_WRAPPER=datadog_wrapper
+ARG SUFFIX
+
+RUN apt-get update
+RUN apt-get install -y zip binutils upx
+
+COPY .binaries/datadog-agent-$SUFFIX /datadog-agent-go
+RUN strip /datadog-agent-go
+RUN upx -1 /datadog-agent-go
+
+RUN mkdir /extensions
+WORKDIR /extensions
+
+COPY .binaries/bottlecap-$SUFFIX /extensions/datadog-agent
+
+COPY ./scripts/$DATADOG_WRAPPER /$DATADOG_WRAPPER
+RUN chmod +x /$DATADOG_WRAPPER
+
+RUN  zip -r datadog_extension.zip /extensions /$DATADOG_WRAPPER /datadog-agent-go
+
+# keep the smallest possible docker image
+FROM scratch
+COPY --from=compresser /extensions/datadog_extension.zip /
+ENTRYPOINT ["/datadog_extension.zip"]
+

.gitlab/scripts/Dockerfile.extension_image

@@ -0,0 +1,5 @@
+FROM scratch
+ARG TARGETARCH
+COPY .layers/datadog_extension-$TARGETARCH/extensions/datadog-agent opt/extensions/datadog-agent
+COPY .layers/datadog_extension-$TARGETARCH/datadog-agent-go opt/datadog-agent-go
+COPY --chmod=0755 scripts/datadog_wrapper opt/datadog_wrapper

.gitlab/scripts/Dockerfile.extension_image.alpine

@@ -0,0 +1,5 @@
+FROM scratch
+ARG TARGETARCH
+COPY .layers/datadog_extension-$TARGETARCH-alpine/extensions/datadog-agent opt/extensions/datadog-agent
+COPY .layers/datadog_extension-$TARGETARCH-alpine/datadog-agent-go opt/datadog-agent-go
+COPY --chmod=0755 scripts/datadog_wrapper opt/datadog_wrapper

.gitlab/scripts/Dockerfile.go_agent.alpine.compile

@@ -0,0 +1,50 @@
+FROM alpine:3.16 AS compiler
+ARG EXTENSION_VERSION
+ARG AGENT_VERSION
+ARG BUILD_TAGS
+
+RUN apk add --no-cache git make musl-dev gcc
+COPY --from=golang:1.23.6-alpine /usr/local/go/ /usr/lib/go
+
+ENV GOROOT=/usr/lib/go
+ENV GOPATH=/go
+ENV PATH=/go/bin:$PATH
+
+RUN mkdir -p ${GOPATH}/src ${GOPATH}/bin
+RUN mkdir -p /tmp/dd/datadog-agent
+
+# cache dependencies
+COPY ./scripts/.cache/go.mod /tmp/dd/datadog-agent
+COPY ./scripts/.cache/go.sum /tmp/dd/datadog-agent
+WORKDIR /tmp/dd/datadog-agent
+
+# copy source files (/tgz gets unzip automatically by Docker)
+ADD ./scripts/.src/datadog-agent.tgz /tmp/dd
+
+# build the extension
+WORKDIR /tmp/dd/datadog-agent/cmd/serverless
+# add the current version number to the tags package before compilation
+
+RUN --mount=type=cache,target=/go/pkg/mod \
+    --mount=type=cache,target=/root/.cache/go-build \
+    if [ -z "$AGENT_VERSION" ]; then \
+        /usr/lib/go/bin/go build -ldflags="-w -extldflags '-static' \
+        -X github.com/DataDog/datadog-agent/pkg/serverless/tags.currentExtensionVersion=$EXTENSION_VERSION" \
+        -tags "${BUILD_TAGS}" -o datadog-agent; \
+    else \
+        /usr/lib/go/bin/go build -ldflags="-w  -extldflags '-static' \
+        -X github.com/DataDog/datadog-agent/pkg/serverless/tags.currentExtensionVersion=$EXTENSION_VERSION \
+        -X github.com/DataDog/datadog-agent/pkg/version.agentVersionDefault=$AGENT_VERSION" \
+        -tags "${BUILD_TAGS}" -o datadog-agent; \
+    fi
+
+RUN /usr/lib/go/bin/go tool nm datadog-agent | grep -w 'github.com/DataDog/datadog-agent/pkg/version.agentVersionDefault' || \
+    (echo "agentVersionDefault variable doesn't exist" && exit 1)
+
+RUN strip datadog-agent
+
+# keep the smallest possible docker image
+FROM scratch
+COPY --from=compiler /tmp/dd/datadog-agent/cmd/serverless/datadog-agent /
+ENTRYPOINT ["/datadog-agent"]
+

.gitlab/scripts/Dockerfile.go_agent.compile

@@ -0,0 +1,52 @@
+FROM public.ecr.aws/lambda/provided:al2 AS compiler
+ARG EXTENSION_VERSION
+ARG AGENT_VERSION
+ARG BUILD_TAGS
+RUN mkdir -p /tmp/dd/datadog-agent
+
+RUN yum install -y wget tar gzip gcc
+RUN arch="$(uname -m)"; \
+    if [ "${arch}" = 'aarch64' ]; then \
+    arch='arm64'; \
+    fi; \
+    if [ "${arch}" = 'x86_64' ]; then \
+    arch='amd64'; \
+    fi; \
+    wget -O go1.23.6.linux-${arch}.tar.gz https://go.dev/dl/go1.23.6.linux-${arch}.tar.gz; \
+    tar -C /usr/local -xzf go1.23.6.linux-${arch}.tar.gz
+
+# cache dependencies
+COPY ./scripts/.cache/go.mod /tmp/dd/datadog-agent
+COPY ./scripts/.cache/go.sum /tmp/dd/datadog-agent
+WORKDIR /tmp/dd/datadog-agent
+
+# copy source files (/tgz gets unzip automatically by Docker)
+ADD ./scripts/.src/datadog-agent.tgz /tmp/dd
+
+# build the extension
+WORKDIR /tmp/dd/datadog-agent/cmd/serverless
+# add the current version number to the tags package before compilation
+
+RUN --mount=type=cache,target=/root/go/pkg/mod \
+    --mount=type=cache,target=/root/.cache/go-build \
+    if [ -z "$AGENT_VERSION" ]; then \
+        /usr/local/go/bin/go build -ldflags="-w \
+        -X github.com/DataDog/datadog-agent/pkg/serverless/tags.currentExtensionVersion=$EXTENSION_VERSION" \
+        -tags "${BUILD_TAGS}" -o datadog-agent; \
+    else \
+        /usr/local/go/bin/go build -ldflags="-w \
+        -X github.com/DataDog/datadog-agent/pkg/serverless/tags.currentExtensionVersion=$EXTENSION_VERSION \
+        -X github.com/DataDog/datadog-agent/pkg/version.agentVersionDefault=$AGENT_VERSION" \
+        -tags "${BUILD_TAGS}" -o datadog-agent; \
+    fi
+
+RUN /usr/local/go/bin/go tool nm datadog-agent | grep -w 'github.com/DataDog/datadog-agent/pkg/version.agentVersionDefault' || \
+    (echo "agentVersionDefault variable doesn't exist" && exit 1)
+
+RUN strip datadog-agent
+
+# keep the smallest possible docker imag
+FROM scratch
+COPY --from=compiler /tmp/dd/datadog-agent/cmd/serverless/datadog-agent /
+ENTRYPOINT ["/datadog-agent"]
+