diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 204dbc94b..b06dbe7f8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,11 +16,12 @@ variables: value: "" stages: + - prepare - generate - - build + - run ci image: - stage: build + stage: prepare image: registry.ddbuild.io/images/docker:20.10 tags: ["arch:arm64"] rules: @@ -44,9 +45,7 @@ generator: cache: *go-cache artifacts: paths: - - .gitlab/pipeline-bottlecap.yaml - - .gitlab/pipeline-go-agent.yaml - - .gitlab/pipeline-lambda-extension.yaml + - .gitlab/pipeline.yaml reports: dotenv: .env script: @@ -54,45 +53,21 @@ generator: - apk add --no-cache gomplate - gomplate --config .gitlab/config.yaml -bottlecap-only: - stage: build +generated pipeline: + stage: run trigger: include: - - artifact: .gitlab/pipeline-bottlecap.yaml - job: generator - strategy: depend - rules: - - when: on_success - -go-agent-only: - stage: build - trigger: - include: - - artifact: .gitlab/pipeline-go-agent.yaml - job: generator - strategy: depend - rules: - - if: $CI_PIPELINE_SOURCE == "web" - when: manual - variables: - AGENT_BRANCH: $AGENT_BRANCH - LAYER_SUFFIX: $LAYER_SUFFIX - AGENT_VERSION: $AGENT_VERSION - -lambda-extension: - stage: build - trigger: - include: - - artifact: .gitlab/pipeline-lambda-extension.yaml + - artifact: .gitlab/pipeline.yaml job: generator strategy: depend + forward: + yaml_variables: true + # Without this pipeline_variables: true bit the pipeline we trigger will + # not get the variables that we set as part of our release process in our + # manual pipeline runs. + pipeline_variables: true needs: - job: generator artifacts: true rules: - - if: $CI_PIPELINE_SOURCE == "web" - variables: - VERSION: $VERSION - AGENT_BRANCH: $AGENT_BRANCH - LAYER_SUFFIX: $LAYER_SUFFIX - AGENT_VERSION: $AGENT_VERSION + - when: on_success diff --git a/.gitlab/config.yaml b/.gitlab/config.yaml index 3e3899556..08b7531b0 100644 --- a/.gitlab/config.yaml +++ b/.gitlab/config.yaml @@ -1,18 +1,14 @@ # gomplate template generation pipeline inputFiles: - - .gitlab/templates/bottlecap.yaml.tpl - - .gitlab/templates/go-agent.yaml.tpl - - .gitlab/templates/lambda-extension.yaml.tpl + - .gitlab/templates/pipeline.yaml.tpl outputFiles: - - .gitlab/pipeline-bottlecap.yaml - - .gitlab/pipeline-go-agent.yaml - - .gitlab/pipeline-lambda-extension.yaml + - .gitlab/pipeline.yaml datasources: - architectures: - url: .gitlab/datasources/architectures.yaml + flavors: + url: .gitlab/datasources/flavors.yaml environments: url: .gitlab/datasources/environments.yaml diff --git a/.gitlab/datasources/architectures.yaml b/.gitlab/datasources/architectures.yaml deleted file mode 100644 index 4b9b1a60b..000000000 --- a/.gitlab/datasources/architectures.yaml +++ /dev/null @@ -1,3 +0,0 @@ -architectures: - - name: amd64 - - name: arm64 diff --git a/.gitlab/datasources/flavors.yaml b/.gitlab/datasources/flavors.yaml new file mode 100644 index 000000000..1f53891a9 --- /dev/null +++ b/.gitlab/datasources/flavors.yaml @@ -0,0 +1,43 @@ +flavors: + - name: amd64 + arch: amd64 + alpine: 0 + needs_code_checks: true + needs_layer_publish: true + suffix: amd64 + + - name: arm64 + arch: arm64 + alpine: 0 + needs_code_checks: true + needs_layer_publish: true + suffix: arm64 + + - name: amd64, alpine + arch: amd64 + alpine: 1 + needs_code_checks: false + needs_layer_publish: false + suffix: amd64-alpine + + - name: arm64, alpine + arch: arm64 + alpine: 1 + needs_code_checks: false + needs_layer_publish: false + suffix: arm64-alpine + +# Unfortunately our mutli-arch images don't fit nicely into the flavors +# structure above. +multi_arch_image_flavors: + - name: basic + alpine: 0 + platform: linux/amd64,linux/arm64 + dependency_names: [amd64, arm64] + suffix: "" + + - name: alpine + alpine: 1 + platform: linux/amd64,linux/arm64 + dependency_names: ["amd64, alpine", "arm64, alpine"] + suffix: "-alpine" diff --git a/.gitlab/scripts/Dockerfile.bottlecap.alpine.compile b/.gitlab/scripts/Dockerfile.bottlecap.alpine.compile new file mode 100644 index 000000000..404b488c7 --- /dev/null +++ b/.gitlab/scripts/Dockerfile.bottlecap.alpine.compile @@ -0,0 +1,31 @@ +FROM alpine:3.16 AS compiler +ARG PLATFORM +RUN apk add --no-cache curl gcc musl-dev make unzip bash autoconf automake libtool g++ + +SHELL ["/bin/bash", "-c"] + +# Install Protocol Buffers, from package instead of manually +RUN apk add --no-cache protoc + +# Install Rust Toolchain +RUN curl https://sh.rustup.rs -sSf | \ + sh -s -- --profile minimal --default-toolchain stable-$PLATFORM-unknown-linux-musl -y +ENV PATH=/root/.cargo/bin:$PATH +RUN rustup component add rust-src --toolchain stable-$PLATFORM-unknown-linux-musl + +# Build Bottlecap +RUN mkdir -p /tmp/dd +COPY ./bottlecap/src /tmp/dd/bottlecap/src +COPY ./bottlecap/Cargo.toml /tmp/dd/bottlecap/Cargo.toml +COPY ./bottlecap/Cargo.lock /tmp/dd/bottlecap/Cargo.lock +# Added `-C link-arg=-lgcc` for alpine. +ENV RUSTFLAGS="-C panic=abort -C link-arg=-lgcc" +WORKDIR /tmp/dd/bottlecap +RUN --mount=type=cache,target=/root/.cargo/registry cargo +stable build --release --target $PLATFORM-unknown-linux-musl +RUN cp /tmp/dd/bottlecap/target/$PLATFORM-unknown-linux-musl/release/bottlecap /tmp/dd/bottlecap/bottlecap + +# keep the smallest possible docker image +FROM scratch +COPY --from=compiler /tmp/dd/bottlecap/bottlecap / +ENTRYPOINT ["/bottlecap"] + diff --git a/.gitlab/scripts/Dockerfile.bottlecap.compile b/.gitlab/scripts/Dockerfile.bottlecap.compile new file mode 100644 index 000000000..881e665b1 --- /dev/null +++ b/.gitlab/scripts/Dockerfile.bottlecap.compile @@ -0,0 +1,25 @@ +FROM public.ecr.aws/lambda/provided:al2 AS compiler +ARG PLATFORM +RUN yum install -y curl gcc gcc-c++ make unzip + +# Install Protocol Buffers compiler by hand, since AL2 does not have a recent enough version. +COPY ./scripts/install-protoc.sh / +RUN chmod +x /install-protoc.sh && /install-protoc.sh +RUN curl https://sh.rustup.rs -sSf | \ + sh -s -- --profile minimal --default-toolchain stable-$PLATFORM-unknown-linux-gnu -y +ENV PATH=/root/.cargo/bin:$PATH +RUN rustup component add rust-src --toolchain stable-$PLATFORM-unknown-linux-gnu +RUN mkdir -p /tmp/dd +COPY ./bottlecap/src /tmp/dd/bottlecap/src +COPY ./bottlecap/Cargo.toml /tmp/dd/bottlecap/Cargo.toml +COPY ./bottlecap/Cargo.lock /tmp/dd/bottlecap/Cargo.lock +ENV RUSTFLAGS="-C panic=abort" +WORKDIR /tmp/dd/bottlecap +RUN --mount=type=cache,target=/usr/local/cargo/registry cargo +stable build --release --target $PLATFORM-unknown-linux-gnu +RUN cp /tmp/dd/bottlecap/target/$PLATFORM-unknown-linux-gnu/release/bottlecap /tmp/dd/bottlecap/bottlecap + +# keep the smallest possible docker image +FROM scratch +COPY --from=compiler /tmp/dd/bottlecap/bottlecap / +ENTRYPOINT ["/bottlecap"] + diff --git a/.gitlab/scripts/Dockerfile.build_layer b/.gitlab/scripts/Dockerfile.build_layer new file mode 100644 index 000000000..238e78f0a --- /dev/null +++ b/.gitlab/scripts/Dockerfile.build_layer @@ -0,0 +1,26 @@ +FROM ubuntu:22.04 as compresser +ARG DATADOG_WRAPPER=datadog_wrapper +ARG SUFFIX + +RUN apt-get update +RUN apt-get install -y zip binutils upx + +COPY .binaries/datadog-agent-$SUFFIX /datadog-agent-go +RUN strip /datadog-agent-go +RUN upx -1 /datadog-agent-go + +RUN mkdir /extensions +WORKDIR /extensions + +COPY .binaries/bottlecap-$SUFFIX /extensions/datadog-agent + +COPY ./scripts/$DATADOG_WRAPPER /$DATADOG_WRAPPER +RUN chmod +x /$DATADOG_WRAPPER + +RUN zip -r datadog_extension.zip /extensions /$DATADOG_WRAPPER /datadog-agent-go + +# keep the smallest possible docker image +FROM scratch +COPY --from=compresser /extensions/datadog_extension.zip / +ENTRYPOINT ["/datadog_extension.zip"] + diff --git a/.gitlab/scripts/Dockerfile.extension_image b/.gitlab/scripts/Dockerfile.extension_image new file mode 100644 index 000000000..0afe9196d --- /dev/null +++ b/.gitlab/scripts/Dockerfile.extension_image @@ -0,0 +1,5 @@ +FROM scratch +ARG TARGETARCH +COPY .layers/datadog_extension-$TARGETARCH/extensions/datadog-agent opt/extensions/datadog-agent +COPY .layers/datadog_extension-$TARGETARCH/datadog-agent-go opt/datadog-agent-go +COPY --chmod=0755 scripts/datadog_wrapper opt/datadog_wrapper diff --git a/.gitlab/scripts/Dockerfile.extension_image.alpine b/.gitlab/scripts/Dockerfile.extension_image.alpine new file mode 100644 index 000000000..822900167 --- /dev/null +++ b/.gitlab/scripts/Dockerfile.extension_image.alpine @@ -0,0 +1,5 @@ +FROM scratch +ARG TARGETARCH +COPY .layers/datadog_extension-$TARGETARCH-alpine/extensions/datadog-agent opt/extensions/datadog-agent +COPY .layers/datadog_extension-$TARGETARCH-alpine/datadog-agent-go opt/datadog-agent-go +COPY --chmod=0755 scripts/datadog_wrapper opt/datadog_wrapper diff --git a/.gitlab/scripts/Dockerfile.go_agent.alpine.compile b/.gitlab/scripts/Dockerfile.go_agent.alpine.compile new file mode 100644 index 000000000..16a38932c --- /dev/null +++ b/.gitlab/scripts/Dockerfile.go_agent.alpine.compile @@ -0,0 +1,50 @@ +FROM alpine:3.16 AS compiler +ARG EXTENSION_VERSION +ARG AGENT_VERSION +ARG BUILD_TAGS + +RUN apk add --no-cache git make musl-dev gcc +COPY --from=golang:1.23.6-alpine /usr/local/go/ /usr/lib/go + +ENV GOROOT=/usr/lib/go +ENV GOPATH=/go +ENV PATH=/go/bin:$PATH + +RUN mkdir -p ${GOPATH}/src ${GOPATH}/bin +RUN mkdir -p /tmp/dd/datadog-agent + +# cache dependencies +COPY ./scripts/.cache/go.mod /tmp/dd/datadog-agent +COPY ./scripts/.cache/go.sum /tmp/dd/datadog-agent +WORKDIR /tmp/dd/datadog-agent + +# copy source files (/tgz gets unzip automatically by Docker) +ADD ./scripts/.src/datadog-agent.tgz /tmp/dd + +# build the extension +WORKDIR /tmp/dd/datadog-agent/cmd/serverless +# add the current version number to the tags package before compilation + +RUN --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/root/.cache/go-build \ + if [ -z "$AGENT_VERSION" ]; then \ + /usr/lib/go/bin/go build -ldflags="-w -extldflags '-static' \ + -X github.com/DataDog/datadog-agent/pkg/serverless/tags.currentExtensionVersion=$EXTENSION_VERSION" \ + -tags "${BUILD_TAGS}" -o datadog-agent; \ + else \ + /usr/lib/go/bin/go build -ldflags="-w -extldflags '-static' \ + -X github.com/DataDog/datadog-agent/pkg/serverless/tags.currentExtensionVersion=$EXTENSION_VERSION \ + -X github.com/DataDog/datadog-agent/pkg/version.agentVersionDefault=$AGENT_VERSION" \ + -tags "${BUILD_TAGS}" -o datadog-agent; \ + fi + +RUN /usr/lib/go/bin/go tool nm datadog-agent | grep -w 'github.com/DataDog/datadog-agent/pkg/version.agentVersionDefault' || \ + (echo "agentVersionDefault variable doesn't exist" && exit 1) + +RUN strip datadog-agent + +# keep the smallest possible docker image +FROM scratch +COPY --from=compiler /tmp/dd/datadog-agent/cmd/serverless/datadog-agent / +ENTRYPOINT ["/datadog-agent"] + diff --git a/.gitlab/scripts/Dockerfile.go_agent.compile b/.gitlab/scripts/Dockerfile.go_agent.compile new file mode 100644 index 000000000..5a7aa3392 --- /dev/null +++ b/.gitlab/scripts/Dockerfile.go_agent.compile @@ -0,0 +1,52 @@ +FROM public.ecr.aws/lambda/provided:al2 AS compiler +ARG EXTENSION_VERSION +ARG AGENT_VERSION +ARG BUILD_TAGS +RUN mkdir -p /tmp/dd/datadog-agent + +RUN yum install -y wget tar gzip gcc +RUN arch="$(uname -m)"; \ + if [ "${arch}" = 'aarch64' ]; then \ + arch='arm64'; \ + fi; \ + if [ "${arch}" = 'x86_64' ]; then \ + arch='amd64'; \ + fi; \ + wget -O go1.23.6.linux-${arch}.tar.gz https://go.dev/dl/go1.23.6.linux-${arch}.tar.gz; \ + tar -C /usr/local -xzf go1.23.6.linux-${arch}.tar.gz + +# cache dependencies +COPY ./scripts/.cache/go.mod /tmp/dd/datadog-agent +COPY ./scripts/.cache/go.sum /tmp/dd/datadog-agent +WORKDIR /tmp/dd/datadog-agent + +# copy source files (/tgz gets unzip automatically by Docker) +ADD ./scripts/.src/datadog-agent.tgz /tmp/dd + +# build the extension +WORKDIR /tmp/dd/datadog-agent/cmd/serverless +# add the current version number to the tags package before compilation + +RUN --mount=type=cache,target=/root/go/pkg/mod \ + --mount=type=cache,target=/root/.cache/go-build \ + if [ -z "$AGENT_VERSION" ]; then \ + /usr/local/go/bin/go build -ldflags="-w \ + -X github.com/DataDog/datadog-agent/pkg/serverless/tags.currentExtensionVersion=$EXTENSION_VERSION" \ + -tags "${BUILD_TAGS}" -o datadog-agent; \ + else \ + /usr/local/go/bin/go build -ldflags="-w \ + -X github.com/DataDog/datadog-agent/pkg/serverless/tags.currentExtensionVersion=$EXTENSION_VERSION \ + -X github.com/DataDog/datadog-agent/pkg/version.agentVersionDefault=$AGENT_VERSION" \ + -tags "${BUILD_TAGS}" -o datadog-agent; \ + fi + +RUN /usr/local/go/bin/go tool nm datadog-agent | grep -w 'github.com/DataDog/datadog-agent/pkg/version.agentVersionDefault' || \ + (echo "agentVersionDefault variable doesn't exist" && exit 1) + +RUN strip datadog-agent + +# keep the smallest possible docker imag +FROM scratch +COPY --from=compiler /tmp/dd/datadog-agent/cmd/serverless/datadog-agent / +ENTRYPOINT ["/datadog-agent"] + diff --git a/.gitlab/scripts/build_bottlecap.sh b/.gitlab/scripts/build_bottlecap.sh deleted file mode 100755 index f59d33ccf..000000000 --- a/.gitlab/scripts/build_bottlecap.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash - -# Unless explicitly stated otherwise all files in this repository are licensed -# under the Apache License Version 2.0. -# This product includes software developed at Datadog (https://www.datadoghq.com/). -# Copyright 2024 Datadog, Inc. - -set -e - -if [ -z "$ARCHITECTURE" ]; then - printf "[ERROR]: ARCHITECTURE not specified\n" - exit 1 -fi - -if [ -z "$ALPINE" ]; then - printf "Building bottlecap" - BUILD_FILE=Dockerfile.bottlecap.build -else - echo "Building bottlecap for alpine" - BUILD_FILE=Dockerfile.bottlecap.alpine.build - BUILD_SUFFIX="-alpine" -fi - -prepare_folders() { - # Move into the root directory, so this script can be called from any directory - SCRIPTS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" - ROOT_DIR=$SCRIPTS_DIR/../.. - cd $ROOT_DIR - - echo $ROOT_DIR - - EXTENSION_DIR=".layers" - TARGET_DIR=$(pwd)/$EXTENSION_DIR - - rm -rf $EXTENSION_DIR/datadog_bottlecap-${ARCHITECTURE}${BUILD_SUFFIX} 2>/dev/null - rm -rf $EXTENSION_DIR/datadog_bottlecap-${ARCHITECTURE}${BUILD_SUFFIX}.zip 2>/dev/null - - cd $ROOT_DIR -} - - -docker_build() { - local arch=$1 - local file=$2 - if [ "$arch" == "amd64" ]; then - PLATFORM="x86_64" - else - PLATFORM="aarch64" - fi - - docker buildx build --platform linux/${arch} \ - -t datadog/build-bottlecap-${arch} \ - -f ./scripts/${file} \ - --build-arg PLATFORM=$PLATFORM \ - --build-arg GO_AGENT_PATH="datadog_extension-${arch}${BUILD_SUFFIX}" \ - . -o $TARGET_DIR/datadog_bottlecap-${arch}${BUILD_SUFFIX} - - cp $TARGET_DIR/datadog_bottlecap-${arch}${BUILD_SUFFIX}/datadog_extension.zip $TARGET_DIR/datadog_bottlecap-${arch}${BUILD_SUFFIX}.zip - - unzip $TARGET_DIR/datadog_bottlecap-${arch}${BUILD_SUFFIX}/datadog_extension.zip -d $TARGET_DIR/datadog_bottlecap-${arch}${BUILD_SUFFIX} - rm -rf $TARGET_DIR/datadog_bottlecap-${arch}${BUILD_SUFFIX}/datadog_extension.zip - rm -rf $TARGET_DIR/datadog_extension-${arch}${BUILD_SUFFIX} - rm -rf $TARGET_DIR/datadog_extension-${arch}${BUILD_SUFFIX}.zip -} - -prepare_folders -docker_build $ARCHITECTURE $BUILD_FILE diff --git a/.gitlab/scripts/build_image.sh b/.gitlab/scripts/build_image.sh index f24fa71b0..4e6122018 100755 --- a/.gitlab/scripts/build_image.sh +++ b/.gitlab/scripts/build_image.sh @@ -12,6 +12,13 @@ set -e DOCKER_TARGET_IMAGE="registry.ddbuild.io/ci/datadog-lambda-extension" EXTENSION_DIR=".layers" +if [ -z "$ALPINE" ]; then + printf "[ERROR]: ALPINE not specified\n" + exit 1 +else + printf "Alpine build requested: ${ALPINE}\n" +fi + if [ -z "$CI_COMMIT_TAG" ]; then # Running on dev printf "Running on dev environment\n" @@ -23,19 +30,17 @@ else fi -if [ -z "$ALPINE" ]; then +if [ "$ALPINE" = "0" ]; then printf "Building image\n" - TARGET_IMAGE="Dockerfile" + TARGET_IMAGE="Dockerfile.extension_image" else printf "Building image for alpine\n" - TARGET_IMAGE="Dockerfile.alpine" - BUILD_SUFFIX="-alpine" + TARGET_IMAGE="Dockerfile.extension_image.alpine" fi docker buildx build \ - --platform linux/amd64,linux/arm64 \ - -f ./scripts/${TARGET_IMAGE} \ - --tag "$DOCKER_TARGET_IMAGE:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}${BUILD_SUFFIX}" \ + --platform $PLATFORM \ + -f .gitlab/scripts/${TARGET_IMAGE} \ + --tag "$DOCKER_TARGET_IMAGE:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}${SUFFIX}" \ --push . - -printf "Image built and pushed to $DOCKER_TARGET_IMAGE:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}${BUILD_SUFFIX} for arm64 and amd64\n" +printf "Image built and pushed to $DOCKER_TARGET_IMAGE:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}${SUFFIX} for $PLATFORM\n" diff --git a/.gitlab/scripts/build_layer.sh b/.gitlab/scripts/build_layer.sh new file mode 100755 index 000000000..f4cdf9d13 --- /dev/null +++ b/.gitlab/scripts/build_layer.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# Unless explicitly stated otherwise all files in this repository are licensed +# under the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2024 Datadog, Inc. + +set -e + +if [ -z "$ARCHITECTURE" ]; then + printf "[ERROR]: ARCHITECTURE not specified\n" + exit 1 +fi + +prepare_folders() { + # Move into the root directory, so this script can be called from any directory + SCRIPTS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" + ROOT_DIR=$SCRIPTS_DIR/../.. + cd $ROOT_DIR + + echo $ROOT_DIR + + EXTENSION_DIR=".layers" + TARGET_DIR=$(pwd)/$EXTENSION_DIR + + rm -rf ${EXTENSION_DIR} 2>/dev/null + mkdir -p $EXTENSION_DIR + + cd $ROOT_DIR +} + + +docker_build() { + local arch=$1 + + docker buildx build --platform linux/${arch} \ + -t datadog/build-extension-${SUFFIX} \ + -f .gitlab/scripts/Dockerfile.build_layer \ + --build-arg SUFFIX=$SUFFIX \ + . -o $TARGET_DIR/datadog-extension-${SUFFIX} + + cp $TARGET_DIR/datadog-extension-${SUFFIX}/datadog_extension.zip $TARGET_DIR/datadog_extension-${SUFFIX}.zip + unzip $TARGET_DIR/datadog-extension-${SUFFIX}/datadog_extension.zip -d $TARGET_DIR/datadog_extension-${SUFFIX} + rm -rf $TARGET_DIR/datadog-extension-${SUFFIX}/ +} + +prepare_folders +docker_build $ARCHITECTURE diff --git a/.gitlab/scripts/build_private_image.sh b/.gitlab/scripts/build_private_image.sh index 00ac83b63..9dbee7d69 100755 --- a/.gitlab/scripts/build_private_image.sh +++ b/.gitlab/scripts/build_private_image.sh @@ -11,24 +11,30 @@ DOCKER_TARGET_IMAGE="425362996713.dkr.ecr.us-east-1.amazonaws.com/self-monitorin EXTENSION_DIR=".layers" IMAGE_TAG="latest" +if [ -z "$ALPINE" ]; then + printf "[ERROR]: ALPINE not specified\n" + exit 1 +else + printf "Alpine build requested: ${ALPINE}\n" +fi + printf "Authenticating Docker to ECR...\n" aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 425362996713.dkr.ecr.us-east-1.amazonaws.com -if [ -z "$ALPINE" ]; then +if [ "$ALPINE" = "0" ]; then printf "Building image\n" - TARGET_IMAGE="Dockerfile" + TARGET_IMAGE="Dockerfile.extension_image" else printf "Building image for alpine\n" - TARGET_IMAGE="Dockerfile.alpine" - BUILD_SUFFIX="-alpine" + TARGET_IMAGE="Dockerfile.extension_image.alpine" fi -LAYER_NAME="Datadog-Extension$BUILD_SUFFIX" +LAYER_NAME="Datadog-Extension$SUFFIX" if [ -z "$LAYER_SUFFIX" ]; then printf "Building container images tagged without suffix\n" else printf "Building container images tagged with suffix: ${LAYER_SUFFIX}\n" - LAYER_NAME="${LAYER_NAME}-${LAYER_SUFFIX}${BUILD_SUFFIX}" + LAYER_NAME="${LAYER_NAME}-${LAYER_SUFFIX}${SUFFIX}" fi # Increment last version @@ -37,10 +43,10 @@ VERSION=$(($latest_version + 1)) printf "Tagging container image with version: $VERSION and latest\n" docker buildx build \ - --platform linux/amd64,linux/arm64 \ - -f ./scripts/${TARGET_IMAGE} \ - --tag "$DOCKER_TARGET_IMAGE:${IMAGE_TAG}${BUILD_SUFFIX}" \ - --tag "$DOCKER_TARGET_IMAGE:${VERSION}${BUILD_SUFFIX}" \ + --platform $PLATFORM \ + -f .gitlab/scripts/${TARGET_IMAGE} \ + --tag "$DOCKER_TARGET_IMAGE:${IMAGE_TAG}${SUFFIX}" \ + --tag "$DOCKER_TARGET_IMAGE:${VERSION}${SUFFIX}" \ --push . -printf "Image built and pushed to $DOCKER_TARGET_IMAGE:${IMAGE_TAG}${BUILD_SUFFIX} for arm64 and amd64\n" +printf "Image built and pushed to $DOCKER_TARGET_IMAGE:${IMAGE_TAG}${SUFFIX} for arm64 and amd64\n" diff --git a/.gitlab/scripts/compile_bottlecap.sh b/.gitlab/scripts/compile_bottlecap.sh new file mode 100755 index 000000000..2d694c0ab --- /dev/null +++ b/.gitlab/scripts/compile_bottlecap.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +# Unless explicitly stated otherwise all files in this repository are licensed +# under the Apache License Version 2.0. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2024 Datadog, Inc. + +set -e + +if [ -z "$ARCHITECTURE" ]; then + printf "[ERROR]: ARCHITECTURE not specified\n" + exit 1 +fi + +if [ -z "$ALPINE" ]; then + printf "[ERROR]: ALPINE not specified\n" + exit 1 +else + printf "Alpine compile requested: ${ALPINE}\n" +fi + + +if [ "$ALPINE" = "0" ]; then + COMPILE_FILE=Dockerfile.bottlecap.compile +else + printf "Compiling for alpine\n" + COMPILE_FILE=Dockerfile.bottlecap.alpine.compile +fi + +prepare_folders() { + # Move into the root directory, so this script can be called from any directory + SCRIPTS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" + ROOT_DIR=$SCRIPTS_DIR/../.. + cd $ROOT_DIR + + echo $ROOT_DIR + + BINARY_DIR=".binaries" + TARGET_DIR=$(pwd)/$BINARY_DIR + + rm -rf $BINARY_DIR 2>/dev/null + mkdir -p $BINARY_DIR + + cd $ROOT_DIR +} + + +docker_build() { + local arch=$1 + local file=$2 + if [ "$arch" == "amd64" ]; then + PLATFORM="x86_64" + else + PLATFORM="aarch64" + fi + + docker buildx build --platform linux/${arch} \ + -t datadog/compile-bottlecap-${SUFFIX} \ + -f .gitlab/scripts/${file} \ + --build-arg PLATFORM=$PLATFORM \ + . -o $TARGET_DIR/compiled-bottlecap-${SUFFIX} + + cp $TARGET_DIR/compiled-bottlecap-${SUFFIX}/bottlecap $TARGET_DIR/bottlecap-${SUFFIX} +} + +prepare_folders +docker_build $ARCHITECTURE $COMPILE_FILE + diff --git a/.gitlab/scripts/build_go_agent.sh b/.gitlab/scripts/compile_go_agent.sh similarity index 59% rename from .gitlab/scripts/build_go_agent.sh rename to .gitlab/scripts/compile_go_agent.sh index 7dbbf02c4..3e7f34c80 100755 --- a/.gitlab/scripts/build_go_agent.sh +++ b/.gitlab/scripts/compile_go_agent.sh @@ -5,9 +5,6 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2024 Datadog, Inc. -# Usage -# ARCHITECTURE=arm64 ./scripts/build_go_agent.sh - set -e if [ -z "$ARCHITECTURE" ]; then @@ -19,8 +16,14 @@ if [ -z "$AGENT_VERSION" ]; then printf "[ERROR]: AGENT_VERSION not specified\n" exit 1 else - printf "Building agent with version: ${AGENT_VERSION}\n" + printf "Compiling agent with version: ${AGENT_VERSION}\n" +fi +if [ -z "$ALPINE" ]; then + printf "[ERROR]: ALPINE not specified\n" + exit 1 +else + printf "Alpine compile requested: ${ALPINE}\n" fi if [ -z "$CI_COMMIT_TAG" ]; then @@ -34,21 +37,11 @@ else fi -if [ -z "$SERVERLESS_INIT" ]; then - printf "Building Datadog Lambda Extension\n" - CMD_PATH="cmd/serverless" +if [ "$ALPINE" = "0" ]; then + COMPILE_FILE=Dockerfile.go_agent.compile else - printf "Building Serverless Init\n" - CMD_PATH="cmd/serverless-init" -fi - - -if [ -z "$ALPINE" ]; then - BUILD_FILE=Dockerfile.build -else - printf "Building for alpine\n" - BUILD_FILE=Dockerfile.alpine.build - BUILD_SUFFIX="-alpine" + printf "Compiling for alpine\n" + COMPILE_FILE=Dockerfile.go_agent.alpine.compile fi # Allow override build tags @@ -63,13 +56,13 @@ fi MAIN_DIR=$(pwd) # datadog-lambda-extension -EXTENSION_DIR=".layers" -TARGET_DIR=$MAIN_DIR/$EXTENSION_DIR +BINARY_DIR=".binaries" +TARGET_DIR=$MAIN_DIR/$BINARY_DIR # Make sure the folder does not exist -rm -rf $EXTENSION_DIR 2>/dev/null +rm -rf $BINARY_DIR 2>/dev/null -mkdir -p $EXTENSION_DIR +mkdir -p $BINARY_DIR # Prepare folder with only *mod and *sum files to enable Docker caching capabilities mkdir -p $MAIN_DIR/scripts/.src $MAIN_DIR/scripts/.cache @@ -83,23 +76,20 @@ cd $AGENT_PATH/.. tar --exclude=.git -czf $MAIN_DIR/scripts/.src/datadog-agent.tgz datadog-agent cd $MAIN_DIR -function docker_build { +function docker_compile { arch=$1 file=$2 docker buildx build --platform linux/${arch} \ - -t datadog/build-go-agent-${arch}:${VERSION} \ - -f ${MAIN_DIR}/scripts/${file} \ + -t datadog/compile-go-agent-${SUFFIX}:${VERSION} \ + -f ${MAIN_DIR}/.gitlab/scripts/${file} \ --build-arg EXTENSION_VERSION="${VERSION}" \ --build-arg AGENT_VERSION="${AGENT_VERSION}" \ - --build-arg CMD_PATH="${CMD_PATH}" \ --build-arg BUILD_TAGS="${BUILD_TAGS}" \ - . -o $TARGET_DIR/datadog_extension-${arch}${BUILD_SUFFIX} - - cp $TARGET_DIR/datadog_extension-${arch}${BUILD_SUFFIX}/datadog_extension.zip $TARGET_DIR/datadog_extension-${arch}${BUILD_SUFFIX}.zip - unzip $TARGET_DIR/datadog_extension-${arch}${BUILD_SUFFIX}/datadog_extension.zip -d $TARGET_DIR/datadog_extension-${arch}${BUILD_SUFFIX} - rm -rf $TARGET_DIR/datadog_extension-${arch}${BUILD_SUFFIX}/datadog_extension.zip + . -o $TARGET_DIR/compiled-datadog-agent-${SUFFIX} + + cp $TARGET_DIR/compiled-datadog-agent-${SUFFIX}/datadog-agent $TARGET_DIR/datadog-agent-${SUFFIX} } -docker_build $ARCHITECTURE $BUILD_FILE +docker_compile $ARCHITECTURE $COMPILE_FILE diff --git a/.gitlab/templates/bottlecap.yaml.tpl b/.gitlab/templates/bottlecap.yaml.tpl deleted file mode 100644 index 3898a5b02..000000000 --- a/.gitlab/templates/bottlecap.yaml.tpl +++ /dev/null @@ -1,103 +0,0 @@ -stages: - - build - - test - - sign - - publish - -default: - retry: - max: 1 - when: - - runner_system_failure - -variables: - DOCKER_TARGET_IMAGE: registry.ddbuild.io/ci/datadog-lambda-extension - DOCKER_TARGET_VERSION: latest - -{{ range $architecture := (ds "architectures").architectures }} - -build layer ({{ $architecture.name }}): - stage: build - image: registry.ddbuild.io/images/docker:20.10 - tags: ["arch:amd64"] - artifacts: - expire_in: 1 hr - paths: - - .layers/datadog_bottlecap-{{ $architecture.name }}.zip - variables: - ARCHITECTURE: {{ $architecture.name }} - script: - - ./scripts/build_bottlecap_layer.sh - -check layer size ({{ $architecture.name }}): - stage: test - image: registry.ddbuild.io/images/docker:20.10 - tags: ["arch:amd64"] - needs: - - build layer ({{ $architecture.name }}) - dependencies: - - build layer ({{ $architecture.name }}) - variables: - LAYER_FILE: datadog_bottlecap-{{ $architecture.name }}.zip - script: - - .gitlab/scripts/check_layer_size.sh - -fmt ({{ $architecture.name }}): - stage: test - tags: ["arch:{{ $architecture.name }}"] - image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} - needs: [] - script: - - cd bottlecap && cargo fmt - -check ({{ $architecture.name }}): - stage: test - tags: ["arch:{{ $architecture.name }}"] - image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} - needs: [] - script: - - cd bottlecap && cargo check - -clippy ({{ $architecture.name }}): - stage: test - tags: ["arch:{{ $architecture.name }}"] - image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} - needs: [] - script: - - cd bottlecap && cargo clippy --all-features - -{{ range $environment := (ds "environments").environments }} - -publish layer {{ $environment.name }} ({{ $architecture.name }}): - stage: publish - tags: ["arch:amd64"] - image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} - rules: - - if: '"{{ $environment.name }}" =~ /^(sandbox|staging)/' - when: manual - allow_failure: true - needs: - - build layer ({{ $architecture.name }}) - - check layer size ({{ $architecture.name }}) - - fmt ({{ $architecture.name }}) - - check ({{ $architecture.name }}) - - clippy ({{ $architecture.name }}) - dependencies: - - build layer ({{ $architecture.name }}) - parallel: - matrix: - - REGION: {{ range (ds "regions").regions }} - - {{ .code }} - {{- end}} - variables: - ARCHITECTURE: {{ $architecture.name }} - LAYER_FILE: datadog_bottlecap-{{ $architecture.name }}.zip - STAGE: {{ $environment.name }} - before_script: - - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh - script: - - .gitlab/scripts/publish_layers.sh - -{{- end }} # environments end - -{{- end }} # architectures end diff --git a/.gitlab/templates/go-agent.yaml.tpl b/.gitlab/templates/go-agent.yaml.tpl deleted file mode 100644 index bc2ed612e..000000000 --- a/.gitlab/templates/go-agent.yaml.tpl +++ /dev/null @@ -1,77 +0,0 @@ -stages: - - build - - test - - sign - - publish - -default: - retry: - max: 1 - when: - - runner_system_failure - -variables: - DOCKER_TARGET_IMAGE: registry.ddbuild.io/ci/datadog-lambda-extension - DOCKER_TARGET_VERSION: latest - GIT_DEPTH: 1 - -{{ range $architecture := (ds "architectures").architectures }} - -build layer ({{ $architecture.name }}): - stage: build - image: registry.ddbuild.io/images/docker:20.10 - tags: ["arch:amd64"] - artifacts: - expire_in: 1 hr - paths: - - .layers/datadog_extension-{{ $architecture.name }}.zip - variables: - ARCHITECTURE: {{ $architecture.name }} - script: - - cd .. && git clone -b $AGENT_BRANCH --single-branch https://github.com/DataDog/datadog-agent.git && cd datadog-lambda-extension - - .gitlab/scripts/build_go_agent.sh - -check layer size ({{ $architecture.name }}): - stage: test - image: registry.ddbuild.io/images/docker:20.10 - tags: ["arch:amd64"] - needs: - - build layer ({{ $architecture.name }}) - dependencies: - - build layer ({{ $architecture.name }}) - variables: - LAYER_FILE: datadog_extension-{{ $architecture.name }}.zip - script: - - .gitlab/scripts/check_layer_size.sh - -{{ range $environment := (ds "environments").environments }} - -publish layer {{ $environment.name }} ({{ $architecture.name }}): - stage: publish - tags: ["arch:amd64"] - image: ${DOCKER_TARGET_IMAGE}:${DOCKER_TARGET_VERSION} - rules: - - if: '"{{ $environment.name }}" =~ /^(sandbox|staging)/' - when: manual - allow_failure: true - needs: - - build layer ({{ $architecture.name }}) - dependencies: - - build layer ({{ $architecture.name }}) - parallel: - matrix: - - REGION: {{ range (ds "regions").regions }} - - {{ .code }} - {{- end}} - variables: - ARCHITECTURE: {{ $architecture.name }} - LAYER_FILE: datadog_extension-{{ $architecture.name }}.zip - STAGE: {{ $environment.name }} - before_script: - - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh - script: - - .gitlab/scripts/publish_layers.sh - -{{- end }} # environments end - -{{- end }} # architectures end diff --git a/.gitlab/templates/lambda-extension.yaml.tpl b/.gitlab/templates/lambda-extension.yaml.tpl deleted file mode 100644 index 46eb4da68..000000000 --- a/.gitlab/templates/lambda-extension.yaml.tpl +++ /dev/null @@ -1,299 +0,0 @@ -stages: - - build - - test - - sign - - publish - -default: - retry: - max: 1 - when: - - runner_system_failure - -variables: - CI_DOCKER_TARGET_IMAGE: registry.ddbuild.io/ci/datadog-lambda-extension - CI_DOCKER_TARGET_VERSION: latest - -{{ range $architecture := (ds "architectures").architectures }} - -build go agent ({{ $architecture.name }}): - stage: build - image: registry.ddbuild.io/images/docker:20.10 - tags: ["arch:amd64"] - artifacts: - expire_in: 1 hr - paths: - - .layers/datadog_extension-{{ $architecture.name }}.zip - - .layers/datadog_extension-{{ $architecture.name }}/* - variables: - ARCHITECTURE: {{ $architecture.name }} - script: - - cd .. && git clone -b $AGENT_BRANCH --single-branch https://github.com/DataDog/datadog-agent.git && cd datadog-lambda-extension - - .gitlab/scripts/build_go_agent.sh - -build bottlecap ({{ $architecture.name }}): - stage: build - image: registry.ddbuild.io/images/docker:20.10 - tags: ["arch:amd64"] - needs: - - build go agent ({{ $architecture.name }}) - dependencies: - - build go agent ({{ $architecture.name }}) - artifacts: - expire_in: 1 hr - paths: - - .layers/datadog_bottlecap-{{ $architecture.name }}.zip - - .layers/datadog_bottlecap-{{ $architecture.name }}/* - variables: - ARCHITECTURE: {{ $architecture.name }} - script: - - .gitlab/scripts/build_bottlecap.sh - -# Alpine -build go agent ({{ $architecture.name }}, alpine): - stage: build - image: registry.ddbuild.io/images/docker:20.10 - tags: ["arch:amd64"] - artifacts: - expire_in: 1 hr - paths: - - .layers/datadog_extension-{{ $architecture.name }}-alpine.zip - - .layers/datadog_extension-{{ $architecture.name }}-alpine/* - variables: - ARCHITECTURE: {{ $architecture.name }} - ALPINE: 1 - script: - - cd .. && git clone -b $AGENT_BRANCH --single-branch https://github.com/DataDog/datadog-agent.git && cd datadog-lambda-extension - - .gitlab/scripts/build_go_agent.sh - -build bottlecap ({{ $architecture.name }}, alpine): - stage: build - image: registry.ddbuild.io/images/docker:20.10 - tags: ["arch:amd64"] - needs: - - build go agent ({{ $architecture.name }}, alpine) - dependencies: - - build go agent ({{ $architecture.name }}, alpine) - artifacts: - expire_in: 1 hr - paths: - - .layers/datadog_bottlecap-{{ $architecture.name }}-alpine.zip - - .layers/datadog_bottlecap-{{ $architecture.name }}-alpine/* - variables: - ARCHITECTURE: {{ $architecture.name }} - ALPINE: 1 - script: - - .gitlab/scripts/build_bottlecap.sh -# Alpine end - -check layer size ({{ $architecture.name }}): - stage: test - image: registry.ddbuild.io/images/docker:20.10 - tags: ["arch:amd64"] - needs: - - build bottlecap ({{ $architecture.name }}) - dependencies: - - build bottlecap ({{ $architecture.name }}) - variables: - LAYER_FILE: datadog_bottlecap-{{ $architecture.name }}.zip - script: - - .gitlab/scripts/check_layer_size.sh - -fmt ({{ $architecture.name }}): - stage: test - tags: ["arch:{{ $architecture.name }}"] - image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION} - needs: [] - script: - - cd bottlecap && cargo fmt - -check ({{ $architecture.name }}): - stage: test - tags: ["arch:{{ $architecture.name }}"] - image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION} - needs: [] - script: - - cd bottlecap && cargo check - -clippy ({{ $architecture.name }}): - stage: test - tags: ["arch:{{ $architecture.name }}"] - image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION} - needs: [] - script: - - cd bottlecap && cargo clippy --all-features - -{{ range $environment := (ds "environments").environments }} - -{{ if or (eq $environment.name "prod") }} -sign layer ({{ $architecture.name }}): - stage: sign - tags: ["arch:amd64"] - image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION} - rules: - - if: '$CI_COMMIT_TAG =~ /^v.*/' - when: manual - needs: - - build bottlecap ({{ $architecture.name }}) - - check layer size ({{ $architecture.name }}) - - fmt ({{ $architecture.name }}) - - check ({{ $architecture.name }}) - - clippy ({{ $architecture.name }}) - dependencies: - - build bottlecap ({{ $architecture.name }}) - artifacts: # Re specify artifacts so the modified signed file is passed - expire_in: 1 day # Signed layers should expire after 1 day - paths: - - .layers/datadog_bottlecap-{{ $architecture.name }}.zip - variables: - LAYER_FILE: datadog_bottlecap-{{ $architecture.name }}.zip - before_script: - - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh - script: - - .gitlab/scripts/sign_layers.sh {{ $environment.name }} -{{ end }} - -publish layer {{ $environment.name }} ({{ $architecture.name }}): - stage: publish - tags: ["arch:amd64"] - image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION} - rules: - - if: '"{{ $environment.name }}" =~ /^(sandbox|staging)/' - when: manual - allow_failure: true - - if: '$CI_COMMIT_TAG =~ /^v.*/' - needs: -{{ if or (eq $environment.name "prod") }} - - sign layer ({{ $architecture.name }}) -{{ else }} - - build bottlecap ({{ $architecture.name }}) - - check layer size ({{ $architecture.name }}) - - fmt ({{ $architecture.name }}) - - check ({{ $architecture.name }}) - - clippy ({{ $architecture.name }}) -{{ end }} - dependencies: -{{ if or (eq $environment.name "prod") }} - - sign layer ({{ $architecture.name }}) -{{ else }} - - build bottlecap ({{ $architecture.name }}) -{{ end }} - parallel: - matrix: - - REGION: {{ range (ds "regions").regions }} - - {{ .code }} - {{- end}} - variables: - ARCHITECTURE: {{ $architecture.name }} - LAYER_FILE: datadog_bottlecap-{{ $architecture.name }}.zip - STAGE: {{ $environment.name }} - before_script: - - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh - script: - - .gitlab/scripts/publish_layers.sh - -{{ if or (eq $environment.name "sandbox") }} -publish private images: - stage: publish - tags: ["arch:amd64"] - image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION} - needs: - - build bottlecap (arm64) - - build bottlecap (amd64) - when: manual - dependencies: - - build bottlecap (arm64) - - build bottlecap (amd64) - before_script: - - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh - script: - - .gitlab/scripts/build_private_image.sh - -publish private images (alpine): - stage: publish - tags: ["arch:amd64"] - image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION} - needs: - - build bottlecap (arm64, alpine) - - build bottlecap (amd64, alpine) - when: manual - dependencies: - - build bottlecap (arm64, alpine) - - build bottlecap (amd64, alpine) - variables: - ALPINE: 1 - before_script: - - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh - script: - - .gitlab/scripts/build_private_image.sh -{{ end }} - -{{- end }} # environments end - -{{- end }} # architectures end - -build images: - stage: build - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - rules: - - if: '$CI_COMMIT_TAG =~ /^v.*/' - needs: - - build bottlecap (arm64) - - build bottlecap (amd64) - dependencies: - - build bottlecap (arm64) - - build bottlecap (amd64) - script: - - .gitlab/scripts/build_image.sh - -build images (alpine): - stage: build - tags: ["arch:amd64"] - image: registry.ddbuild.io/images/docker:20.10 - rules: - - if: '$CI_COMMIT_TAG =~ /^v.*/' - needs: - - build bottlecap (arm64, alpine) - - build bottlecap (amd64, alpine) - dependencies: - - build bottlecap (arm64, alpine) - - build bottlecap (amd64, alpine) - variables: - ALPINE: 1 - script: - - .gitlab/scripts/build_image.sh - -publish images: - stage: publish - rules: - - if: '$CI_COMMIT_TAG =~ /^v.*/' - needs: - - build images - when: manual - trigger: - project: DataDog/public-images - branch: main - strategy: depend - variables: - IMG_SOURCES: ${CI_DOCKER_TARGET_IMAGE}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA} - IMG_DESTINATIONS: lambda-extension:${VERSION},lambda-extension:latest - IMG_REGISTRIES: dockerhub,ecr-public,gcr-datadoghq - IMG_SIGNING: false - -publish images (alpine): - stage: publish - rules: - - if: '$CI_COMMIT_TAG =~ /^v.*/' - needs: - - build images (alpine) - when: manual - trigger: - project: DataDog/public-images - branch: main - strategy: depend - variables: - IMG_SOURCES: ${CI_DOCKER_TARGET_IMAGE}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}-alpine - IMG_DESTINATIONS: lambda-extension:${VERSION}-alpine,lambda-extension:latest-alpine - IMG_REGISTRIES: dockerhub,ecr-public,gcr-datadoghq - IMG_SIGNING: false diff --git a/.gitlab/templates/pipeline.yaml.tpl b/.gitlab/templates/pipeline.yaml.tpl new file mode 100644 index 000000000..ba04d9daf --- /dev/null +++ b/.gitlab/templates/pipeline.yaml.tpl @@ -0,0 +1,297 @@ +stages: + - test + - compile + - build + - self-monitoring + - sign + - publish + +default: + retry: + max: 1 + when: + - runner_system_failure + +variables: + CI_DOCKER_TARGET_IMAGE: registry.ddbuild.io/ci/datadog-lambda-extension + CI_DOCKER_TARGET_VERSION: latest + +{{ range $flavor := (ds "flavors").flavors }} + +{{ if $flavor.needs_code_checks }} + +cargo fmt ({{ $flavor.arch }}): + stage: test + tags: ["arch:{{ $flavor.arch }}"] + image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION} + needs: [] + script: + - cd bottlecap && cargo fmt + +cargo check ({{ $flavor.arch }}): + stage: test + tags: ["arch:{{ $flavor.arch }}"] + image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION} + needs: [] + script: + - cd bottlecap && cargo check + +cargo clippy ({{ $flavor.arch }}): + stage: test + tags: ["arch:{{ $flavor.arch }}"] + image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION} + needs: [] + script: + - cd bottlecap && cargo clippy --all-features + +{{ end }} # end needs_code_checks + +go agent ({{ $flavor.name }}): + stage: compile + image: registry.ddbuild.io/images/docker:20.10 + tags: ["arch:amd64"] + needs: [] + artifacts: + expire_in: 1 hr + paths: + - .binaries/datadog-agent-{{ $flavor.suffix }} + variables: + ARCHITECTURE: {{ $flavor.arch }} + ALPINE: {{ $flavor.alpine }} + SUFFIX: {{ $flavor.suffix }} + script: + - echo "Building go agent based on $AGENT_BRANCH" + # TODO: do this clone once in a separate job so that we can make sure that + # we're using the same exact code for all of the builds (main can move + # between different runs of the various compile jobs, for example) + - cd .. && git clone -b $AGENT_BRANCH --single-branch https://github.com/DataDog/datadog-agent.git && cd datadog-agent && git rev-parse HEAD && cd ../datadog-lambda-extension + - .gitlab/scripts/compile_go_agent.sh + +bottlecap ({{ $flavor.name }}): + stage: compile + image: registry.ddbuild.io/images/docker:20.10 + tags: ["arch:amd64"] + needs: [] + artifacts: + expire_in: 1 hr + paths: + - .binaries/bottlecap-{{ $flavor.suffix }} + variables: + ARCHITECTURE: {{ $flavor.arch }} + ALPINE: {{ $flavor.alpine }} + SUFFIX: {{ $flavor.suffix }} + script: + - .gitlab/scripts/compile_bottlecap.sh + +layer ({{ $flavor.name }}): + stage: build + image: registry.ddbuild.io/images/docker:20.10 + tags: ["arch:amd64"] + needs: + - go agent ({{ $flavor.name }}) + - bottlecap ({{ $flavor.name }}) + - cargo fmt ({{ $flavor.arch }}) + - cargo check ({{ $flavor.arch }}) + - cargo clippy ({{ $flavor.arch }}) + dependencies: + - go agent ({{ $flavor.name }}) + - bottlecap ({{ $flavor.name }}) + artifacts: + expire_in: 1 hr + paths: + - .layers/datadog_extension-{{ $flavor.suffix }}.zip + - .layers/datadog_extension-{{ $flavor.suffix }}/* + variables: + ARCHITECTURE: {{ $flavor.arch }} + SUFFIX: {{ $flavor.suffix }} + script: + - .gitlab/scripts/build_layer.sh + +{{ if $flavor.needs_layer_publish }} + +check layer size ({{ $flavor.name }}): + stage: build + image: registry.ddbuild.io/images/docker:20.10 + tags: ["arch:amd64"] + needs: + - layer ({{ $flavor.name }}) + dependencies: + - layer ({{ $flavor.name }}) + variables: + LAYER_FILE: datadog_extension-{{ $flavor.suffix }}.zip + script: + - .gitlab/scripts/check_layer_size.sh + +{{ range $environment := (ds "environments").environments }} + +{{ if or (eq $environment.name "prod") }} + +sign layer ({{ $flavor.name }}): + stage: sign + tags: ["arch:amd64"] + image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION} + rules: + - if: '$CI_COMMIT_TAG =~ /^v.*/' + when: manual + needs: + - layer ({{ $flavor.name }}) + - check layer size ({{ $flavor.name }}) + dependencies: + - layer ({{ $flavor.name }}) + artifacts: # Re specify artifacts so the modified signed file is passed + expire_in: 1 day # Signed layers should expire after 1 day + paths: + - .layers/datadog_extension-{{ $flavor.suffix }}.zip + variables: + LAYER_FILE: datadog_extension-{{ $flavor.suffix }}.zip + before_script: + - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh + script: + - .gitlab/scripts/sign_layers.sh {{ $environment.name }} + +{{ end }} # if prod + +publish layer {{ $environment.name }} ({{ $flavor.name }}): + stage: publish + tags: ["arch:amd64"] + image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION} + rules: + - if: '"{{ $environment.name }}" =~ /^(sandbox|staging)/' + when: manual + allow_failure: true + - if: '$CI_COMMIT_TAG =~ /^v.*/' + + needs: +{{ if eq $environment.name "prod" }} + - check layer size ({{ $flavor.name }}) + - sign layer ({{ $flavor.name }}) +{{ else }} + - layer ({{ $flavor.name }}) +{{ end }} #end if prod + + dependencies: +{{ if or (eq $environment.name "prod") }} + - sign layer ({{ $flavor.name }}) +{{ else }} + - layer ({{ $flavor.name }}) +{{ end }} #end if prod + + parallel: + matrix: + - REGION: {{ range (ds "regions").regions }} + - {{ .code }} + {{- end}} + variables: + ARCHITECTURE: {{ $flavor.arch }} + LAYER_FILE: datadog_extension-{{ $flavor.suffix }}.zip + STAGE: {{ $environment.name }} + before_script: + - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh + script: + - .gitlab/scripts/publish_layers.sh + +{{ if eq $environment.name "sandbox" }} + +publish layer sandbox [us-east-1] ({{ $flavor.name }}): + stage: self-monitoring + tags: ["arch:amd64"] + image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION} + rules: + - when: manual + allow_failure: true + + needs: + - layer ({{ $flavor.name }}) + + dependencies: + - layer ({{ $flavor.name }}) + + variables: + REGION: us-east-1 + ARCHITECTURE: {{ $flavor.arch }} + LAYER_FILE: datadog_extension-{{ $flavor.suffix }}.zip + STAGE: {{ $environment.name }} + before_script: + - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh + script: + - .gitlab/scripts/publish_layers.sh + +{{ end }} # if environment sandbox + +{{ end }} # end environments + +{{ end }} # end needs_layer_publish + +{{ end }} # end flavors + +{{ range $multi_arch_image_flavor := (ds "flavors").multi_arch_image_flavors }} + +{{ range $environment := (ds "environments").environments }} + +{{ if eq $environment.name "sandbox" }} + +publish private images ({{ $multi_arch_image_flavor.name }}): + stage: self-monitoring + tags: ["arch:amd64"] + image: ${CI_DOCKER_TARGET_IMAGE}:${CI_DOCKER_TARGET_VERSION} + when: manual + needs: + {{ range $multi_arch_image_flavor.dependency_names }} + - layer ({{ . }}) + {{ end }} # end dependency_names + dependencies: + {{ range $multi_arch_image_flavor.dependency_names }} + - layer ({{ . }}) + {{ end }} # end dependency_names + variables: + ALPINE: {{ $multi_arch_image_flavor.alpine }} + SUFFIX: {{ $multi_arch_image_flavor.suffix }} + PLATFORM: {{ $multi_arch_image_flavor.platform }} + before_script: + - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source .gitlab/scripts/get_secrets.sh + script: + - .gitlab/scripts/build_private_image.sh + +{{ end }} # end if environment sandbox + +{{ end }} # end environments + +image ({{ $multi_arch_image_flavor.name }}): + stage: build + tags: ["arch:amd64"] + image: registry.ddbuild.io/images/docker:20.10 + rules: + - if: '$CI_COMMIT_TAG =~ /^v.*/' + needs: + {{ range $multi_arch_image_flavor.dependency_names }} + - layer ({{ . }}) + {{ end }} # end dependency_names + dependencies: + {{ range $multi_arch_image_flavor.dependency_names }} + - layer ({{ . }}) + {{ end }} # end dependency_names + variables: + ALPINE: {{ $multi_arch_image_flavor.alpine }} + SUFFIX: {{ $multi_arch_image_flavor.suffix }} + PLATFORM: {{ $multi_arch_image_flavor.platform }} + script: + - .gitlab/scripts/build_image.sh + +publish image ({{ $multi_arch_image_flavor.name }}): + stage: publish + rules: + - if: '$CI_COMMIT_TAG =~ /^v.*/' + needs: + - image ({{ $multi_arch_image_flavor.name }}) + when: manual + trigger: + project: DataDog/public-images + branch: main + strategy: depend + variables: + IMG_SOURCES: ${CI_DOCKER_TARGET_IMAGE}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}{{ $multi_arch_image_flavor.suffix }} + IMG_DESTINATIONS: lambda-extension:${VERSION}{{ $multi_arch_image_flavor.suffix }},lambda-extension:latest{{ $multi_arch_image_flavor.suffix }} + IMG_REGISTRIES: dockerhub,ecr-public,gcr-datadoghq + IMG_SIGNING: false + +{{ end }} # end multi_arch_image_flavors diff --git a/scripts/datadog_wrapper b/scripts/datadog_wrapper index 0b84e462d..386da0d4d 100644 --- a/scripts/datadog_wrapper +++ b/scripts/datadog_wrapper @@ -1,4 +1,9 @@ #!/bin/bash + +# This is a wrapper script that is intended be used in conjunction with the +# AWS_LAMBDA_EXEC_WRAPPER environment variable on an AWS Lambda function. It +# enables universal instrumentation support for our Java and .NET runtimes. + args=("$@") # lowercase DD_LOG_LEVEL