Skip to content

Commit c676964

Browse files
apiarian-datadogapiarian-datadog
committed
chore: separate out the compile stage for go
1 parent 13db576 commit c676964

File tree

5 files changed

+242
-0
lines changed

5 files changed

+242
-0
lines changed

‎.gitlab/datasources/flavors.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,23 @@ flavors:
22
- name: amd64
33
arch: amd64
44
needs_code_checks: true
5+
flavor_suffix: amd64
6+
alpine: 0
57

68
- name: arm64
79
arch: arm64
810
needs_code_checks: true
11+
flavor_suffix: arm64
12+
alpine: 0
913

1014
- name: amd64, alpine
1115
arch: amd64
1216
needs_code_checks: false
17+
flavor_suffix: amd64-alpine
18+
alpine: 1
1319

1420
- name: arm64, alpine
1521
arch: arm64
1622
needs_code_checks: false
23+
flavor_suffix: arm64-alpine
24+
alpine: 1

‎.gitlab/scripts/compile_go_agent.sh‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
#!/bin/bash
2+
3+
# Unless explicitly stated otherwise all files in this repository are licensed
4+
# under the Apache License Version 2.0.
5+
# This product includes software developed at Datadog (https://www.datadoghq.com/).
6+
# Copyright 2024 Datadog, Inc.
7+
8+
set -e
9+
10+
if [ -z "$ARCHITECTURE" ]; then
11+
printf "[ERROR]: ARCHITECTURE not specified\n"
12+
exit 1
13+
fi
14+
15+
if [ -z "$AGENT_VERSION" ]; then
16+
printf "[ERROR]: AGENT_VERSION not specified\n"
17+
exit 1
18+
else
19+
printf "Compiling agent with version: ${AGENT_VERSION}\n"
20+
fi
21+
22+
if [ -z "$ALPINE" ]; then
23+
printf "[ERROR]: ALPINE not specified\n"
24+
exit 1
25+
else
26+
printf "Alpine compile requested: ${ALPINE}\n"
27+
fi
28+
29+
if [ -z "$CI_COMMIT_TAG" ]; then
30+
# Running on dev
31+
printf "Running on dev environment\n"
32+
VERSION="dev"
33+
else
34+
printf "Found version tag in environment\n"
35+
VERSION="${CI_COMMIT_TAG//[!0-9]/}"
36+
printf "Version: ${VERSION}\n"
37+
fi
38+
39+
40+
if [ -z "$SERVERLESS_INIT" ]; then
41+
printf "Compiling Datadog Lambda Extension\n"
42+
CMD_PATH="cmd/serverless"
43+
else
44+
printf "Compiling Serverless Init\n"
45+
CMD_PATH="cmd/serverless-init"
46+
fi
47+
48+
49+
if [ -z "$ALPINE" ]; then
50+
COMPILE_FILE=Dockerfile.compile
51+
else
52+
printf "Compiling for alpine\n"
53+
COMPILE_FILE=Dockerfile.alpine.compile
54+
fi
55+
56+
# Allow override build tags
57+
if [ -z "$BUILD_TAGS" ]; then
58+
BUILD_TAGS="serverless otlp"
59+
fi
60+
61+
# Allow override agent path
62+
if [ -z "$AGENT_PATH" ]; then
63+
AGENT_PATH="../datadog-agent"
64+
fi
65+
66+
MAIN_DIR=$(pwd) # datadog-lambda-extension
67+
68+
BINARY_DIR=".binaries"
69+
TARGET_DIR=$MAIN_DIR/$BINARY_DIR
70+
71+
# Make sure the folder does not exist
72+
rm -rf $BINARY_DIR 2>/dev/null
73+
74+
mkdir -p $BINARY_DIR
75+
76+
# Prepare folder with only *mod and *sum files to enable Docker caching capabilities
77+
mkdir -p $MAIN_DIR/scripts/.src $MAIN_DIR/scripts/.cache
78+
printf "Copy mod files to build a cache\n"
79+
cp $AGENT_PATH/go.mod $MAIN_DIR/scripts/.cache
80+
cp $AGENT_PATH/go.sum $MAIN_DIR/scripts/.cache
81+
82+
# Compress all files to speed up docker copy
83+
touch $MAIN_DIR/scripts/.src/datadog-agent.tgz
84+
cd $AGENT_PATH/..
85+
tar --exclude=.git -czf $MAIN_DIR/scripts/.src/datadog-agent.tgz datadog-agent
86+
cd $MAIN_DIR
87+
88+
function docker_compile {
89+
arch=$1
90+
file=$2
91+
92+
docker buildx build --platform linux/${arch} \
93+
-t datadog/compile-go-agent-${SUFFIX}:${VERSION} \
94+
-f ${MAIN_DIR}/scripts/${file} \
95+
--build-arg EXTENSION_VERSION="${VERSION}" \
96+
--build-arg AGENT_VERSION="${AGENT_VERSION}" \
97+
--build-arg CMD_PATH="${CMD_PATH}" \
98+
--build-arg BUILD_TAGS="${BUILD_TAGS}" \
99+
. -o $TARGET_DIR/compiled-datadog-agent-${SUFFIX}
100+
101+
cp $TARGET_DIR/compiled-datadog-agent-${SUFFIX}/datadog-agent $TARGET_DIR/datadog-agent-${SUFFIX}
102+
}
103+
104+
docker_compile $ARCHITECTURE $COMPILE_FILE
105+

‎.gitlab/templates/pipeline.yaml.tpl‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
stages:
22
- check code
3+
- compile
34
- build
45
- test
56
- sign
@@ -45,6 +46,26 @@ cargo clippy ({{ $flavor.name }}):
4546

4647
{{ end }} # end needs_code_checks
4748

49+
compile go agent ({{ $flavor.name }}):
50+
stage: compile
51+
image: registry.ddbuild.io/images/docker:20.10
52+
tags: ["arch:amd64"]
53+
artifacts:
54+
expire_in: 1 hr
55+
paths:
56+
- .binaries/datadog-agent-{{ $flavor.suffix }}
57+
variables:
58+
ARCHITECTURE: {{ $flavor.arch }}
59+
ALPINE: {{ $flavor.alpine }}
60+
SUFFIX: {{ $flavor.suffix }}
61+
script:
62+
- echo "Building go agent based on $AGENT_BRANCH"
63+
# TODO: do this clone once in a separate job so that we can make sure that
64+
# we're using the same exact code for all of the builds (main can move
65+
# between different runs of the various compile jobs, for example)
66+
- cd .. && git clone -b $AGENT_BRANCH --single-branch https://github.com/DataDog/datadog-agent.git && git rev-parse HEAD && cd datadog-lambda-extension
67+
- ./gitlab/scripts/compile_go_agent.sh
68+
4869
{{ end }} # end flavors
4970

5071
{{ range $architecture := (ds "architectures").architectures }}

‎scripts/Dockerfile.alpine.compile‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
# syntax = docker/dockerfile:experimental
2+
3+
FROM alpine:3.16 as compiler
4+
ARG EXTENSION_VERSION
5+
ARG AGENT_VERSION
6+
ARG CMD_PATH
7+
ARG BUILD_TAGS
8+
9+
RUN apk add --no-cache git make musl-dev gcc
10+
COPY --from=golang:1.23.6-alpine /usr/local/go/ /usr/lib/go
11+
12+
ENV GOROOT /usr/lib/go
13+
ENV GOPATH /go
14+
ENV PATH /go/bin:$PATH
15+
16+
RUN mkdir -p ${GOPATH}/src ${GOPATH}/bin
17+
RUN mkdir -p /tmp/dd/datadog-agent
18+
19+
# cache dependencies
20+
COPY ./scripts/.cache/go.mod /tmp/dd/datadog-agent
21+
COPY ./scripts/.cache/go.sum /tmp/dd/datadog-agent
22+
WORKDIR /tmp/dd/datadog-agent
23+
24+
# copy source files (/tgz gets unzip automatically by Docker)
25+
ADD ./scripts/.src/datadog-agent.tgz /tmp/dd
26+
27+
# build the extension
28+
WORKDIR /tmp/dd/datadog-agent/"${CMD_PATH}"
29+
# add the current version number to the tags package before compilation
30+
31+
RUN --mount=type=cache,target=/go/pkg/mod \
32+
--mount=type=cache,target=/root/.cache/go-build \
33+
if [ -z "$AGENT_VERSION" ]; then \
34+
/usr/lib/go/bin/go build -ldflags="-w -extldflags '-static' \
35+
-X github.com/DataDog/datadog-agent/pkg/serverless/tags.currentExtensionVersion=$EXTENSION_VERSION" \
36+
-tags "${BUILD_TAGS}" -o datadog-agent; \
37+
else \
38+
/usr/lib/go/bin/go build -ldflags="-w -extldflags '-static' \
39+
-X github.com/DataDog/datadog-agent/pkg/serverless/tags.currentExtensionVersion=$EXTENSION_VERSION \
40+
-X github.com/DataDog/datadog-agent/pkg/version.agentVersionDefault=$AGENT_VERSION" \
41+
-tags "${BUILD_TAGS}" -o datadog-agent; \
42+
fi
43+
44+
RUN /usr/lib/go/bin/go tool nm datadog-agent | grep -w 'github.com/DataDog/datadog-agent/pkg/version.agentVersionDefault' || \
45+
(echo "agentVersionDefault variable doesn't exist" && exit 1)
46+
47+
RUN strip datadog-agent
48+
49+
# keep the smallest possible docker image
50+
FROM scratch
51+
COPY --from=compiler /tmp/dd/datadog-agent/"${CMD_PATH}"/datadog-agent /
52+
ENTRYPOINT ["/datadog-agent"]
53+

‎scripts/Dockerfile.compile‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
# syntax = docker/dockerfile:experimental
2+
3+
FROM public.ecr.aws/lambda/provided:al2 as compiler
4+
ARG EXTENSION_VERSION
5+
ARG AGENT_VERSION
6+
ARG CMD_PATH
7+
ARG BUILD_TAGS
8+
RUN mkdir -p /tmp/dd/datadog-agent
9+
10+
RUN yum install -y wget tar gzip gcc
11+
RUN arch="$(uname -m)"; \
12+
if [ "${arch}" = 'aarch64' ]; then \
13+
arch='arm64'; \
14+
fi; \
15+
if [ "${arch}" = 'x86_64' ]; then \
16+
arch='amd64'; \
17+
fi; \
18+
wget -O go1.23.6.linux-${arch}.tar.gz https://go.dev/dl/go1.23.6.linux-${arch}.tar.gz; \
19+
tar -C /usr/local -xzf go1.23.6.linux-${arch}.tar.gz
20+
21+
# cache dependencies
22+
COPY ./scripts/.cache/go.mod /tmp/dd/datadog-agent
23+
COPY ./scripts/.cache/go.sum /tmp/dd/datadog-agent
24+
WORKDIR /tmp/dd/datadog-agent
25+
26+
# copy source files (/tgz gets unzip automatically by Docker)
27+
ADD ./scripts/.src/datadog-agent.tgz /tmp/dd
28+
29+
# build the extension
30+
WORKDIR /tmp/dd/datadog-agent/"${CMD_PATH}"
31+
# add the current version number to the tags package before compilation
32+
33+
RUN --mount=type=cache,target=/root/go/pkg/mod \
34+
--mount=type=cache,target=/root/.cache/go-build \
35+
if [ -z "$AGENT_VERSION" ]; then \
36+
/usr/local/go/bin/go build -ldflags="-w \
37+
-X github.com/DataDog/datadog-agent/pkg/serverless/tags.currentExtensionVersion=$EXTENSION_VERSION" \
38+
-tags "${BUILD_TAGS}" -o datadog-agent; \
39+
else \
40+
/usr/local/go/bin/go build -ldflags="-w \
41+
-X github.com/DataDog/datadog-agent/pkg/serverless/tags.currentExtensionVersion=$EXTENSION_VERSION \
42+
-X github.com/DataDog/datadog-agent/pkg/version.agentVersionDefault=$AGENT_VERSION" \
43+
-tags "${BUILD_TAGS}" -o datadog-agent; \
44+
fi
45+
46+
RUN /usr/local/go/bin/go tool nm datadog-agent | grep -w 'github.com/DataDog/datadog-agent/pkg/version.agentVersionDefault' || \
47+
(echo "agentVersionDefault variable doesn't exist" && exit 1)
48+
49+
RUN strip datadog-agent
50+
51+
# keep the smallest possible docker imag
52+
FROM scratch
53+
COPY --from=compiler /tmp/dd/datadog-agent/"${CMD_PATH}"/datadog-agent /
54+
ENTRYPOINT ["/datadog-agent"]
55+

0 commit comments

Comments
 (0)