Skip to content

Commit e177d55

Browse files
chore: [SVLS-6279] fips-flavored builds (#556)
In order to support FIPS flavored go agent builds (added in DataDog/datadog-agent#33799 ) we're making the following changes: - refactoring the build pipeline to add fips flavors - rearranging the environment datasource to make it a dictionary instead of a list - replacing our old publish_govcloud.sh script with a new publish_govcloud_layers.sh script which uses the same publish layers script as the commercial gitlab job
1 parent 992a733 commit e177d55

17 files changed

+431
-154
lines changed

.gitlab-ci.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ variables:
1111
AGENT_VERSION:
1212
description: "Latest release version of the datadog-agent to tag the build with."
1313
value: "7.61.0"
14-
LAYER_SUFFIX:
14+
PIPELINE_LAYER_SUFFIX:
1515
description: "Suffix to be appended to the layer name (default empty)."
1616
value: ""
1717

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,13 @@
11
environments:
2-
- name: sandbox
2+
sandbox:
33
external_id: sandbox-publish-externalid
44
role_to_assume: sandbox-layer-deployer
55
account: 425362996713
6-
- name: prod
6+
add_layer_version_permissions: 0
7+
automatically_bump_version: 1
8+
prod:
79
external_id: prod-publish-externalid
810
role_to_assume: dd-serverless-layer-deployer-role
911
account: 464622532012
12+
add_layer_version_permissions: 1
13+
automatically_bump_version: 0

.gitlab/datasources/flavors.yaml

Lines changed: 68 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,42 +2,105 @@ flavors:
22
- name: amd64
33
arch: amd64
44
alpine: 0
5-
needs_code_checks: true
5+
fips: 0
6+
needs_layer_sign: true
67
needs_layer_publish: true
78
suffix: amd64
9+
layer_name_base_suffix: ""
10+
max_layer_compressed_size_mb: 23
11+
max_layer_uncompressed_size_mb: 54
812

913
- name: arm64
1014
arch: arm64
1115
alpine: 0
12-
needs_code_checks: true
16+
fips: 0
17+
needs_layer_sign: true
1318
needs_layer_publish: true
1419
suffix: arm64
20+
layer_name_base_suffix: "-ARM"
21+
max_layer_compressed_size_mb: 21
22+
max_layer_uncompressed_size_mb: 50
1523

1624
- name: amd64, alpine
1725
arch: amd64
1826
alpine: 1
19-
needs_code_checks: false
27+
fips: 0
28+
needs_layer_sign: false
2029
needs_layer_publish: false
2130
suffix: amd64-alpine
2231

2332
- name: arm64, alpine
2433
arch: arm64
2534
alpine: 1
26-
needs_code_checks: false
35+
fips: 0
36+
needs_layer_sign: false
2737
needs_layer_publish: false
2838
suffix: arm64-alpine
2939

40+
- name: amd64, fips
41+
arch: amd64
42+
alpine: 0
43+
fips: 1
44+
needs_layer_sign: true
45+
needs_layer_publish: false
46+
suffix: amd64-fips
47+
max_layer_compressed_size_mb: 24
48+
max_layer_uncompressed_size_mb: 56
49+
50+
- name: arm64, fips
51+
arch: arm64
52+
alpine: 0
53+
fips: 1
54+
needs_layer_sign: true
55+
needs_layer_publish: false
56+
suffix: arm64-fips
57+
max_layer_compressed_size_mb: 21
58+
max_layer_uncompressed_size_mb: 52
59+
60+
- name: amd64, fips, alpine
61+
arch: amd64
62+
alpine: 1
63+
fips: 1
64+
needs_layer_sign: false
65+
needs_layer_publish: false
66+
suffix: amd64-alpine-fips
67+
68+
- name: arm64, fips, alpine
69+
arch: arm64
70+
alpine: 1
71+
fips: 1
72+
needs_layer_sign: false
73+
needs_layer_publish: false
74+
suffix: arm64-alpine-fips
75+
3076
# Unfortunately our mutli-arch images don't fit nicely into the flavors
31-
# structure above.
77+
# structure above. Since we are making multi-arch images, the suffixes here
78+
# omit the "arch" component and only include the alpine/fips bits.
3279
multi_arch_image_flavors:
3380
- name: basic
3481
alpine: 0
82+
fips: 0
3583
platform: linux/amd64,linux/arm64
3684
dependency_names: [amd64, arm64]
3785
suffix: ""
3886

3987
- name: alpine
4088
alpine: 1
89+
fips: 0
4190
platform: linux/amd64,linux/arm64
4291
dependency_names: ["amd64, alpine", "arm64, alpine"]
4392
suffix: "-alpine"
93+
94+
- name: fips
95+
alpine: 0
96+
fips: 1
97+
platform: linux/amd64,linux/arm64
98+
dependency_names: ["amd64, fips", "arm64, fips"]
99+
suffix: "-fips"
100+
101+
- name: fips, alpine
102+
alpine: 1
103+
fips: 1
104+
platform: linux/amd64,linux/arm64
105+
dependency_names: ["amd64, fips, alpine", "arm64, fips, alpine"]
106+
suffix: "-alpine-fips"

.gitlab/scripts/build_private_image.sh

Lines changed: 9 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -11,30 +11,19 @@ DOCKER_TARGET_IMAGE="425362996713.dkr.ecr.us-east-1.amazonaws.com/self-monitorin
1111
EXTENSION_DIR=".layers"
1212
IMAGE_TAG="latest"
1313

14-
if [ -z "$ALPINE" ]; then
15-
printf "[ERROR]: ALPINE not specified\n"
16-
exit 1
17-
else
18-
printf "Alpine build requested: ${ALPINE}\n"
19-
fi
20-
2114
printf "Authenticating Docker to ECR...\n"
2215
aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 425362996713.dkr.ecr.us-east-1.amazonaws.com
2316

24-
if [ "$ALPINE" = "0" ]; then
25-
printf "Building image\n"
26-
TARGET_IMAGE="Dockerfile.extension_image"
27-
else
28-
printf "Building image for alpine\n"
29-
TARGET_IMAGE="Dockerfile.extension_image.alpine"
30-
fi
31-
17+
# NOTE: this probably does not work the way that we expect it to, especially
18+
# when suffixes are involved. This is a known bug but we don't really check
19+
# anything other than the basic `self-monitoring-lambda-extension:latest` image
20+
# in our self-monitoring, so it's not a thing we're going to fix right now.
3221
LAYER_NAME="Datadog-Extension"
33-
if [ -z "$LAYER_SUFFIX" ]; then
22+
if [ -z "$PIPELINE_LAYER_SUFFIX" ]; then
3423
printf "Building container images tagged without suffix\n"
3524
else
36-
printf "Building container images tagged with suffix: ${LAYER_SUFFIX}\n"
37-
LAYER_NAME="${LAYER_NAME}-${LAYER_SUFFIX}"
25+
printf "Building container images tagged with suffix: ${PIPELINE_LAYER_SUFFIX}\n"
26+
LAYER_NAME="${LAYER_NAME}-${PIPELINE_LAYER_SUFFIX}"
3827
fi
3928

4029
# Increment last version
@@ -44,7 +33,8 @@ printf "Tagging container image with version: $VERSION and latest\n"
4433

4534
docker buildx build \
4635
--platform $PLATFORM \
47-
-f ./images/${TARGET_IMAGE} \
36+
-f ./images/Dockerfile.extension_image \
37+
--build-arg SUFFIX=$SUFFIX \
4838
--tag "$DOCKER_TARGET_IMAGE:${IMAGE_TAG}${SUFFIX}" \
4939
--tag "$DOCKER_TARGET_IMAGE:${VERSION}${SUFFIX}" \
5040
--push .

.gitlab/scripts/check_layer_size.sh

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,16 @@
99

1010
set -e
1111

12+
if [ -z "$MAX_LAYER_COMPRESSED_SIZE_MB" ]; then
13+
printf "[ERROR]: MAX_LAYER_COMPRESSED_SIZE_MB not specified\n"
14+
exit 1
15+
fi
16+
17+
if [ -z "$MAX_LAYER_UNCOMPRESSED_SIZE_MB" ]; then
18+
printf "[ERROR]: MAX_LAYER_UNCOMPRESSED_SIZE_MB not specified\n"
19+
exit 1
20+
fi
21+
1222
validate_size() {
1323
local max_size=$1
1424
local file_size=$2
@@ -23,8 +33,8 @@ if [ -z "$LAYER_FILE" ]; then
2333
exit 1
2434
fi
2535

26-
MAX_LAYER_COMPRESSED_SIZE_KB=$(( 23 * 1024)) # 23 MB, amd64 is 22, while arm64 is 20
27-
MAX_LAYER_UNCOMPRESSED_SIZE_KB=$(( 54 * 1024 )) # 53 MB, amd is 53, while arm64 is 47
36+
MAX_LAYER_COMPRESSED_SIZE_KB=$(( $MAX_LAYER_COMPRESSED_SIZE_MB * 1024))
37+
MAX_LAYER_UNCOMPRESSED_SIZE_KB=$(( $MAX_LAYER_UNCOMPRESSED_SIZE_MB * 1024 ))
2838

2939
FILE=".layers"/$LAYER_FILE
3040
FILE_SIZE=$(stat --printf="%s" "$FILE")

.gitlab/scripts/compile_bottlecap.sh

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,13 @@ else
2424
printf "Alpine compile requested: ${ALPINE}\n"
2525
fi
2626

27+
if [ -z "$FIPS" ]; then
28+
printf "[ERROR]: FIPS not specified\n"
29+
exit 1
30+
else
31+
printf "Fips compile requested: ${FIPS}\n"
32+
fi
33+
2734
if [ "$ALPINE" = "0" ]; then
2835
COMPILE_IMAGE=Dockerfile.bottlecap.compile
2936
else
@@ -58,6 +65,7 @@ docker_build() {
5865
-t datadog/compile-bottlecap \
5966
-f ./images/${file} \
6067
--build-arg PLATFORM=$PLATFORM \
68+
--build-arg FIPS="${FIPS}" \
6169
. -o $BINARY_PATH
6270

6371
# Copy the compiled binary to the target directory with the expected name

.gitlab/scripts/compile_go_agent.sh

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,13 @@ else
3131
printf "Alpine compile requested: ${ALPINE}\n"
3232
fi
3333

34+
if [ -z "$FIPS" ]; then
35+
printf "[ERROR]: FIPS not specified\n"
36+
exit 1
37+
else
38+
printf "Fips compile requested: ${FIPS}\n"
39+
fi
40+
3441
if [ -z "$CI_COMMIT_TAG" ]; then
3542
# Running on dev
3643
printf "Running on dev environment\n"
@@ -49,9 +56,18 @@ else
4956
COMPILE_IMAGE=Dockerfile.go_agent.alpine.compile
5057
fi
5158

59+
if [ -z "$SUFFIX" ]; then
60+
printf "No suffix provided, using ${ARCHITECTURE}\n"
61+
SUFFIX=$ARCHITECTURE
62+
fi
63+
5264
# Allow override build tags
5365
if [ -z "$BUILD_TAGS" ]; then
54-
BUILD_TAGS="serverless otlp"
66+
if [ "$FIPS" = "0" ]; then
67+
BUILD_TAGS="serverless otlp"
68+
else
69+
BUILD_TAGS="serverless otlp serverlessfips"
70+
fi
5571
fi
5672

5773
# Allow override agent path
@@ -89,6 +105,7 @@ function docker_compile {
89105
--build-arg EXTENSION_VERSION="${VERSION}" \
90106
--build-arg AGENT_VERSION="${AGENT_VERSION}" \
91107
--build-arg BUILD_TAGS="${BUILD_TAGS}" \
108+
--build-arg FIPS="${FIPS}" \
92109
. -o $BINARY_PATH
93110

94111
# Copy the compiled binary to the target directory with the expected name

0 commit comments

Comments
 (0)