chore: [SVLS-6279] fips-flavored builds (#556)

In order to support FIPS flavored go agent builds (added in DataDog/datadog-agent#33799 ) we're making the following changes:
- refactoring the build pipeline to add fips flavors
- rearranging the environment datasource to make it a dictionary instead of a list
- replacing our old publish_govcloud.sh script with a new publish_govcloud_layers.sh script which uses the same publish layers script as the commercial gitlab job

Author: apiarian-datadog

.gitlab-ci.yml

@@ -11,7 +11,7 @@ variables:
   AGENT_VERSION:
     description: "Latest release version of the datadog-agent to tag the build with."
     value: "7.61.0"
-  LAYER_SUFFIX:
+  PIPELINE_LAYER_SUFFIX:
     description: "Suffix to be appended to the layer name (default empty)."
     value: ""
 

.gitlab/datasources/environments.yaml

@@ -1,9 +1,13 @@
 environments:
-  - name: sandbox
+  sandbox:
     external_id: sandbox-publish-externalid
     role_to_assume: sandbox-layer-deployer
     account: 425362996713
-  - name: prod
+    add_layer_version_permissions: 0
+    automatically_bump_version: 1
+  prod:
     external_id: prod-publish-externalid
     role_to_assume: dd-serverless-layer-deployer-role
     account: 464622532012
+    add_layer_version_permissions: 1
+    automatically_bump_version: 0

.gitlab/datasources/flavors.yaml

@@ -2,42 +2,105 @@ flavors:
   - name: amd64
     arch: amd64
     alpine: 0
-    needs_code_checks: true
+    fips: 0
+    needs_layer_sign: true
     needs_layer_publish: true
     suffix: amd64
+    layer_name_base_suffix: ""
+    max_layer_compressed_size_mb: 23
+    max_layer_uncompressed_size_mb: 54
 
   - name: arm64
     arch: arm64
     alpine: 0
-    needs_code_checks: true
+    fips: 0
+    needs_layer_sign: true
     needs_layer_publish: true
     suffix: arm64
+    layer_name_base_suffix: "-ARM"
+    max_layer_compressed_size_mb: 21
+    max_layer_uncompressed_size_mb: 50
 
   - name: amd64, alpine
     arch: amd64
     alpine: 1
-    needs_code_checks: false
+    fips: 0
+    needs_layer_sign: false
     needs_layer_publish: false
     suffix: amd64-alpine
 
   - name: arm64, alpine
     arch: arm64
     alpine: 1
-    needs_code_checks: false
+    fips: 0
+    needs_layer_sign: false
     needs_layer_publish: false
     suffix: arm64-alpine
 
+  - name: amd64, fips
+    arch: amd64
+    alpine: 0
+    fips: 1
+    needs_layer_sign: true
+    needs_layer_publish: false
+    suffix: amd64-fips
+    max_layer_compressed_size_mb: 24
+    max_layer_uncompressed_size_mb: 56
+
+  - name: arm64, fips
+    arch: arm64
+    alpine: 0
+    fips: 1
+    needs_layer_sign: true
+    needs_layer_publish: false
+    suffix: arm64-fips
+    max_layer_compressed_size_mb: 21
+    max_layer_uncompressed_size_mb: 52
+
+  - name: amd64, fips, alpine
+    arch: amd64
+    alpine: 1
+    fips: 1
+    needs_layer_sign: false
+    needs_layer_publish: false
+    suffix: amd64-alpine-fips
+
+  - name: arm64, fips, alpine
+    arch: arm64
+    alpine: 1
+    fips: 1
+    needs_layer_sign: false
+    needs_layer_publish: false
+    suffix: arm64-alpine-fips
+
 # Unfortunately our mutli-arch images don't fit nicely into the flavors
-# structure above.
+# structure above. Since we are making multi-arch images, the suffixes here
+# omit the "arch" component and only include the alpine/fips bits.
 multi_arch_image_flavors:
   - name: basic
     alpine: 0
+    fips: 0
     platform: linux/amd64,linux/arm64
     dependency_names: [amd64, arm64]
     suffix: ""
 
   - name: alpine
     alpine: 1
+    fips: 0
     platform: linux/amd64,linux/arm64
     dependency_names: ["amd64, alpine", "arm64, alpine"]
     suffix: "-alpine"
+
+  - name: fips
+    alpine: 0
+    fips: 1
+    platform: linux/amd64,linux/arm64
+    dependency_names: ["amd64, fips", "arm64, fips"]
+    suffix: "-fips"
+
+  - name: fips, alpine
+    alpine: 1
+    fips: 1
+    platform: linux/amd64,linux/arm64
+    dependency_names: ["amd64, fips, alpine", "arm64, fips, alpine"]
+    suffix: "-alpine-fips"

.gitlab/scripts/build_private_image.sh

@@ -11,30 +11,19 @@ DOCKER_TARGET_IMAGE="425362996713.dkr.ecr.us-east-1.amazonaws.com/self-monitorin
 EXTENSION_DIR=".layers"
 IMAGE_TAG="latest"
 
-if [ -z "$ALPINE" ]; then
-    printf "[ERROR]: ALPINE not specified\n"
-    exit 1
-else
-    printf "Alpine build requested: ${ALPINE}\n"
-fi
-
 printf "Authenticating Docker to ECR...\n"
 aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 425362996713.dkr.ecr.us-east-1.amazonaws.com
 
-if [ "$ALPINE" = "0" ]; then
-    printf "Building image\n"
-    TARGET_IMAGE="Dockerfile.extension_image"
-else
-    printf "Building image for alpine\n"
-    TARGET_IMAGE="Dockerfile.extension_image.alpine"
-fi
-
+# NOTE: this probably does not work the way that we expect it to, especially
+# when suffixes are involved. This is a known bug but we don't really check
+# anything other than the basic `self-monitoring-lambda-extension:latest` image
+# in our self-monitoring, so it's not a thing we're going to fix right now.
 LAYER_NAME="Datadog-Extension"
-if [ -z "$LAYER_SUFFIX" ]; then
+if [ -z "$PIPELINE_LAYER_SUFFIX" ]; then
     printf "Building container images tagged without suffix\n"
 else
-    printf "Building container images tagged with suffix: ${LAYER_SUFFIX}\n"
-    LAYER_NAME="${LAYER_NAME}-${LAYER_SUFFIX}"
+    printf "Building container images tagged with suffix: ${PIPELINE_LAYER_SUFFIX}\n"
+    LAYER_NAME="${LAYER_NAME}-${PIPELINE_LAYER_SUFFIX}"
 fi
 
 # Increment last version
@@ -44,7 +33,8 @@ printf "Tagging container image with version: $VERSION and latest\n"
 
 docker buildx build \
     --platform $PLATFORM \
-    -f ./images/${TARGET_IMAGE} \
+    -f ./images/Dockerfile.extension_image \
+    --build-arg SUFFIX=$SUFFIX \
     --tag "$DOCKER_TARGET_IMAGE:${IMAGE_TAG}${SUFFIX}" \
     --tag "$DOCKER_TARGET_IMAGE:${VERSION}${SUFFIX}" \
     --push .

.gitlab/scripts/check_layer_size.sh

@@ -9,6 +9,16 @@
 
 set -e
 
+if [ -z "$MAX_LAYER_COMPRESSED_SIZE_MB" ]; then
+    printf "[ERROR]: MAX_LAYER_COMPRESSED_SIZE_MB not specified\n"
+    exit 1
+fi
+
+if [ -z "$MAX_LAYER_UNCOMPRESSED_SIZE_MB" ]; then
+    printf "[ERROR]: MAX_LAYER_UNCOMPRESSED_SIZE_MB not specified\n"
+    exit 1
+fi
+
 validate_size() {
   local max_size=$1
   local file_size=$2
@@ -23,8 +33,8 @@ if [ -z "$LAYER_FILE" ]; then
     exit 1
 fi
 
-MAX_LAYER_COMPRESSED_SIZE_KB=$(( 23 * 1024)) # 23 MB, amd64 is 22, while arm64 is 20
-MAX_LAYER_UNCOMPRESSED_SIZE_KB=$(( 54 * 1024 )) # 53 MB, amd is 53, while arm64 is 47
+MAX_LAYER_COMPRESSED_SIZE_KB=$(( $MAX_LAYER_COMPRESSED_SIZE_MB * 1024))
+MAX_LAYER_UNCOMPRESSED_SIZE_KB=$(( $MAX_LAYER_UNCOMPRESSED_SIZE_MB * 1024 ))
 
 FILE=".layers"/$LAYER_FILE
 FILE_SIZE=$(stat --printf="%s" "$FILE")

.gitlab/scripts/compile_bottlecap.sh

@@ -24,6 +24,13 @@ else
     printf "Alpine compile requested: ${ALPINE}\n"
 fi
 
+if [ -z "$FIPS" ]; then
+    printf "[ERROR]: FIPS not specified\n"
+    exit 1
+else
+    printf "Fips compile requested: ${FIPS}\n"
+fi
+
 if [ "$ALPINE" = "0" ]; then
     COMPILE_IMAGE=Dockerfile.bottlecap.compile
 else
@@ -58,6 +65,7 @@ docker_build() {
         -t datadog/compile-bottlecap \
         -f ./images/${file} \
         --build-arg PLATFORM=$PLATFORM \
+        --build-arg FIPS="${FIPS}" \
         . -o $BINARY_PATH
 
     # Copy the compiled binary to the target directory with the expected name

.gitlab/scripts/compile_go_agent.sh

@@ -31,6 +31,13 @@ else
     printf "Alpine compile requested: ${ALPINE}\n"
 fi
 
+if [ -z "$FIPS" ]; then
+    printf "[ERROR]: FIPS not specified\n"
+    exit 1
+else
+    printf "Fips compile requested: ${FIPS}\n"
+fi
+
 if [ -z "$CI_COMMIT_TAG" ]; then
     # Running on dev
     printf "Running on dev environment\n"
@@ -49,9 +56,18 @@ else
     COMPILE_IMAGE=Dockerfile.go_agent.alpine.compile
 fi
 
+if [ -z "$SUFFIX" ]; then
+    printf "No suffix provided, using ${ARCHITECTURE}\n"
+    SUFFIX=$ARCHITECTURE
+fi
+
 # Allow override build tags
 if [ -z "$BUILD_TAGS" ]; then
-    BUILD_TAGS="serverless otlp"
+    if [ "$FIPS" = "0" ]; then
+        BUILD_TAGS="serverless otlp"
+    else
+        BUILD_TAGS="serverless otlp serverlessfips"
+    fi
 fi
 
 # Allow override agent path
@@ -89,6 +105,7 @@ function docker_compile {
         --build-arg EXTENSION_VERSION="${VERSION}" \
         --build-arg AGENT_VERSION="${AGENT_VERSION}" \
         --build-arg BUILD_TAGS="${BUILD_TAGS}" \
+        --build-arg FIPS="${FIPS}" \
         . -o $BINARY_PATH
 
     # Copy the compiled binary to the target directory with the expected name