Add exe compression upx (#545)

* remove comments to reduce diff

* add upx compression in the build

* use fastest compession

* pin xz version

* pin curl version too

* use repo upx version

* change size requirements

Author: alexgallotta

.gitlab/scripts/check_layer_size.sh

@@ -9,31 +9,35 @@
 
 set -e
 
+validate_size() {
+  local max_size=$1
+  local file_size=$2
+    if [ "$file_size" -gt "$max_size" ]; then
+        echo "Size $file_size exceeded limit $max_size kb"
+        exit 1
+    fi
+}
+
 if [ -z "$LAYER_FILE" ]; then
     echo "[ERROR]: LAYER_FILE not specified"
     exit 1
 fi
 
-MAX_LAYER_COMPRESSED_SIZE_KB=$(expr 20 \* 1024) # 20 MB, amd64 is 19, while arm64 is 18
-MAX_LAYER_UNCOMPRESSED_SIZE_KB=$(expr 53 \* 1024) # 53 MB, amd is 53, while arm64 is 47
-
-LAYERS_DIR=".layers"
+MAX_LAYER_COMPRESSED_SIZE_KB=$(( 23 * 1024)) # 23 MB, amd64 is 22, while arm64 is 20
+MAX_LAYER_UNCOMPRESSED_SIZE_KB=$(( 30 * 1024)) # 53 MB -> 30MB with UPX
 
-FILE=$LAYERS_DIR/$LAYER_FILE
-FILE_SIZE=$(stat --printf="%s" $FILE)
+FILE=".layers"/$LAYER_FILE
+FILE_SIZE=$(stat --printf="%s" "$FILE")
 FILE_SIZE_KB="$(( ${FILE_SIZE%% *} / 1024))"
-echo "Layer file ${FILE} has zipped size ${FILE_SIZE_KB} kb"
-if [ "$FILE_SIZE_KB" -gt "$MAX_LAYER_COMPRESSED_SIZE_KB" ]; then
-    echo "Zipped size exceeded limit $MAX_LAYER_COMPRESSED_SIZE_KB kb"
-    exit 1
-fi
+
 mkdir tmp
-unzip -q $FILE -d tmp
+unzip -q "$FILE" -d tmp
 UNZIPPED_FILE_SIZE=$(du -shb tmp/ | cut -f1)
 UNZIPPED_FILE_SIZE_KB="$(( ${UNZIPPED_FILE_SIZE%% *} / 1024))"
 rm -rf tmp
+
+echo "Layer file ${FILE} has zipped size ${FILE_SIZE_KB} kb"
 echo "Layer file ${FILE} has unzipped size ${UNZIPPED_FILE_SIZE_KB} kb"
-if [ "$UNZIPPED_FILE_SIZE_KB" -gt "$MAX_LAYER_UNCOMPRESSED_SIZE_KB" ]; then
-    echo "Unzipped size exceeded limit $MAX_LAYER_UNCOMPRESSED_SIZE_KB kb"
-    exit 1
-fi
+
+validate_size "$MAX_LAYER_COMPRESSED_SIZE_KB" $FILE_SIZE_KB
+validate_size "$MAX_LAYER_UNCOMPRESSED_SIZE_KB" $UNZIPPED_FILE_SIZE_KB

scripts/Dockerfile.alpine.build

@@ -55,7 +55,6 @@ WORKDIR /extensions
 COPY --from=builder /tmp/dd/datadog-agent/"${CMD_PATH}"/datadog-agent /extensions/datadog-agent
 RUN strip /extensions/datadog-agent
 
-
 COPY ./scripts/$DATADOG_WRAPPER /$DATADOG_WRAPPER
 RUN chmod +x /$DATADOG_WRAPPER
 RUN  zip -r datadog_extension.zip /extensions /$DATADOG_WRAPPER

scripts/Dockerfile.bottlecap.alpine.build

@@ -32,10 +32,12 @@ ARG DATADOG_WRAPPER=datadog_wrapper
 ARG GO_AGENT_PATH
 
 RUN apt-get update
-RUN apt-get install -y zip binutils
+RUN apt-get install -y zip binutils upx
 
 COPY .layers/$GO_AGENT_PATH/extensions/datadog-agent /datadog-agent-go
-RUN strip /datadog-agent-go # just in case
+RUN strip /datadog-agent-go
+RUN upx -1 /datadog-agent-go
+
 
 RUN mkdir /extensions
 WORKDIR /extensions

scripts/Dockerfile.bottlecap.build

@@ -7,14 +7,10 @@ RUN yum install -y curl gcc gcc-c++ make unzip
 # Install Protocol Buffers compiler by hand, since AL2 does not have a recent enough version.
 COPY ./scripts/install-protoc.sh /
 RUN chmod +x /install-protoc.sh && /install-protoc.sh
-
-# Install Rust Toolchain
 RUN curl https://sh.rustup.rs -sSf | \
     sh -s -- --profile minimal --default-toolchain stable-$PLATFORM-unknown-linux-gnu -y
 ENV PATH=/root/.cargo/bin:$PATH
 RUN rustup component add rust-src --toolchain stable-$PLATFORM-unknown-linux-gnu
-
-# Build Bottlecap
 RUN mkdir -p /tmp/dd
 COPY ./bottlecap/src /tmp/dd/bottlecap/src
 COPY ./bottlecap/Cargo.toml /tmp/dd/bottlecap/Cargo.toml
@@ -24,16 +20,16 @@ WORKDIR /tmp/dd/bottlecap
 RUN --mount=type=cache,target=/usr/local/cargo/registry cargo +stable build --release --target $PLATFORM-unknown-linux-gnu
 RUN cp /tmp/dd/bottlecap/target/$PLATFORM-unknown-linux-gnu/release/bottlecap /tmp/dd/bottlecap/bottlecap
 
-# Zip Extension
 FROM ubuntu:22.04 as compresser
 ARG DATADOG_WRAPPER=datadog_wrapper
 ARG GO_AGENT_PATH
 
 RUN apt-get update
-RUN apt-get install -y zip binutils
-
+RUN apt-get install -y zip binutils upx
 COPY .layers/$GO_AGENT_PATH/extensions/datadog-agent /datadog-agent-go
-RUN strip /datadog-agent-go # just in case
+RUN strip /datadog-agent-go
+RUN upx -1 /datadog-agent-go
+
 
 RUN mkdir /extensions
 WORKDIR /extensions

scripts/Dockerfile.bottlecap.dev

@@ -19,13 +19,15 @@ WORKDIR /tmp/dd/bottlecap
 RUN --mount=type=cache,target=/usr/local/cargo/registry cargo +stable build --release --target $PLATFORM-unknown-linux-gnu
 RUN cp /tmp/dd/bottlecap/target/$PLATFORM-unknown-linux-gnu/release/bottlecap /tmp/dd/bottlecap/bottlecap
 
-# zip the extension
 FROM ubuntu:22.04 as compresser
 ARG DATADOG_WRAPPER=datadog_wrapper
 
 RUN apt-get update
-RUN apt-get install -y zip binutils
+RUN apt-get install -y zip binutils upx
 COPY --from=public.ecr.aws/datadog/lambda-extension:64 /opt/datadog-agent-go  /datadog-agent-go
+RUN du -h /datadog-agent-go
+RUN upx -1 /datadog-agent-go
+RUN du -h /datadog-agent-go
 RUN mkdir /extensions
 WORKDIR /extensions