Build statsd_exporter from source (#777)

* feat: build statsd_exporter from source

* fix: hadolint / comments

* fix: hadolint

* fix: pre-commit lint fixes

* fix: shellcheck

* chore: reverted changes related to druid, will create separate PR for those

* fix: use heredoc and caching

* docs: adapted changelog

* fix: use heredoc for the package installation part

* fix: merge RUN layers

* docs: clarified hadolint ignore comments

Author: dervoeti

CHANGELOG.md

@@ -34,6 +34,7 @@ All notable changes to this project will be documented in this file.
 - superset: Add version `3.1.3` and `4.0.2` ([#768]).
 - druid: Support for 30.0.0 using Java 17 ([#731])
 - hbase: Support for HBase 2.4.18 ([#740]).
+- airflow & superset: Build `statsd_exporter` from source ([#777]).
 
 ### Changed
 
@@ -132,6 +133,7 @@ All notable changes to this project will be documented in this file.
 [#762]: https://github.com/stackabletech/docker-images/pull/762
 [#767]: https://github.com/stackabletech/docker-images/pull/767
 [#768]: https://github.com/stackabletech/docker-images/pull/768
+[#777]: https://github.com/stackabletech/docker-images/pull/777
 
 ## [24.3.0] - 2024-03-20
 

airflow/Dockerfile

@@ -10,11 +10,14 @@ ARG GIT_SYNC
 # which should contain a image location (e.g. registry.k8s.io/git-sync/git-sync:v3.6.8)
 FROM oci.stackable.tech/sdp/git-sync:${GIT_SYNC} as gitsync-image
 
+# Not tagging base image because it is built as part of the same process
+# hadolint ignore=DL3006
+FROM stackable/image/statsd_exporter AS statsd_exporter-builder
+
 FROM stackable/image/vector AS airflow-build-image
 
 ARG PRODUCT
 ARG PYTHON
-ARG STATSD_EXPORTER
 ARG TARGETARCH
 ARG TARGETOS
 
@@ -26,8 +29,6 @@ ENV AIRFLOW_EXTRAS=async,amazon,celery,cncf.kubernetes,docker,dask,elasticsearch
 
 RUN microdnf update && \
     microdnf install \
-        # Used to extract statsd_exporter
-        tar \
         cyrus-sasl-devel \
         # Needed by ./configure to build gevent, see snippet [1] at the end of file
         diffutils \
@@ -53,11 +54,8 @@ RUN python3 -m venv --system-site-packages /stackable/app && \
     # Needed for pandas S3 integration to e.g. write and read csv and parquet files to/from S3
     pip install --no-cache-dir s3fs
 
-# Choosing correct binary with respect to architecture
 WORKDIR /stackable
-RUN curl --fail -L "https://repo.stackable.tech/repository/packages/statsd_exporter/statsd_exporter-${STATSD_EXPORTER}.${TARGETOS}-${TARGETARCH}.tar.gz" | \
-    tar -xzC /stackable && ln -s "statsd_exporter-${STATSD_EXPORTER}.${TARGETOS}-${TARGETARCH}/statsd_exporter" statsd_exporter
-
+COPY --from=statsd_exporter-builder /statsd_exporter/statsd_exporter /stackable/statsd_exporter
 
 FROM stackable/image/vector AS airflow-main-image
 

conf.py

@@ -32,6 +32,7 @@
 testing_tools = importlib.import_module("testing-tools.versions")
 zookeeper = importlib.import_module("zookeeper.versions")
 tools = importlib.import_module("tools.versions")
+statsd_exporter = importlib.import_module("statsd_exporter.versions")
 
 products = [
     {"name": "airflow", "versions": airflow.versions},
@@ -58,6 +59,7 @@
     {"name": "testing-tools", "versions": testing_tools.versions},
     {"name": "zookeeper", "versions": zookeeper.versions},
     {"name": "tools", "versions": tools.versions},
+    {"name": "statsd_exporter", "versions": statsd_exporter.versions},
 ]
 
 open_shift_projects = {

statsd_exporter/Dockerfile

@@ -0,0 +1,37 @@
+# syntax=docker/dockerfile:1.8.1@sha256:e87caa74dcb7d46cd820352bfea12591f3dba3ddc4285e19c7dcd13359f7cefd
+
+# Ignoring DL3038 globally because set `assumeyes=True` in dnf.conf in our base image
+# Ignoring DL4006 globally because we inherit the SHELL from our base image
+# hadolint global ignore=DL3038,DL4006
+
+# Not tagging base image because this is an intermediate image
+# hadolint ignore=DL3006
+FROM stackable/image/stackable-base
+ARG PRODUCT
+
+WORKDIR /statsd_exporter
+
+# Ignoring DL3041 because we don't pin versions
+# Ignoring DL3003 because using `WORKDIR` would require us to split the RUN command into multiple layers
+# hadolint ignore=DL3041,DL3003
+RUN --mount=type=cache,id=go-statsd-exporter,uid=1000,target=/go_cache <<EOF
+microdnf update
+
+# Tar and gzip are used to unpack the statsd_exporter source
+# Golang is used to build statsd_exporter
+microdnf install \
+  tar \
+  gzip \
+  golang
+
+microdnf clean all
+rm -rf /var/cache/yum
+
+export GOPATH=/go_cache
+curl --fail -L "https://repo.stackable.tech/repository/packages/statsd_exporter/statsd_exporter-${PRODUCT}.src.tar.gz" | tar -xzC .
+(
+  cd "statsd_exporter-${PRODUCT}" || exit
+  go build -o ../statsd_exporter
+)
+rm -rf "statsd_exporter-${PRODUCT}"
+EOF

statsd_exporter/README.md

@@ -0,0 +1,7 @@
+# statsd_exporter
+
+This is the Docker image that builds statsd_exporter.
+
+This image does _not_ need to be built manually.
+It will be built automatically if specified as a dependency in our `conf.py` file.
+Which will then be transformed to a dependency in a Docker `buildx bake` file.

.scripts/upload_new_statsd_exporter_version.sh renamed to statsd_exporter/upload_new_statsd_exporter_version.sh

@@ -5,11 +5,6 @@ set -euo pipefail
 VERSION=${1:?"Missing version number argument (arg 1)"}
 NEXUS_USER=${2:?"Missing Nexus username argument (arg 2)"}
 
-ARCHITECTURES=(
-    arm64
-    amd64
-)
-
 read -r -s -p "Nexus Password: " NEXUS_PASSWORD
 echo ""
 
@@ -36,14 +31,11 @@ trap cleanup EXIT
 
 cd "$WORK_DIR" || exit
 
-for arch in "${ARCHITECTURES[@]}"; do
-  # Statsd_exporter does not currently publish signatures or SBOMs
-  echo "Downloading STATSD EXPORTER"
-  curl --fail -LOs "https://github.com/prometheus/statsd_exporter/releases/download/v$VERSION/statsd_exporter-$VERSION.linux-$arch.tar.gz"
+echo "Downloading STATSD EXPORTER source code"
+curl --fail -LOs "https://github.com/prometheus/statsd_exporter/archive/refs/tags/v$VERSION.tar.gz" && mv "v$VERSION.tar.gz" "statsd_exporter-$VERSION.src.tar.gz"
 
-  echo "Uploading to Nexus"
-  curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "statsd_exporter-$VERSION.linux-$arch.tar.gz" 'https://repo.stackable.tech/repository/packages/statsd_exporter/'
-done
+echo "Uploading to Nexus"
+curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "statsd_exporter-$VERSION.src.tar.gz" 'https://repo.stackable.tech/repository/packages/statsd_exporter/'
 
-echo "Successfully uploaded new version of STATSD-EXPORTER ($VERSION) to Nexus"
+echo "Successfully uploaded new version of STATSD-EXPORTER source code ($VERSION) to Nexus"
 echo "https://repo.stackable.tech/service/rest/repository/browse/packages/statsd_exporter/"

statsd_exporter/versions.py

@@ -0,0 +1,6 @@
+versions = [
+    {
+        "product": "0.26.1",
+        "stackable-base": "1.0.0",
+    }
+]

superset/Dockerfile

@@ -4,21 +4,22 @@
 # Ignoring DL4006 globally because we inherit the SHELL from our base image
 # hadolint global ignore=DL3038,DL4006
 
+# Not tagging base image because it is built as part of the same process
+# hadolint ignore=DL3006
+FROM stackable/image/statsd_exporter AS statsd_exporter-builder
+
 FROM stackable/image/vector AS builder
 
 ARG PRODUCT
 ARG PYTHON
 ARG AUTHLIB
-ARG STATSD_EXPORTER
 ARG TARGETARCH
 ARG TARGETOS
 
 COPY superset/constraints-${PRODUCT}.txt /tmp/constraints.txt
 
 RUN microdnf update \
     && microdnf install \
-        # Used to extract statsd_exporter
-        tar \
         cyrus-sasl-devel \
         # Needed to find all patch files, used in `apply_patches.sh`
         findutils \
@@ -75,10 +76,8 @@ RUN python3 -m venv /stackable/app \
 COPY superset/stackable/patches /patches
 RUN /patches/apply_patches.sh ${PRODUCT}
 
-   # Choosing correct binary with respect to architecture
 WORKDIR /stackable
-RUN curl --fail -L "https://repo.stackable.tech/repository/packages/statsd_exporter/statsd_exporter-${STATSD_EXPORTER}.${TARGETOS}-${TARGETARCH}.tar.gz" | \
-    tar -xzC /stackable && ln -s "statsd_exporter-${STATSD_EXPORTER}.${TARGETOS}-${TARGETARCH}/statsd_exporter" statsd_exporter
+COPY --from=statsd_exporter-builder /statsd_exporter/statsd_exporter /stackable/statsd_exporter
 
 # Final image
 FROM stackable/image/vector