feat: build Airflow and Superset from source (#1304)

* wip

* add build of UI assets for 3.x

* wip

* feat: build airflow 2 from source

* fix: airflow build

* feat: build superset from source

* chore: move statsd_exporter to final stage

* chore: removed testpatch

* chore: changelog

* chore: lint fix

* chore: use HOME env consistently

---------

Co-authored-by: Andrew Kenworthy <[email protected]>

Author: dervoeti

.github/workflows/build_airflow.yaml

@@ -17,7 +17,7 @@ on:
       # bake --product PRODUCT -d | grep -v 'docker buildx bake' | jq '.target | keys[]'
       - airflow/**
       - vector/**
-      - stackable-base/**
+      - stackable-devel/**
       - .github/actions/**
       - .github/workflows/build_airflow.yaml
       - .github/workflows/reusable_build_image.yaml

.github/workflows/build_superset.yaml

@@ -17,7 +17,7 @@ on:
       # bake --product PRODUCT -d | grep -v 'docker buildx bake' | jq '.target | keys[]'
       - superset/**
       - vector/**
-      - stackable-base/**
+      - stackable-devel/**
       - .github/actions/**
       - .github/workflows/build_superset.yaml
       - .github/workflows/reusable_build_image.yaml

CHANGELOG.md

@@ -31,6 +31,7 @@ All notable changes to this project will be documented in this file.
 - nifi: Add `2.6.0` ([#1293]).
 - hive: Add `4.1.0` ([#1295]).
 - hbase: Add `2.6.3` ([#1296]).
+- airflow,superset: Build from source ([#1304]).
 
 ### Changed
 
@@ -95,6 +96,7 @@ All notable changes to this project will be documented in this file.
 [#1296]: https://github.com/stackabletech/docker-images/pull/1296
 [#1300]: https://github.com/stackabletech/docker-images/pull/1300
 [#1301]: https://github.com/stackabletech/docker-images/pull/1301
+[#1304]: https://github.com/stackabletech/docker-images/pull/1304
 
 ## [25.7.0] - 2025-07-23
 

airflow/Dockerfile

@@ -37,13 +37,13 @@ uv run pytest --disable-warnings
 uv build
 EOF
 
-FROM local-image/vector AS airflow-build-image
+FROM local-image/stackable-devel AS airflow-build-image
 
 ARG PRODUCT_VERSION
-ARG SHARED_STATSD_EXPORTER_VERSION
 ARG PYTHON_VERSION
 ARG TARGETARCH
 ARG STACKABLE_USER_UID
+ARG NODEJS_VERSION
 ARG S3FS_VERSION
 ARG CYCLONEDX_BOM_VERSION
 ARG UV_VERSION
@@ -54,7 +54,8 @@ ARG UV_VERSION
 # Requires implementation of https://github.com/apache/airflow/blob/2.2.5/scripts/docker/install_mysql.sh
 ARG AIRFLOW_EXTRAS
 
-RUN microdnf update && \
+RUN microdnf module enable -y nodejs:${NODEJS_VERSION} && \
+    microdnf update && \
     microdnf install \
         cyrus-sasl-devel \
         # Needed by ./configure to build gevent, see snippet [1] at the end of file
@@ -72,6 +73,9 @@ RUN microdnf update && \
         python${PYTHON_VERSION}-wheel \
         # The airflow odbc provider can compile without the development files (headers and libraries) (see https://github.com/stackabletech/docker-images/pull/683)
         unixODBC \
+        # Needed for Airflow UI assets
+        npm \
+        nodejs \
         # Needed to modify the SBOM
         jq && \
     microdnf clean all && \
@@ -81,6 +85,11 @@ COPY airflow/stackable/constraints/${PRODUCT_VERSION}/constraints-python${PYTHON
 COPY airflow/stackable/constraints/${PRODUCT_VERSION}/build-constraints-python${PYTHON_VERSION}.txt /tmp/build-constraints.txt
 COPY --from=opa-auth-manager-builder /tmp/opa-auth-manager/dist/opa_auth_manager-0.1.0-py3-none-any.whl /tmp/
 
+COPY --chown=${STACKABLE_USER_UID}:0 airflow/stackable/patches/patchable.toml /stackable/src/airflow/stackable/patches/patchable.toml
+COPY --chown=${STACKABLE_USER_UID}:0 airflow/stackable/patches/${PRODUCT_VERSION} /stackable/src/airflow/stackable/patches/${PRODUCT_VERSION}
+
+WORKDIR /stackable
+
 RUN <<EOF
 python${PYTHON_VERSION} -m venv --system-site-packages /stackable/app
 
@@ -90,8 +99,44 @@ source /stackable/app/bin/activate
 # Also install uv to get support for build constraints
 pip install --no-cache-dir --upgrade pip
 pip install --no-cache-dir uv==${UV_VERSION}
+uv tool install hatch
+
+cd "$(/stackable/patchable --images-repo-root=src checkout airflow ${PRODUCT_VERSION})"
+
+if [ -d "./airflow-core" ]; then
+  # Airflow 3.x
+  cd airflow-core/src/airflow/ui
+
+  # build front-end assets
+  npm install -g [email protected]
+  pnpm install --frozen-lockfile
+  pnpm run build
+
+  # build airflow wheel from airflow root folder
+  # this picks up the UI assets from the pnpm build, and the dependencies from the root folder
+  cd ../../..
+  /root/.local/bin/hatch build -t wheel
+  # First install the full apache-airflow package to get all dependencies including database drivers
+  uv pip install --no-cache-dir apache-airflow[${AIRFLOW_EXTRAS}]==${PRODUCT_VERSION} --constraint /tmp/constraints.txt --build-constraints /tmp/build-constraints.txt
+  # Then install the locally built core wheel to override the core package
+  uv pip install --no-cache-dir dist/apache_airflow_core-${PRODUCT_VERSION}-py3-none-any.whl[${AIRFLOW_EXTRAS}] --constraint /tmp/constraints.txt --build-constraints /tmp/build-constraints.txt
+else
+  # Airflow 2.x
+  # build front-end assets
+  cd airflow/www
+  npm install -g [email protected]
+  yarn install --frozen-lockfile
+  yarn run build
+
+  # build airflow wheel from airflow root folder
+  cd ../..
+  /root/.local/bin/hatch build -t wheel
+  # First install the full apache-airflow package to get all dependencies including database drivers
+  uv pip install --no-cache-dir apache-airflow[${AIRFLOW_EXTRAS}]==${PRODUCT_VERSION} --constraint /tmp/constraints.txt --build-constraints /tmp/build-constraints.txt
+  # Then install the locally built wheel to override with patched version
+  uv pip install --no-cache-dir dist/apache_airflow-${PRODUCT_VERSION}-py3-none-any.whl[${AIRFLOW_EXTRAS}] --constraint /tmp/constraints.txt --build-constraints /tmp/build-constraints.txt
+fi
 
-uv pip install --no-cache-dir apache-airflow[${AIRFLOW_EXTRAS}]==${PRODUCT_VERSION} --constraint /tmp/constraints.txt --build-constraints /tmp/build-constraints.txt
 # Needed for pandas S3 integration to e.g. write and read csv and parquet files to/from S3
 uv pip install --no-cache-dir s3fs==${S3FS_VERSION} cyclonedx-bom==${CYCLONEDX_BOM_VERSION}
 # Needed for OIDC
@@ -102,6 +147,7 @@ uv pip install --no-cache-dir /tmp/opa_auth_manager-0.1.0-py3-none-any.whl
 # Create the SBOM for Airflow
 # Important: All `pip install` commands must be above this line, otherwise the SBOM will be incomplete
 cyclonedx-py environment --schema-version 1.5 --outfile /tmp/sbom.json
+uv pip uninstall cyclonedx-bom
 
 # Break circular dependencies by removing the apache-airflow dependency from the providers
 jq '.dependencies |= map(if .ref | test("^apache-airflow-providers-") then
@@ -111,10 +157,6 @@ else
 end)' /tmp/sbom.json > /stackable/app/airflow-${PRODUCT_VERSION}.cdx.json
 EOF
 
-COPY --from=statsd_exporter-builder /statsd_exporter/statsd_exporter /stackable/statsd_exporter
-COPY --from=statsd_exporter-builder /statsd_exporter/statsd_exporter-${SHARED_STATSD_EXPORTER_VERSION}.cdx.json /stackable/statsd_exporter-${SHARED_STATSD_EXPORTER_VERSION}.cdx.json
-COPY --from=gitsync-image --chown=${STACKABLE_USER_UID}:0 /git-sync /stackable/git-sync
-
 RUN <<EOF
 mkdir -pv /stackable/airflow
 mkdir -pv /stackable/airflow/dags
@@ -130,6 +172,7 @@ ARG PYTHON_VERSION
 ARG RELEASE_VERSION
 ARG TINI_VERSION
 ARG TARGETARCH
+ARG SHARED_STATSD_EXPORTER_VERSION
 ARG STACKABLE_USER_UID
 
 LABEL name="Apache Airflow" \
@@ -146,11 +189,14 @@ ENV PATH=$PATH:/bin:$HOME/app/bin
 ENV AIRFLOW_HOME=$HOME/airflow
 
 COPY --from=airflow-build-image --chown=${STACKABLE_USER_UID}:0 /stackable/ ${HOME}/
-COPY --from=airflow-build-image --chown=${STACKABLE_USER_UID}:0 /stackable/git-sync ${HOME}/git-sync
 
 COPY --chown=${STACKABLE_USER_UID}:0 airflow/stackable/utils/entrypoint.sh /entrypoint.sh
 COPY --chown=${STACKABLE_USER_UID}:0 airflow/stackable/utils/run-airflow.sh /run-airflow.sh
 
+COPY --from=statsd_exporter-builder --chown=${STACKABLE_USER_UID}:0 /statsd_exporter/statsd_exporter ${HOME}/statsd_exporter
+COPY --from=statsd_exporter-builder --chown=${STACKABLE_USER_UID}:0 /statsd_exporter/statsd_exporter-${SHARED_STATSD_EXPORTER_VERSION}.cdx.json ${HOME}/statsd_exporter-${SHARED_STATSD_EXPORTER_VERSION}.cdx.json
+COPY --from=gitsync-image --chown=${STACKABLE_USER_UID}:0 /git-sync ${HOME}/git-sync
+
 COPY airflow/licenses /licenses
 
 # Update image and install needed packages
@@ -185,6 +231,7 @@ curl -o /usr/bin/tini "https://repo.stackable.tech/repository/packages/tini/tini
 chmod a+x /entrypoint.sh
 chmod a+x /run-airflow.sh
 chmod +x /usr/bin/tini
+chmod g=u /stackable/statsd_exporter ${HOME}/statsd_exporter-${SHARED_STATSD_EXPORTER_VERSION}.cdx.json ${HOME}/git-sync
 EOF
 
 # ----------------------------------------

airflow/boil-config.toml

@@ -1,6 +1,7 @@
 [versions."2.9.3".local-images]
 "shared/statsd-exporter" = "0.28.0"
 vector = "0.49.0"
+stackable-devel = "1.0.0"
 
 [versions."2.9.3".build-arguments]
 python-version = "3.9"
@@ -11,10 +12,12 @@ tini-version = "0.19.0"
 uv-version = "0.7.8"
 airflow-extras = "async,amazon,celery,cncf.kubernetes,docker,dask,elasticsearch,ftp,grpc,hashicorp,http,ldap,google,google_auth,microsoft.azure,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv,trino"
 opa-auth-manager = "airflow-2"
+nodejs-version = "20"
 
 [versions."2.10.5".local-images]
 "shared/statsd-exporter" = "0.28.0"
 vector = "0.49.0"
+stackable-devel = "1.0.0"
 
 [versions."2.10.5".build-arguments]
 python-version = "3.12"
@@ -25,10 +28,12 @@ tini-version = "0.19.0"
 uv-version = "0.7.8"
 airflow-extras = "async,amazon,celery,cncf.kubernetes,docker,dask,elasticsearch,ftp,grpc,hashicorp,http,ldap,google,google_auth,microsoft.azure,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv,trino"
 opa-auth-manager = "airflow-2"
+nodejs-version = "20"
 
 [versions."3.0.1".local-images]
 "shared/statsd-exporter" = "0.28.0"
 vector = "0.49.0"
+stackable-devel = "1.0.0"
 
 [versions."3.0.1".build-arguments]
 python-version = "3.12"
@@ -39,10 +44,12 @@ tini-version = "0.19.0"
 uv-version = "0.7.8"
 airflow-extras = "async,amazon,celery,cncf-kubernetes,docker,elasticsearch,fab,ftp,grpc,hashicorp,http,ldap,google,microsoft-azure,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,trino"
 opa-auth-manager = "airflow-3"
+nodejs-version = "20"
 
 [versions."3.0.6".local-images]
 "shared/statsd-exporter" = "0.28.0"
 vector = "0.49.0"
+stackable-devel = "1.0.0"
 
 [versions."3.0.6".build-arguments]
 python-version = "3.12"
@@ -53,3 +60,4 @@ tini-version = "0.19.0"
 uv-version = "0.7.8"
 airflow-extras = "async,amazon,celery,cncf-kubernetes,docker,elasticsearch,fab,ftp,grpc,hashicorp,http,ldap,google,microsoft-azure,odbc,pandas,postgres,redis,sendgrid,sftp,slack,ssh,statsd,trino"
 opa-auth-manager = "airflow-3"
+nodejs-version = "20"

airflow/stackable/patches/2.10.5/patchable.toml

@@ -0,0 +1,2 @@
+mirror = "https://github.com/stackabletech/airflow.git"
+base = "b93c3db6b1641b0840bd15ac7d05bc58ff2cccbf"

airflow/stackable/patches/2.9.3/patchable.toml

@@ -0,0 +1,2 @@
+mirror = "https://github.com/stackabletech/airflow.git"
+base = "81845de9d95a733b4eb7826aaabe23ba9813eba3"

airflow/stackable/patches/3.0.1/patchable.toml

@@ -0,0 +1,2 @@
+mirror = "https://github.com/stackabletech/airflow.git"
+base = "4ecebc2973587ebaa2cb12482de82e93d15c092f"

airflow/stackable/patches/3.0.6/patchable.toml

@@ -0,0 +1,2 @@
+mirror = "https://github.com/stackabletech/airflow.git"
+base = "e965c2e676d85ced65a485d4b2601addc2fd3e97"

airflow/stackable/patches/patchable.toml

@@ -0,0 +1,2 @@
+upstream = "https://github.com/apache/airflow.git"
+default-mirror = "https://github.com/stackabletech/airflow.git"