Merge remote-tracking branch 'origin/main' into fix/spark-k8s-fix-size-consolidation

Author: maltesander

CHANGELOG.md

@@ -6,11 +6,31 @@ All notable changes to this project will be documented in this file.
 
 ### Added
 
+- airflow: check for correct permissions and ownerships in /stackable folder via
+  `check-permissions-ownership.sh` provided in stackable-base image ([#1054]).
+- druid: check for correct permissions and ownerships in /stackable folder via
+  `check-permissions-ownership.sh` provided in stackable-base image ([#1039]).
+- hadoop: check for correct permissions and ownerships in /stackable folder via
+  `check-permissions-ownership.sh` provided in stackable-base image ([#1029]).
+- hbase: check for correct permissions and ownerships in /stackable folder via
+  `check-permissions-ownership.sh` provided in stackable-base image ([#1028]).
+- hive: check for correct permissions and ownerships in /stackable folder via
+  `check-permissions-ownership.sh` provided in stackable-base image ([#1040]).
 - spark-connect-client: A new image for Spark connect tests and demos ([#1034])
+- kafka: check for correct permissions and ownerships in /stackable folder via
+  `check-permissions-ownership.sh` provided in stackable-base image ([#1041]).
 - nifi: check for correct permissions and ownerships in /stackable folder via
   `check-permissions-ownership.sh` provided in stackable-base image ([#1027]).
+- opa: check for correct permissions and ownerships in /stackable folder via
+  `check-permissions-ownership.sh` provided in stackable-base image ([#1038]).
 - spark-k8s: check for correct permissions and ownerships in /stackable folder via
   `check-permissions-ownership.sh` provided in stackable-base image ([#1055]).
+- superset: check for correct permissions and ownerships in /stackable folder via
+  `check-permissions-ownership.sh` provided in stackable-base image ([#1053]).
+- trino: check for correct permissions and ownerships in /stackable folder via
+  `check-permissions-ownership.sh` provided in stackable-base image ([#1025]).
+- zookeeper: check for correct permissions and ownerships in /stackable folder via
+  `check-permissions-ownership.sh` provided in stackable-base image ([#1043]).
 
 ### Changed
 
@@ -19,15 +39,33 @@ All notable changes to this project will be documented in this file.
 
 ### Fixed
 
+- druid: reduce docker image size by removing the recursive chown/chmods in the final image ([#1039]).
+- hadoop: reduce docker image size by removing the recursive chown/chmods in the final image ([#1029]).
+- hbase: reduce docker image size by removing the recursive chown/chmods in the final image ([#1028]).
+- hive: reduce docker image size by removing the recursive chown/chmods in the final image ([#1040]).
+- kafka: reduce docker image size by removing the recursive chown/chmods in the final image ([#1041]).
+- Add `--locked` flag to `cargo install` commands for reproducible builds ([#1044]).
 - nifi: reduce docker image size by removing the recursive chown/chmods in the final image ([#1027]).
+- opa: reduce docker image size by removing the recursive chown/chmods in the final image ([#1038]).
 - spark-k8s: reduce docker image size by removing the recursive chown/chmods in the final image ([#1042]).
-- Add `--locked` flag to `cargo install` commands for reproducible builds ([#1044]).
+- trino: reduce docker image size by removing the recursive chown/chmods in the final image ([#1025]).
+- zookeeper: reduce docker image size by removing the recursive chown/chmods in the final image ([#1043]).
 
+[#1025]: https://github.com/stackabletech/docker-images/pull/1025
 [#1027]: https://github.com/stackabletech/docker-images/pull/1027
+[#1028]: https://github.com/stackabletech/docker-images/pull/1028
+[#1029]: https://github.com/stackabletech/docker-images/pull/1029
 [#1034]: https://github.com/stackabletech/docker-images/pull/1034
+[#1038]: https://github.com/stackabletech/docker-images/pull/1038
+[#1039]: https://github.com/stackabletech/docker-images/pull/1039
+[#1040]: https://github.com/stackabletech/docker-images/pull/1040
+[#1041]: https://github.com/stackabletech/docker-images/pull/1041
 [#1042]: https://github.com/stackabletech/docker-images/pull/1042
+[#1043]: https://github.com/stackabletech/docker-images/pull/1043
 [#1044]: https://github.com/stackabletech/docker-images/pull/1044
 [#1050]: https://github.com/stackabletech/docker-images/pull/1050
+[#1053]: https://github.com/stackabletech/docker-images/pull/1053
+[#1054]: https://github.com/stackabletech/docker-images/pull/1054
 [#1055]: https://github.com/stackabletech/docker-images/pull/1055
 
 ## [25.3.0] - 2025-03-21

airflow/Dockerfile

@@ -28,6 +28,7 @@ ARG PRODUCT
 ARG STATSD_EXPORTER
 ARG PYTHON
 ARG TARGETARCH
+ARG STACKABLE_USER_UID
 
 COPY airflow/constraints-${PRODUCT}-python${PYTHON}.txt /tmp/constraints.txt
 COPY --from=opa-auth-manager-builder /tmp/opa-auth-manager/dist/opa_auth_manager-0.1.0-py3-none-any.whl /tmp/
@@ -85,9 +86,17 @@ else
 end)' /tmp/sbom.json > /stackable/app/airflow-${PRODUCT}.cdx.json
 EOF
 
-WORKDIR /stackable
 COPY --from=statsd_exporter-builder /statsd_exporter/statsd_exporter /stackable/statsd_exporter
 COPY --from=statsd_exporter-builder /statsd_exporter/statsd_exporter-${STATSD_EXPORTER}.cdx.json /stackable/statsd_exporter-${STATSD_EXPORTER}.cdx.json
+COPY --from=gitsync-image --chown=${STACKABLE_USER_UID}:0 /git-sync /stackable/git-sync
+
+RUN <<EOF
+mkdir -pv /stackable/airflow
+mkdir -pv /stackable/airflow/dags
+mkdir -pv /stackable/airflow/logs
+chmod --recursive g=u /stackable
+EOF
+
 
 FROM stackable/image/vector AS airflow-main-image
 
@@ -99,22 +108,26 @@ ARG TARGETARCH
 ARG STACKABLE_USER_UID
 
 LABEL name="Apache Airflow" \
-      maintainer="[email protected]" \
-      vendor="Stackable GmbH" \
-      version="${PRODUCT}" \
-      release="${RELEASE}" \
-      summary="The Stackable image for Apache Airflow." \
-      description="This image is deployed by the Stackable Operator for Apache Airflow."
-
-COPY airflow/licenses /licenses
-COPY --chown=${STACKABLE_USER_UID}:0 airflow/stackable/utils/entrypoint.sh /entrypoint.sh
-COPY --chown=${STACKABLE_USER_UID}:0 airflow/stackable/utils/run-airflow.sh /run-airflow.sh
+    maintainer="[email protected]" \
+    vendor="Stackable GmbH" \
+    version="${PRODUCT}" \
+    release="${RELEASE}" \
+    summary="The Stackable image for Apache Airflow." \
+    description="This image is deployed by the Stackable Operator for Apache Airflow."
 
 ENV HOME=/stackable
 ENV AIRFLOW_USER_HOME_DIR=/stackable
 ENV PATH=$PATH:/bin:$HOME/app/bin
 ENV AIRFLOW_HOME=$HOME/airflow
 
+COPY --from=airflow-build-image --chown=${STACKABLE_USER_UID}:0 /stackable/ ${HOME}/
+COPY --from=airflow-build-image --chown=${STACKABLE_USER_UID}:0 /stackable/git-sync ${HOME}/git-sync
+
+COPY --chown=${STACKABLE_USER_UID}:0 airflow/stackable/utils/entrypoint.sh /entrypoint.sh
+COPY --chown=${STACKABLE_USER_UID}:0 airflow/stackable/utils/run-airflow.sh /run-airflow.sh
+
+COPY airflow/licenses /licenses
+
 # Update image and install needed packages
 RUN <<EOF
 microdnf update
@@ -142,33 +155,33 @@ rm -rf /var/cache/yum
 # Get the correct `tini` binary for our architecture.
 # It is used as an init alternative in the entrypoint
 curl -o /usr/bin/tini "https://repo.stackable.tech/repository/packages/tini/tini-${TINI}-${TARGETARCH}"
+
+# fix missing permissions
 chmod a+x /entrypoint.sh
 chmod a+x /run-airflow.sh
 chmod +x /usr/bin/tini
+EOF
 
-mkdir -pv ${AIRFLOW_HOME}
-mkdir -pv ${AIRFLOW_HOME}/dags
-mkdir -pv ${AIRFLOW_HOME}/logs
+# ----------------------------------------
+# Checks
+# This section is to run final checks to ensure the created final images
+# adhere to several minimal requirements like:
+# - check file permissions and ownerships
+# ----------------------------------------
 
-# All files and folders owned by root to support running as arbitrary users
-# This is best practice as all container users will belong to the root group (0)
-chown -R ${STACKABLE_USER_UID}:0 /stackable
-chmod -R g=u /stackable
+# Check that permissions and ownership in ${HOME} are set correctly
+# This will fail and stop the build if any mismatches are found.
+RUN <<EOF
+/bin/check-permissions-ownership.sh ${HOME} ${STACKABLE_USER_UID} 0
 EOF
 
 # ----------------------------------------
-# Attention: We are changing the group of all files in /stackable directly above
-# If you do any file based actions (copying / creating etc.) below this comment you
-# absolutely need to make sure that the correct permissions are applied!
-# chown ${STACKABLE_USER_UID}:0
+# Attention: Do not perform any file based actions (copying/creating etc.) below this comment because the permissions would not be checked.
 # ----------------------------------------
 
 USER ${STACKABLE_USER_UID}
 WORKDIR /stackable
 
-COPY --from=airflow-build-image --chown=${STACKABLE_USER_UID}:0 /stackable/ ${HOME}/
-COPY --from=gitsync-image --chown=${STACKABLE_USER_UID}:0 /git-sync /stackable/git-sync
-
 ENTRYPOINT ["/usr/bin/tini", "--", "/run-airflow.sh"]
 CMD []
 

druid/Dockerfile

@@ -25,8 +25,8 @@ microdnf update
 #
 # patch: Required for the apply-patches.sh script
 microdnf install \
-python-pyyaml \
-patch
+  python-pyyaml \
+  patch
 
 microdnf clean all
 rm -rf /var/cache/yum
@@ -47,9 +47,9 @@ COPY --chown=stackable:0 druid/stackable/patches/${PRODUCT} /stackable/apache-dr
 # are still working in the cache directory.
 
 RUN --mount=type=cache,id=maven-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository \
-    --mount=type=cache,id=npm-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.npm \
-    --mount=type=cache,id=cache-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.cache \
-    <<EOF
+  --mount=type=cache,id=npm-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.npm \
+  --mount=type=cache,id=cache-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.cache \
+  <<EOF
 curl "https://repo.stackable.tech/repository/packages/druid/apache-druid-${PRODUCT}-src.tar.gz" | tar -xzC .
 cd apache-druid-${PRODUCT}-src
 ./patches/apply_patches.sh ${PRODUCT}
@@ -74,6 +74,9 @@ fi
 
 # Install OPA authorizer extension.
 curl "https://repo.stackable.tech/repository/packages/druid/druid-opa-authorizer-${AUTHORIZER}.tar.gz" | tar -xzC /stackable/apache-druid-${PRODUCT}/extensions
+
+# change groups
+chmod -R g=u /stackable
 EOF
 
 FROM stackable/image/java-base AS final
@@ -106,32 +109,45 @@ LABEL io.k8s.display-name="${NAME}"
 
 
 COPY --chown=${STACKABLE_USER_UID}:0 --from=druid-builder /stackable/apache-druid-${PRODUCT} /stackable/apache-druid-${PRODUCT}
+
 COPY --chown=${STACKABLE_USER_UID}:0 druid/stackable/bin /stackable/bin
 COPY --chown=${STACKABLE_USER_UID}:0 druid/licenses /licenses
 
 RUN <<EOF
 microdnf update
 microdnf clean all
 rpm -qa --qf "%{NAME}-%{VERSION}-%{RELEASE}\n" | sort > /stackable/package_manifest.txt
+chown ${STACKABLE_USER_UID}:0 /stackable/package_manifest.txt
+chmod g=u /stackable/package_manifest.txt
 rm -rf /var/cache/yum
 
-ln -s /stackable/apache-druid-${PRODUCT} /stackable/druid
+ln -sf /stackable/apache-druid-${PRODUCT} /stackable/druid
+chown -h ${STACKABLE_USER_UID}:0 stackable/druid
 
 # Force to overwrite the existing 'run-druid'
 ln -sf /stackable/bin/run-druid /stackable/druid/bin/run-druid
+chown -h ${STACKABLE_USER_UID}:0 /stackable/druid/bin/run-druid
 
-# All files and folders owned by root group to support running as arbitrary users.
-# This is best practice as all container users will belong to the root group (0).
-chown -R ${STACKABLE_USER_UID}:0 /stackable
-chmod -R g=u /stackable
+# fix missing permissions
+chmod -R g=u /stackable/bin
+chmod g=u /stackable/apache-druid-${PRODUCT}
 EOF
 
 # ----------------------------------------
-# Attention: We are changing the group of all files in /stackable directly above
-# If you do any file based actions (copying / creating etc.) below this comment you
-# absolutely need to make sure that the correct permissions are applied!
-# chown ${STACKABLE_USER_UID}:0
+# Checks
+# This section is to run final checks to ensure the created final images
+# adhere to several minimal requirements like:
+# - check file permissions and ownerships
+# ----------------------------------------
+
+# Check that permissions and ownership in /stackable are set correctly
+# This will fail and stop the build if any mismatches are found.
+RUN <<EOF
+/bin/check-permissions-ownership.sh /stackable ${STACKABLE_USER_UID} 0
+EOF
+
 # ----------------------------------------
+# Attention: Do not perform any file based actions (copying/creating etc.) below this comment because the permissions would not be checked.
 
 USER ${STACKABLE_USER_UID}
 ENV PATH="${PATH}":/stackable/druid/bin