Make uid/gid configurable & change group of files

.github/workflows/release.yml

@@ -80,7 +80,7 @@ jobs:
         uses: anchore/sbom-action/download-syft@61119d458adab75f756bc0b9e4bde25725f86a7a # v0.17.2
       - name: Install image-tools-stackabletech
         run: pip install image-tools-stackabletech==0.0.13
-      - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0
+      - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
         with:
           registry: docker.stackable.tech
           username: github

airflow/Dockerfile

@@ -99,7 +99,7 @@ rm -rf /var/cache/yum
 
 # Get the correct `tini` binary for our architecture.
 # It is used as an init alternative in the entrypoint
-curl --fail -o /usr/bin/tini "https://repo.stackable.tech/repository/packages/tini/tini-${TINI}-${TARGETARCH}"
+curl -o /usr/bin/tini "https://repo.stackable.tech/repository/packages/tini/tini-${TINI}-${TARGETARCH}"
 chmod a+x /entrypoint.sh
 chmod a+x /run-airflow.sh
 chmod +x /usr/bin/tini

druid/Dockerfile

@@ -7,6 +7,7 @@ ARG JACKSON_DATAFORMAT_XML
 ARG STAX2_API
 ARG WOODSTOX_CORE
 ARG AUTHORIZER
+ARG STACKABLE_USER_UID
 
 # Setting this to anything other than "true" will keep the cache folders around (e.g. for Maven, NPM etc.)
 # This can be used to speed up builds when disk space is of no concern.

hadoop/Dockerfile

@@ -8,12 +8,12 @@ ARG JMX_EXPORTER
 ARG PROTOBUF
 ARG TARGETARCH
 ARG TARGETOS
+ARG STACKABLE_USER_UID
 
-USER ${STACKABLE_USER_UID}
 WORKDIR /stackable
 
-COPY hadoop/stackable/jmx /stackable/jmx
-COPY hadoop/stackable/fuse_dfs_wrapper /stackable/fuse_dfs_wrapper
+COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/jmx /stackable/jmx
+COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/fuse_dfs_wrapper /stackable/fuse_dfs_wrapper
 
 # The symlink from JMX Exporter 0.16.1 to the versionless link exists because old HDFS Operators (up until and including 23.7) used to hardcode
 # the version of JMX Exporter like this: "-javaagent:/stackable/jmx/jmx_prometheus_javaagent-0.16.1.jar"
@@ -54,7 +54,7 @@ RUN microdnf update && \
 
 WORKDIR /stackable
 
-COPY hadoop/stackable/patches /stackable/patches
+COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/patches /stackable/patches
 
 # Hadoop Pipes requires libtirpc to build, whose headers are not packaged in RedHat UBI, so skip building this module
 # Build from source to enable FUSE module, and to apply custom patches.
@@ -92,6 +92,7 @@ FROM stackable/image/java-devel AS hdfs-utils-builder
 
 ARG HDFS_UTILS
 ARG PRODUCT
+ARG STACKABLE_USER_UID
 
 USER ${STACKABLE_USER_UID}
 WORKDIR /stackable
@@ -113,6 +114,7 @@ FROM stackable/image/java-base AS final
 ARG PRODUCT
 ARG RELEASE
 ARG HDFS_UTILS
+ARG STACKABLE_USER_UID
 
 LABEL name="Apache Hadoop" \
       maintainer="[email protected]" \
@@ -126,25 +128,26 @@ COPY --chown=${STACKABLE_USER_UID}:0 --from=builder /stackable/hadoop-${PRODUCT}
 COPY --chown=${STACKABLE_USER_UID}:0 --from=builder /stackable/jmx /stackable/jmx/
 COPY --chown=${STACKABLE_USER_UID}:0 --from=builder /stackable/async-profiler /stackable/async-profiler/
 COPY --chown=${STACKABLE_USER_UID}:0 --from=hdfs-utils-builder /stackable/hadoop-${PRODUCT}/share/hadoop/common/lib/hdfs-utils-${HDFS_UTILS}.jar /stackable/hadoop-${PRODUCT}/share/hadoop/common/lib/hdfs-utils-${HDFS_UTILS}.jar
-COPY hadoop/stackable/fuse_dfs_wrapper /stackable/hadoop/bin
+COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/fuse_dfs_wrapper /stackable/
 
 
 # fuse is required for fusermount (called by fuse_dfs)
 # fuse-libs is required for fuse_dfs (not included in fuse)
 # openssl -> not sure
 RUN <<EOF
 microdnf update
+# tar is required for `kubectl cp` which can be used to copy the log files
+# or profiler flamegraph from the Pod
+# It is already installed in the base image but leaving here for documentation purposes
 microdnf install \
   fuse \
   fuse-libs \
-  # tar is required for `kubectl cp` which can be used to copy the log files
-  # or profiler flamegraph from the Pod
-  # It is already installed in the base image but leaving here for documentation purposes
   tar
 microdnf clean all
 rm -rf /var/cache/yum
 
 ln -s /stackable/hadoop-${PRODUCT} /stackable/hadoop
+mv /stackable/fuse_dfs_wrapper /stackable/hadoop/bin
 
 # Remove unneeded binaries:
 #  - code sources
@@ -173,7 +176,7 @@ EOF
 
 COPY hadoop/licenses /licenses
 
-USER stackable
+USER ${STACKABLE_USER_UID}
 
 ENV HOME=/stackable
 ENV LD_LIBRARY_PATH=/stackable/hadoop/lib/native:/usr/lib/jvm/jre/lib/server

hbase/Dockerfile

@@ -14,21 +14,22 @@ ARG JMX_EXPORTER
 ARG HADOOP
 ARG TARGETARCH
 ARG TARGETOS
+ARG STACKABLE_USER_UID
 
 # Setting this to anything other than "true" will keep the cache folders around (e.g. for Maven, NPM etc.)
 # This can be used to speed up builds when disk space is of no concern.
 ARG DELETE_CACHES="true"
 
 COPY hbase/licenses /licenses
 
-USER stackable
+USER ${STACKABLE_USER_UID}
 WORKDIR /stackable
 
-COPY --chown=stackable:stackable hbase/stackable/patches /stackable/patches
-COPY --chown=stackable:stackable hbase/stackable/jmx/config${JMX_EXPORTER} /stackable/jmx
+COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/patches /stackable/patches
+COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/jmx/config${JMX_EXPORTER} /stackable/jmx
 
 # Cache mounts are owned by root by default
-# We need to explicitly give the uid to use which is hardcoded to "1000" in stackable-base
+# We need to explicitly give the uid to use
 # And every cache needs its own id, we can't share them between stages because we might delete the caches
 # at the end of a run while other stages are still using it.
 # While this might work in theory it didn't in practice (FileNotFound exceptions etc.)
@@ -38,7 +39,7 @@ COPY --chown=stackable:stackable hbase/stackable/jmx/config${JMX_EXPORTER} /stac
 # builder containers will share the same cache and the `rm -rf` commands will fail
 # with a "directory not empty" error on the first builder to finish, as other builders
 # are still working in the cache directory.
-RUN --mount=type=cache,id=maven-hbase-${PRODUCT},uid=1000,target=/stackable/.m2/repository <<EOF
+RUN --mount=type=cache,id=maven-hbase-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository <<EOF
 ###
 ### HBase
 ###
@@ -91,11 +92,12 @@ FROM stackable/image/java-devel AS opa-authorizer-builder
 
 ARG OPA_AUTHORIZER
 ARG DELETE_CACHES
+ARG STACKABLE_USER_UID
 
-USER stackable
+USER ${STACKABLE_USER_UID}
 WORKDIR /stackable
 
-RUN --mount=type=cache,id=maven-opa,uid=1000,target=/stackable/.m2/repository <<EOF
+RUN --mount=type=cache,id=maven-opa,uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository <<EOF
 
 ###
 ### OPA Authorizer (only for 2.6 upwards)
@@ -125,6 +127,7 @@ FROM stackable/image/java-devel AS hbase-operator-tools-builder
 ARG HBASE_OPERATOR_TOOLS
 ARG HBASE_THIRDPARTY
 ARG PRODUCT
+ARG STACKABLE_USER_UID
 
 # Setting this to anything other than "true" will keep the cache folders around (e.g. for Maven, NPM etc.)
 # This can be used to speed up builds when disk space is of no concern.
@@ -134,15 +137,15 @@ ARG DELETE_CACHES="true"
 # The variable names are intentionally passed to envsubst in single-quotes,
 # so that they are not expanded. Disabling ShellCheck rules in a Dockerfile
 # does not work, so please ignore the according warning (SC2016).
-COPY --chown=stackable:stackable hbase/stackable/bin/hbck2.env /stackable/bin/
-COPY --chown=stackable:stackable hbase/stackable/patches /stackable/patches
+COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/bin/hbck2.env /stackable/bin/
+COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/patches /stackable/patches
 
-USER stackable
+USER ${STACKABLE_USER_UID}
 WORKDIR /stackable
 
 # Cache mounts are owned by root by default
-# We need to explicitly give the uid to use which is hardcoded to "1000" in stackable-base
-RUN --mount=type=cache,id=maven-hbase-operator-tools,uid=1000,target=/stackable/.m2/repository <<EOF
+# We need to explicitly give the uid to use
+RUN --mount=type=cache,id=maven-hbase-operator-tools,uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository <<EOF
 
 curl "https://repo.stackable.tech/repository/packages/hbase-operator-tools/hbase-operator-tools-${HBASE_OPERATOR_TOOLS}-src.tar.gz" | tar -xzC .
 mv hbase-operator-tools-${HBASE_OPERATOR_TOOLS} hbase-operator-tools-${HBASE_OPERATOR_TOOLS}-src
@@ -190,16 +193,17 @@ FROM stackable/image/java-devel AS hadoop-s3-builder
 
 ARG PRODUCT
 ARG HADOOP
+ARG STACKABLE_USER_UID
 
-USER stackable
+USER ${STACKABLE_USER_UID}
 WORKDIR /stackable
 
-COPY --from=hadoop-builder --chown=stackable:stackable \
+COPY --from=hadoop-builder --chown=${STACKABLE_USER_UID}:0 \
     /stackable/hadoop/share/hadoop/tools/lib/aws-java-sdk-bundle-*.jar \
     /stackable/hadoop/share/hadoop/tools/lib/hadoop-aws-${HADOOP}.jar \
     /stackable/hadoop/share/hadoop/tools/lib/
 
-COPY --chown=stackable:stackable hbase/stackable/bin/export-snapshot-to-s3.env /stackable/bin/
+COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/bin/export-snapshot-to-s3.env /stackable/bin/
 
 RUN <<EOF
 # Resolve paths in bin/export-snapshot-to-s3
@@ -220,16 +224,17 @@ ARG ASYNC_PROFILER
 ARG PHOENIX
 ARG HBASE_PROFILE
 ARG HADOOP
+ARG STACKABLE_USER_UID
 
 # Setting this to anything other than "true" will keep the cache folders around (e.g. for Maven, NPM etc.)
 # This can be used to speed up builds when disk space is of no concern.
 ARG DELETE_CACHES="true"
 
-COPY --chown=stackable:stackable hbase/stackable/patches /stackable/patches
-USER stackable
+COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/patches /stackable/patches
+USER ${STACKABLE_USER_UID}
 WORKDIR /stackable
 
-RUN --mount=type=cache,id=maven-phoenix,uid=1000,target=/stackable/.m2/repository <<EOF
+RUN --mount=type=cache,id=maven-phoenix,uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository <<EOF
 cd /stackable
 curl "https://repo.stackable.tech/repository/packages/phoenix/phoenix-${PHOENIX}-src.tar.gz" | tar -xzC .
 mv phoenix-${PHOENIX} phoenix-${PHOENIX}-src
@@ -282,6 +287,7 @@ ARG HADOOP
 ARG PHOENIX
 ARG HBASE_PROFILE
 ARG HBASE_OPERATOR_TOOLS
+ARG STACKABLE_USER_UID
 
 ARG NAME="Apache HBase"
 ARG DESCRIPTION="This image is deployed by the Stackable Operator for Apache HBase"
@@ -305,27 +311,27 @@ LABEL io.openshift.tags="ubi9,stackable,hbase,sdp,nosql"
 LABEL io.k8s.description="${DESCRIPTION}"
 LABEL io.k8s.display-name="${NAME}"
 
-COPY --chown=stackable:stackable --from=hbase-builder /stackable/hbase-${PRODUCT} /stackable/hbase-${PRODUCT}/
-COPY --chown=stackable:stackable --from=hbase-builder /stackable/async-profiler /stackable/async-profiler/
-COPY --chown=stackable:stackable --from=hbase-builder /stackable/jmx /stackable/jmx/
+COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-builder /stackable/hbase-${PRODUCT} /stackable/hbase-${PRODUCT}/
+COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-builder /stackable/async-profiler /stackable/async-profiler/
+COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-builder /stackable/jmx /stackable/jmx/
 
-COPY --chown=stackable:stackable --from=hbase-operator-tools-builder /stackable/hbase-operator-tools-${HBASE_OPERATOR_TOOLS} /stackable/hbase-operator-tools-${HBASE_OPERATOR_TOOLS}/
-COPY --chown=stackable:stackable --from=hbase-operator-tools-builder /stackable/bin/hbck2 /stackable/bin/hbck2
+COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-operator-tools-builder /stackable/hbase-operator-tools-${HBASE_OPERATOR_TOOLS} /stackable/hbase-operator-tools-${HBASE_OPERATOR_TOOLS}/
+COPY --chown=${STACKABLE_USER_UID}:0 --from=hbase-operator-tools-builder /stackable/bin/hbck2 /stackable/bin/hbck2
 
-COPY --chown=stackable:stackable --from=phoenix-builder /stackable/phoenix /stackable/phoenix/
+COPY --chown=${STACKABLE_USER_UID}:0 --from=phoenix-builder /stackable/phoenix /stackable/phoenix/
 
-COPY --chown=stackable:stackable --from=hadoop-s3-builder /stackable/bin/export-snapshot-to-s3 /stackable/bin/export-snapshot-to-s3
-COPY --chown=stackable:stackable --from=hadoop-s3-builder /stackable/hadoop/share/hadoop/tools/lib/ /stackable/hadoop/share/hadoop/tools/lib/
+COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-s3-builder /stackable/bin/export-snapshot-to-s3 /stackable/bin/export-snapshot-to-s3
+COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-s3-builder /stackable/hadoop/share/hadoop/tools/lib/ /stackable/hadoop/share/hadoop/tools/lib/
 
 # Copy the dependencies from Hadoop which are required for the Azure Data Lake
 # Storage (ADLS) to /stackable/hbase-${PRODUCT}/lib which is on the classpath.
 # hadoop-azure-${HADOOP}.jar contains the AzureBlobFileSystem which is required
 # by hadoop-common-${HADOOP}.jar if the scheme of a file system is "abfs://".
-COPY --chown=stackable:stackable --from=hadoop-builder \
+COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder \
     /stackable/hadoop/share/hadoop/tools/lib/hadoop-azure-${HADOOP}.jar \
     /stackable/hbase-${PRODUCT}/lib/
 
-COPY --chown=stackable:stackable --from=opa-authorizer-builder /stackable/hbase-opa-authorizer/target/hbase-opa-authorizer*.jar /stackable/hbase-${PRODUCT}/lib
+COPY --chown=${STACKABLE_USER_UID}:0 --from=opa-authorizer-builder /stackable/hbase-opa-authorizer/target/hbase-opa-authorizer*.jar /stackable/hbase-${PRODUCT}/lib
 
 RUN <<EOF
 microdnf update
@@ -346,9 +352,14 @@ rm -rf /var/cache/yum
 ln --symbolic --logical --verbose "/stackable/hbase-${PRODUCT}" /stackable/hbase
 ln --symbolic --logical --verbose "/stackable/hbase-operator-tools-${HBASE_OPERATOR_TOOLS}" /stackable/hbase-operator-tools
 ln --symbolic --logical --verbose "/stackable/phoenix/phoenix-server-hbase-${HBASE_PROFILE}.jar" "/stackable/hbase/lib/phoenix-server-hbase-${HBASE_PROFILE}.jar"
+
+# All files and folders owned by root to support running as arbitrary users
+# This is best practice as all container users will belong to the root group (0)
+chown -R ${STACKABLE_USER_UID}:0 /stackable
+chmod -R g=u /stackable
 EOF
 
-USER stackable
+USER ${STACKABLE_USER_UID}
 ENV HBASE_CONF_DIR=/stackable/hbase/conf
 ENV HOME=/stackable
 ENV PATH="${PATH}:/stackable/bin:/stackable/hbase/bin"

hello-world/Dockerfile

@@ -4,6 +4,7 @@ FROM stackable/image/java-base
 
 ARG PRODUCT
 ARG RELEASE
+ARG STACKABLE_USER_UID
 
 LABEL name="Hello World" \
       maintainer="[email protected]" \
@@ -13,13 +14,20 @@ LABEL name="Hello World" \
       summary="The Stackable image for the Stackable Hello World Operator" \
       description="This image is deployed by the Stackable Hello World Operator."
 
-RUN microdnf update && \
-    microdnf clean all && \
-    rm -rf /var/cache/yum
+RUN <<EOF
+microdnf update
+microdnf clean all
+rm -rf /var/cache/yum
 
-USER stackable
-WORKDIR /stackable
+curl "https://repo.stackable.tech/repository/packages/hello-world/hello-world-${PRODUCT}.jar" -o /stackable/hello-world.jar
+
+# All files and folders owned by root to support running as arbitrary users
+# This is best practice as all container users will belong to the root group (0)
+chown -R ${STACKABLE_USER_UID}:0 /stackable
+chmod -R g=u /stackable
+EOF
 
-RUN curl "https://repo.stackable.tech/repository/packages/hello-world/hello-world-${PRODUCT}.jar" -o hello-world.jar
+USER ${STACKABLE_USER_UID}
+WORKDIR /stackable
 
 CMD ["java", "-jar", "hello-world.jar"]

hive/Dockerfile

@@ -12,19 +12,20 @@ FROM stackable/image/java-devel AS hive-builder
 ARG PRODUCT
 ARG HADOOP
 ARG JMX_EXPORTER
+ARG STACKABLE_USER_UID
 
 # Setting this to anything other than "true" will keep the cache folders around (e.g. for Maven, NPM etc.)
 # This can be used to speed up builds when disk space is of no concern.
 ARG DELETE_CACHES="true"
 
-COPY --chown=stackable:stackable hive/stackable /stackable
+COPY --chown=${STACKABLE_USER_UID}:0 hive/stackable /stackable
 
-USER stackable
+USER ${STACKABLE_USER_UID}
 WORKDIR /stackable
 
 # Cache mounts are owned by root by default
-# We need to explicitly give the uid to use which is hardcoded to "1000" in stackable-base
-RUN --mount=type=cache,id=maven-hive,uid=1000,target=/stackable/.m2/repository <<EOF
+# We need to explicitly give the uid to use
+RUN --mount=type=cache,id=maven-hive-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository <<EOF
 curl "https://repo.stackable.tech/repository/packages/hive/apache-hive-${PRODUCT}-src.tar.gz" | tar -xzC .
 
 patches/apply_patches.sh ${PRODUCT}
@@ -76,6 +77,7 @@ ARG RELEASE
 ARG AWS_JAVA_SDK_BUNDLE
 ARG AZURE_STORAGE
 ARG AZURE_KEYVAULT_CORE
+ARG STACKABLE_USER_UID
 
 
 ARG NAME="Apache Hive metastore"
@@ -107,16 +109,16 @@ rpm -qa --qf "%{NAME}-%{VERSION}-%{RELEASE}\n" | sort > /stackable/package_manif
 rm -rf /var/cache/yum
 EOF
 
-USER stackable
+USER ${STACKABLE_USER_UID}
 WORKDIR /stackable
 
-COPY --chown=stackable:stackable --from=hive-builder /stackable/apache-hive-metastore-${PRODUCT}-bin /stackable/apache-hive-metastore-${PRODUCT}-bin
+COPY --chown=${STACKABLE_USER_UID}:0 --from=hive-builder /stackable/apache-hive-metastore-${PRODUCT}-bin /stackable/apache-hive-metastore-${PRODUCT}-bin
 RUN ln -s /stackable/apache-hive-metastore-${PRODUCT}-bin /stackable/hive-metastore
 
 # It is useful to see which version of Hadoop is used at a glance
 # Therefore the use of the full name here
 # TODO: Do we really need all of Hadoop in here?
-COPY --chown=stackable:stackable --from=hadoop-builder /stackable/hadoop /stackable/hadoop-${HADOOP}
+COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder /stackable/hadoop /stackable/hadoop-${HADOOP}
 RUN ln -s /stackable/hadoop-${HADOOP} /stackable/hadoop
 
 # The next two sections for S3 and Azure use hardcoded version numbers on purpose instead of wildcards
@@ -131,7 +133,7 @@ RUN cp /stackable/hadoop/share/hadoop/tools/lib/hadoop-azure-${HADOOP}.jar /stac
 RUN cp /stackable/hadoop/share/hadoop/tools/lib/azure-storage-${AZURE_STORAGE}.jar /stackable/hive-metastore/lib/
 RUN cp /stackable/hadoop/share/hadoop/tools/lib/azure-keyvault-core-${AZURE_KEYVAULT_CORE}.jar /stackable/hive-metastore/lib/
 
-COPY --chown=stackable:stackable --from=hive-builder /stackable/jmx /stackable/jmx
+COPY --chown=${STACKABLE_USER_UID}:0 --from=hive-builder /stackable/jmx /stackable/jmx
 COPY hive/licenses /licenses
 
 ENV HADOOP_HOME=/stackable/hadoop