diff --git a/CHANGELOG.md b/CHANGELOG.md index 1904596bc..493e812db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ All notable changes to this project will be documented in this file. ### Added +- hadoop: check for correct permissions and ownerships in /stackable folder via + `check-permissions-ownership.sh` provided in stackable-base image ([#1029]). - spark-connect-client: A new image for Spark connect tests and demos ([#1034]) - nifi: check for correct permissions and ownerships in /stackable folder via `check-permissions-ownership.sh` provided in stackable-base image ([#1027]). @@ -16,11 +18,13 @@ All notable changes to this project will be documented in this file. ### Fixed +- hadoop: reduce docker image size by removing the recursive chown/chmods in the final image ([#1029]). - nifi: reduce docker image size by removing the recursive chown/chmods in the final image ([#1027]). - spark-k8s: reduce docker image size by removing the recursive chown/chmods in the final image ([#1042]). - Add `--locked` flag to `cargo install` commands for reproducible builds ([#1044]). [#1027]: https://github.com/stackabletech/docker-images/pull/1027 +[#1029]: https://github.com/stackabletech/docker-images/pull/1029 [#1034]: https://github.com/stackabletech/docker-images/pull/1034 [#1042]: https://github.com/stackabletech/docker-images/pull/1042 [#1044]: https://github.com/stackabletech/docker-images/pull/1044 diff --git a/hadoop/Dockerfile b/hadoop/Dockerfile index acb4e0947..30f129944 100644 --- a/hadoop/Dockerfile +++ b/hadoop/Dockerfile @@ -11,82 +11,100 @@ ARG TARGETARCH ARG TARGETOS ARG STACKABLE_USER_UID -WORKDIR /stackable/jmx - -# The symlink from JMX Exporter 0.16.1 to the versionless link exists because old HDFS Operators (up until and including 23.7) used to hardcode -# the version of JMX Exporter like this: "-javaagent:/stackable/jmx/jmx_prometheus_javaagent-0.16.1.jar" -# This is a TEMPORARY fix which means that we can keep the hardcoded path in HDFS operator FOR NOW as it will still point to a newer version of JMX Exporter, despite the "0.16.1" in the name. -# At the same time a new HDFS Operator will still work with older images which do not have the symlink to the versionless jar. -# After one of our next releases (23.11 or 24.x) we should update the operator to point at the non-versioned symlink (jmx_prometheus_javaagent.jar) -# And then we can also remove the symlink to 0.16.1 from this Dockerfile. -RUN curl "https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" -o "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" && \ - chmod -x "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" && \ - ln -s "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" /stackable/jmx/jmx_prometheus_javaagent.jar && \ - ln -s /stackable/jmx/jmx_prometheus_javaagent.jar /stackable/jmx/jmx_prometheus_javaagent-0.16.1.jar - -WORKDIR /stackable - -RUN ARCH="${TARGETARCH/amd64/x64}" && \ - curl "https://repo.stackable.tech/repository/packages/async-profiler/async-profiler-${ASYNC_PROFILER}-${TARGETOS}-${ARCH}.tar.gz" | tar -xzC . && \ - ln -s "/stackable/async-profiler-${ASYNC_PROFILER}-${TARGETOS}-${ARCH}" /stackable/async-profiler - # This Protobuf version is the exact version as used in the Hadoop Dockerfile # See https://github.com/apache/hadoop/blob/trunk/dev-support/docker/pkg-resolver/install-protobuf.sh # (this was hardcoded in the Dockerfile in earlier versions of Hadoop, make sure to look at the exact version in Github) WORKDIR /opt/protobuf-src -RUN curl https://repo.stackable.tech/repository/packages/protobuf/protobuf-java-${PROTOBUF}.tar.gz -o /opt/protobuf.tar.gz && \ - tar xzf /opt/protobuf.tar.gz --strip-components 1 --no-same-owner && \ - ./configure --prefix=/opt/protobuf && \ - make "-j$(nproc)" && \ - make install && \ +RUN < /stackable/package_manifest.txt +chown ${STACKABLE_USER_UID}:0 /stackable/package_manifest.txt +chmod g=u /stackable/package_manifest.txt rm -rf /var/cache/yum -ln -s /stackable/hadoop-${PRODUCT} /stackable/hadoop -mv /stackable/fuse_dfs_wrapper /stackable/hadoop/bin - -# Remove unneeded binaries: -# - code sources -# - mapreduce/yarn binaries that were built as cross-project dependencies -# - minicluster (only used for testing) and test .jars -# - json-io: this is a transitive dependency pulled in by cedarsoft/java-utils/json-io and is excluded in 3.4.0. See CVE-2023-34610. -rm -rf /stackable/hadoop/share/hadoop/common/sources/ -rm -rf /stackable/hadoop/share/hadoop/hdfs/sources/ -rm -rf /stackable/hadoop/share/hadoop/tools/sources/ -rm -rf /stackable/hadoop/share/hadoop/tools/lib/json-io-*.jar -rm -rf /stackable/hadoop/share/hadoop/tools/lib/hadoop-mapreduce-client-*.jar -rm -rf /stackable/hadoop/share/hadoop/tools/lib/hadoop-yarn-server*.jar -find . -name 'hadoop-minicluster-*.jar' -type f -delete -find . -name 'hadoop-client-minicluster-*.jar' -type f -delete -find . -name 'hadoop-*tests.jar' -type f -delete - # Without this fuse_dfs does not work # It is so non-root users (as we are) can mount a FUSE device and let other users access it echo "user_allow_other" > /etc/fuse.conf - -# All files and folders owned by root group to support running as arbitrary users. -# This is best practice as all container users will belong to the root group (0). -chown -R ${STACKABLE_USER_UID}:0 /stackable -chmod -R g=u /stackable EOF -COPY hadoop/licenses /licenses - # ---------------------------------------- -# Attention: We are changing the group of all files in /stackable directly above -# If you do any file based actions (copying / creating etc.) below this comment you -# absolutely need to make sure that the correct permissions are applied! -# chown ${STACKABLE_USER_UID}:0 +# Checks +# This section is to run final checks to ensure the created final images +# adhere to several minimal requirements like: +# - check file permissions and ownerships +# ---------------------------------------- + +# Check that permissions and ownership in /stackable are set correctly +# This will fail and stop the build if any mismatches are found. +RUN <