Add experimental support for NiFi 2.0.0

CHANGELOG.md

@@ -24,6 +24,7 @@ All notable changes to this project will be documented in this file.
 - spark: Add HBase connector ([#878], [#882]).
 - hbase: hbase-entrypoint.sh script to start and gracefully stop services ([#898]).
 - tools: install yq command line tool for YAML manipulation ([#912]).
+- nifi: Add version 2.0.0 ([#917]).
 
 ### Changed
 
@@ -56,6 +57,7 @@ All notable changes to this project will be documented in this file.
 - zookeeper: Remove `3.8.4` ([#851]).
 - nifi: Remove `1.21.0` and `1.25.0` ([#868]).
 - druid: Remove `28.0.1` ([#880]).
+- nifi: Removed binaries from upload nifi script ([#917]).
 
 ### Fixed
 
@@ -102,6 +104,7 @@ All notable changes to this project will be documented in this file.
 [#912]: https://github.com/stackabletech/docker-images/pull/912
 [#913]: https://github.com/stackabletech/docker-images/pull/913
 [#914]: https://github.com/stackabletech/docker-images/pull/914
+[#917]: https://github.com/stackabletech/docker-images/pull/917
 [#920]: https://github.com/stackabletech/docker-images/pull/920
 
 ## [24.7.0] - 2024-07-24

nifi/Dockerfile

@@ -19,69 +19,41 @@ RUN microdnf update && \
 #
 WORKDIR /tmp
 RUN if [[ "${PRODUCT}" == 2.* ]] ; then \
-        curl "https://repo.stackable.tech/repository/packages/maven/apache-maven-${MAVEN_VERSION}-bin.tar.gz" | tar -xzC . && \
-        ln -sf /tmp/apache-maven-${MAVEN_VERSION}/bin/mvn /usr/bin/mvn ; \
+    curl "https://repo.stackable.tech/repository/packages/maven/apache-maven-${MAVEN_VERSION}-bin.tar.gz" | tar -xzC . && \
+    ln -sf /tmp/apache-maven-${MAVEN_VERSION}/bin/mvn /usr/bin/mvn ; \
     fi
 
 USER ${STACKABLE_USER_UID}
 WORKDIR /stackable
 
 COPY --chown=${STACKABLE_USER_UID}:0 nifi/stackable/patches /stackable/patches
 
-# NOTE: NiFi 1.21.0 source build does not work with the current arm64 git runners due to java heap issues:
-#
-# [ERROR] Failed to execute goal org.apache.maven.plugins:maven-assembly-plugin:3.5.0:single (make shared resource) on project nifi-registry-assembly:
-# Failed to create assembly: Error creating assembly archive bin: Problem creating zip: Execution exception: Java heap space
-#
-# Since this will be deprecated in the release 24.7 and then removed we copy the NiFi 1.21.0 binaries instead
-# of building from source. The if condition can be removed once 1.21.0 is no longer supported and only the
-# else branch is required to build from source.
-#
-RUN if [[ "${PRODUCT}" == "1.21.0" ]] ; then \
-        curl 'https://repo.stackable.tech/repository/m2/tech/stackable/nifi/stackable-bcrypt/1.0-SNAPSHOT/stackable-bcrypt-1.0-20240508.153334-1-jar-with-dependencies.jar' \
-        # This used to be located in /bin/stackable-bcrypt.jar. We create a softlink for /bin/stackable-bcrypt.jar in the main container for backwards compatibility.
-        -o /stackable/stackable-bcrypt.jar && \
-        # zip is different than tar and cannot be just piped, therefore the intermediate save and remove step to unzip
-        curl https://repo.stackable.tech/repository/packages/nifi/nifi-${PRODUCT}-bin.zip -o /stackable/nifi-${PRODUCT}-bin.zip && \
-        unzip /stackable/nifi-${PRODUCT}-bin.zip && \
-        rm /stackable/nifi-${PRODUCT}-bin.zip && \
-        # Remove generated docs in binary
-        rm -rf /stackable/nifi-${PRODUCT}/docs && \
-        # Add Iceberg extensions as they are not included by default and are important enough
-        # They need to be build from source, as https://mvnrepository.com/artifact/org.apache.nifi/nifi-iceberg-processors-nar does not ship the org.apache.hadoop.fs.s3a.S3AFileSystem (see https://github.com/apache/nifi/pull/6368#issuecomment-1502175258)
-        # See https://repo.stackable.tech/repository/packages/nifi/iceberg-nars/README.md for details on how to build them
-        cd /stackable/nifi-${PRODUCT}/lib/ && \
-        curl -O "https://repo.stackable.tech/repository/packages/nifi/iceberg-nars/nifi-iceberg-processors-nar-${PRODUCT}-with-aws.nar" && \
-        curl -O "https://repo.stackable.tech/repository/packages/nifi/iceberg-nars/nifi-iceberg-services-nar-${PRODUCT}-with-aws.nar" && \
-        curl -O "https://repo.stackable.tech/repository/packages/nifi/iceberg-nars/nifi-iceberg-services-api-nar-${PRODUCT}-with-aws.nar" ; \
-    else \
-        curl 'https://repo.stackable.tech/repository/m2/tech/stackable/nifi/stackable-bcrypt/1.0-SNAPSHOT/stackable-bcrypt-1.0-20240508.153334-1-jar-with-dependencies.jar' \
-        # This used to be located in /bin/stackable-bcrypt.jar. We create a softlink for /bin/stackable-bcrypt.jar in the main container for backwards compatibility.
-        -o /stackable/stackable-bcrypt.jar && \
-        # Get the source release from nexus
-        curl "https://repo.stackable.tech/repository/packages/nifi/nifi-${PRODUCT}-source-release.zip" -o "/stackable/nifi-${PRODUCT}-source-release.zip" && \
-        unzip "nifi-${PRODUCT}-source-release.zip" && \
-        # Clean up downloaded source after unzipping
-        rm -rf "nifi-${PRODUCT}-source-release.zip" && \
-        # The NiFi "binary" ends up in a folder named "nifi-${PRODUCT}" which should be copied to /stackable
-        # from /stackable/nifi-${PRODUCT}-src/nifi-assembly/target/nifi-${PRODUCT}-bin/nifi-${PRODUCT} (see later steps)
-        # Therefore we add the suffix "-src" to be able to copy the binary and remove the unzipped sources afterwards.
-        mv nifi-${PRODUCT} nifi-${PRODUCT}-src && \
-        # Apply patches
-        chmod +x patches/apply_patches.sh && \
-        patches/apply_patches.sh ${PRODUCT} && \
-        # Build NiFi
-        cd /stackable/nifi-${PRODUCT}-src/ && \
-        mvn clean install -Dmaven.javadoc.skip=true -DskipTests --activate-profiles include-iceberg,include-hadoop-aws && \
-        # Copy the binaries to the /stackable folder
-        mv /stackable/nifi-${PRODUCT}-src/nifi-assembly/target/nifi-${PRODUCT}-bin/nifi-${PRODUCT} /stackable/nifi-${PRODUCT} && \
-        # Copy the SBOM as well
-        mv /stackable/nifi-${PRODUCT}-src/nifi-assembly/target/bom.json /stackable/nifi-${PRODUCT}/nifi-${PRODUCT}.cdx.json && \
-        # Remove the unzipped sources
-        rm -rf /stackable/nifi-${PRODUCT}-src && \
-        # Remove generated docs in binary
-        rm -rf /stackable/nifi-${PRODUCT}/docs ; \
-    fi
+RUN curl 'https://repo.stackable.tech/repository/m2/tech/stackable/nifi/stackable-bcrypt/1.0-SNAPSHOT/stackable-bcrypt-1.0-20240508.153334-1-jar-with-dependencies.jar' \
+    # This used to be located in /bin/stackable-bcrypt.jar. We create a softlink for /bin/stackable-bcrypt.jar in the main container for backwards compatibility.
+    -o /stackable/stackable-bcrypt.jar && \
+    # Get the source release from nexus
+    curl "https://repo.stackable.tech/repository/packages/nifi/nifi-${PRODUCT}-source-release.zip" -o "/stackable/nifi-${PRODUCT}-source-release.zip" && \
+    unzip "nifi-${PRODUCT}-source-release.zip" && \
+    # Clean up downloaded source after unzipping
+    rm -rf "nifi-${PRODUCT}-source-release.zip" && \
+    # The NiFi "binary" ends up in a folder named "nifi-${PRODUCT}" which should be copied to /stackable
+    # from /stackable/nifi-${PRODUCT}-src/nifi-assembly/target/nifi-${PRODUCT}-bin/nifi-${PRODUCT} (see later steps)
+    # Therefore we add the suffix "-src" to be able to copy the binary and remove the unzipped sources afterwards.
+    mv nifi-${PRODUCT} nifi-${PRODUCT}-src && \
+    # Apply patches
+    chmod +x patches/apply_patches.sh && \
+    patches/apply_patches.sh ${PRODUCT} && \
+    # Build NiFi
+    cd /stackable/nifi-${PRODUCT}-src/ && \
+    mvn clean install -Dmaven.javadoc.skip=true -DskipTests --activate-profiles include-iceberg,include-hadoop-aws && \
+    # Copy the binaries to the /stackable folder
+    mv /stackable/nifi-${PRODUCT}-src/nifi-assembly/target/nifi-${PRODUCT}-bin/nifi-${PRODUCT} /stackable/nifi-${PRODUCT} && \
+    # Copy the SBOM as well
+    mv /stackable/nifi-${PRODUCT}-src/nifi-assembly/target/bom.json /stackable/nifi-${PRODUCT}/nifi-${PRODUCT}.cdx.json && \
+    # Remove the unzipped sources
+    rm -rf /stackable/nifi-${PRODUCT}-src && \
+    # Remove generated docs in binary
+    rm -rf /stackable/nifi-${PRODUCT}/docs
 
 FROM stackable/image/java-base AS final
 
@@ -90,12 +62,12 @@ ARG RELEASE
 ARG STACKABLE_USER_UID
 
 LABEL name="Apache NiFi" \
-      maintainer="[email protected]" \
-      vendor="Stackable GmbH" \
-      version="${PRODUCT}" \
-      release="${RELEASE}" \
-      summary="The Stackable image for Apache NiFi." \
-      description="This image is deployed by the Stackable Operator for Apache NiFi."
+    maintainer="[email protected]" \
+    vendor="Stackable GmbH" \
+    version="${PRODUCT}" \
+    release="${RELEASE}" \
+    summary="The Stackable image for Apache NiFi." \
+    description="This image is deployed by the Stackable Operator for Apache NiFi."
 
 COPY --chown=${STACKABLE_USER_UID}:0 --from=nifi-builder /stackable/nifi-${PRODUCT} /stackable/nifi-${PRODUCT}/
 COPY --chown=${STACKABLE_USER_UID}:0 --from=nifi-builder /stackable/stackable-bcrypt.jar /stackable/stackable-bcrypt.jar

nifi/stackable/patches/2.0.0-M4/001-NIFI-no-zip-assembly-2.0.0-M4.patch → nifi/stackable/patches/2.0.0/001-NIFI-no-zip-assembly-2.0.0.patch

@@ -1,5 +1,5 @@
 diff --git a/nifi-assembly/pom.xml b/nifi-assembly/pom.xml
-index 83eb8214f9..0764b3716d 100644
+index e980e507c6..cb19c89367 100644
 --- a/nifi-assembly/pom.xml
 +++ b/nifi-assembly/pom.xml
 @@ -66,7 +66,6 @@ language governing permissions and limitations under the License. -->

nifi/stackable/patches/2.0.0-M4/002-NIFI-no-host-header-check-2.0.0-M4.patch → nifi/stackable/patches/2.0.0/002-NIFI-no-host-header-check-2.0.0.patch

@@ -26,7 +26,7 @@ Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
 <+>UTF-8
 ===================================================================
 diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/HostHeaderHandler.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/HostHeaderHandler.java
-index 97337d63e2..0f7a272de7 100644
+index 97337d63e2..12ce1d8646 100644
 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/HostHeaderHandler.java
 +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/HostHeaderHandler.java
 @@ -47,6 +47,7 @@ public class HostHeaderHandler extends Handler.Abstract {

nifi/stackable/patches/2.0.0-M4/003-patch-cyclonedx-plugin.patch → nifi/stackable/patches/2.0.0/003-patch-cyclonedx-plugin.patch

@@ -1,8 +1,8 @@
 diff --git a/pom.xml b/pom.xml
-index 6af359c..0be2176 100644
+index 3190206e88..7e754f691d 100644
 --- a/pom.xml
 +++ b/pom.xml
-@@ -954,6 +954,24 @@
+@@ -974,6 +974,24 @@
                      </excludes>
                  </configuration>
              </plugin>

nifi/upload_new_nifi_version.sh

@@ -32,30 +32,28 @@ trap cleanup EXIT
 cd "$WORK_DIR" || exit
 
 src_file="nifi-$VERSION-source-release.zip"
-bin_file="nifi-$VERSION-bin.zip"
 download_url="https://archive.apache.org/dist/nifi/${VERSION}"
 
 echo "Downloading NiFi source (this can take a while, it is intentionally downloading from a slow mirror that contains all old versions)"
 curl --fail -LOs "${download_url}/${src_file}"
 curl --fail -LOs "${download_url}/${src_file}.asc"
 curl --fail -LOs "${download_url}/${src_file}.sha512"
 
-echo "Downloading NiFi binary (this can take a while, it is intentionally downloading from a slow mirror that contains all old versions)"
-curl --fail -LOs "${download_url}/${bin_file}"
-curl --fail -LOs "${download_url}/${bin_file}.asc"
-curl --fail -LOs "${download_url}/${bin_file}.sha512"
-
 # It is probably redundant to check both the checksum and the signature but it's cheap and why not
 echo "Validating SHA512 Checksums"
-if ! (sha512sum "$bin_file" | cut -d ' ' -f 1 | diff - "$bin_file.sha512" && sha512sum "$src_file" | cut -d ' ' -f 1 | diff - "$src_file.sha512"); then
+# The 'echo -e "$(<"${src_file}".sha512)")' part removes possible new lines in the provided .sha512 file.
+# This is due to the NiFi sha512 files sometimes ending on newline and sometimes dont.
+# See https://archive.apache.org/dist/nifi/2.0.0/nifi-2.0.0-source-release.zip.sha512 vs
+#     https://archive.apache.org/dist/nifi/1.27.0/nifi-1.27.0-source-release.zip.sha512
+if ! (sha512sum "$src_file" | cut -d ' ' -f 1 | diff - <(echo -e "$(<"${src_file}".sha512)")); then
   echo "ERROR: One of the SHA512 sums does not match"
   exit 1
 fi
 
 echo "Validating signatures"
 echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://archive.apache.org/dist/nifi/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using "curl https://archive.apache.org/dist/nifi/KEYS | gpg --import")'
 
-if ! (gpg --verify "$bin_file.asc" "$bin_file" 2> /dev/null && gpg --verify "$src_file.asc" "$src_file" 2> /dev/null); then
+if ! (gpg --verify "$src_file.asc" "$src_file" 2> /dev/null); then
   echo "ERROR: One of the signatures could not be verified"
   exit 1
 fi
@@ -66,10 +64,6 @@ curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}" 'https:
 curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}.asc" 'https://repo.stackable.tech/repository/packages/nifi/' || EXIT_STATUS=$?
 curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}.sha512" 'https://repo.stackable.tech/repository/packages/nifi/' || EXIT_STATUS=$?
 
-curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}" 'https://repo.stackable.tech/repository/packages/nifi/' || EXIT_STATUS=$?
-curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}.asc" 'https://repo.stackable.tech/repository/packages/nifi/' || EXIT_STATUS=$?
-curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}.sha512" 'https://repo.stackable.tech/repository/packages/nifi/' || EXIT_STATUS=$?
-
 if [ $EXIT_STATUS -ne 0 ]; then
   echo "ERROR: Upload failed"
   exit 1

nifi/versions.py

@@ -5,7 +5,7 @@
         "java-devel": "11",  # There is an error when trying to use the jdk 21 (since nifi 1.26.0)
     },
     {
-        "product": "2.0.0-M4",
+        "product": "2.0.0",
         "java-base": "21",
         "java-devel": "21",
     },