diff --git a/CHANGELOG.md b/CHANGELOG.md index 77f4e5427..7c8ed5a83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ All notable changes to this project will be documented in this file. - spark: Add HBase connector ([#878], [#882]). - hbase: hbase-entrypoint.sh script to start and gracefully stop services ([#898]). - tools: install yq command line tool for YAML manipulation ([#912]). +- nifi: Add version 2.0.0 ([#917]). ### Changed @@ -56,6 +57,7 @@ All notable changes to this project will be documented in this file. - zookeeper: Remove `3.8.4` ([#851]). - nifi: Remove `1.21.0` and `1.25.0` ([#868]). - druid: Remove `28.0.1` ([#880]). +- nifi: Removed binaries from upload nifi script ([#917]). ### Fixed @@ -102,6 +104,7 @@ All notable changes to this project will be documented in this file. [#912]: https://github.com/stackabletech/docker-images/pull/912 [#913]: https://github.com/stackabletech/docker-images/pull/913 [#914]: https://github.com/stackabletech/docker-images/pull/914 +[#917]: https://github.com/stackabletech/docker-images/pull/917 [#920]: https://github.com/stackabletech/docker-images/pull/920 ## [24.7.0] - 2024-07-24 diff --git a/nifi/Dockerfile b/nifi/Dockerfile index 4fd145c65..fb94a9554 100644 --- a/nifi/Dockerfile +++ b/nifi/Dockerfile @@ -19,8 +19,8 @@ RUN microdnf update && \ # WORKDIR /tmp RUN if [[ "${PRODUCT}" == 2.* ]] ; then \ - curl "https://repo.stackable.tech/repository/packages/maven/apache-maven-${MAVEN_VERSION}-bin.tar.gz" | tar -xzC . && \ - ln -sf /tmp/apache-maven-${MAVEN_VERSION}/bin/mvn /usr/bin/mvn ; \ + curl "https://repo.stackable.tech/repository/packages/maven/apache-maven-${MAVEN_VERSION}-bin.tar.gz" | tar -xzC . && \ + ln -sf /tmp/apache-maven-${MAVEN_VERSION}/bin/mvn /usr/bin/mvn ; \ fi USER ${STACKABLE_USER_UID} @@ -28,60 +28,32 @@ WORKDIR /stackable COPY --chown=${STACKABLE_USER_UID}:0 nifi/stackable/patches /stackable/patches -# NOTE: NiFi 1.21.0 source build does not work with the current arm64 git runners due to java heap issues: -# -# [ERROR] Failed to execute goal org.apache.maven.plugins:maven-assembly-plugin:3.5.0:single (make shared resource) on project nifi-registry-assembly: -# Failed to create assembly: Error creating assembly archive bin: Problem creating zip: Execution exception: Java heap space -# -# Since this will be deprecated in the release 24.7 and then removed we copy the NiFi 1.21.0 binaries instead -# of building from source. The if condition can be removed once 1.21.0 is no longer supported and only the -# else branch is required to build from source. -# -RUN if [[ "${PRODUCT}" == "1.21.0" ]] ; then \ - curl 'https://repo.stackable.tech/repository/m2/tech/stackable/nifi/stackable-bcrypt/1.0-SNAPSHOT/stackable-bcrypt-1.0-20240508.153334-1-jar-with-dependencies.jar' \ - # This used to be located in /bin/stackable-bcrypt.jar. We create a softlink for /bin/stackable-bcrypt.jar in the main container for backwards compatibility. - -o /stackable/stackable-bcrypt.jar && \ - # zip is different than tar and cannot be just piped, therefore the intermediate save and remove step to unzip - curl https://repo.stackable.tech/repository/packages/nifi/nifi-${PRODUCT}-bin.zip -o /stackable/nifi-${PRODUCT}-bin.zip && \ - unzip /stackable/nifi-${PRODUCT}-bin.zip && \ - rm /stackable/nifi-${PRODUCT}-bin.zip && \ - # Remove generated docs in binary - rm -rf /stackable/nifi-${PRODUCT}/docs && \ - # Add Iceberg extensions as they are not included by default and are important enough - # They need to be build from source, as https://mvnrepository.com/artifact/org.apache.nifi/nifi-iceberg-processors-nar does not ship the org.apache.hadoop.fs.s3a.S3AFileSystem (see https://github.com/apache/nifi/pull/6368#issuecomment-1502175258) - # See https://repo.stackable.tech/repository/packages/nifi/iceberg-nars/README.md for details on how to build them - cd /stackable/nifi-${PRODUCT}/lib/ && \ - curl -O "https://repo.stackable.tech/repository/packages/nifi/iceberg-nars/nifi-iceberg-processors-nar-${PRODUCT}-with-aws.nar" && \ - curl -O "https://repo.stackable.tech/repository/packages/nifi/iceberg-nars/nifi-iceberg-services-nar-${PRODUCT}-with-aws.nar" && \ - curl -O "https://repo.stackable.tech/repository/packages/nifi/iceberg-nars/nifi-iceberg-services-api-nar-${PRODUCT}-with-aws.nar" ; \ - else \ - curl 'https://repo.stackable.tech/repository/m2/tech/stackable/nifi/stackable-bcrypt/1.0-SNAPSHOT/stackable-bcrypt-1.0-20240508.153334-1-jar-with-dependencies.jar' \ - # This used to be located in /bin/stackable-bcrypt.jar. We create a softlink for /bin/stackable-bcrypt.jar in the main container for backwards compatibility. - -o /stackable/stackable-bcrypt.jar && \ - # Get the source release from nexus - curl "https://repo.stackable.tech/repository/packages/nifi/nifi-${PRODUCT}-source-release.zip" -o "/stackable/nifi-${PRODUCT}-source-release.zip" && \ - unzip "nifi-${PRODUCT}-source-release.zip" && \ - # Clean up downloaded source after unzipping - rm -rf "nifi-${PRODUCT}-source-release.zip" && \ - # The NiFi "binary" ends up in a folder named "nifi-${PRODUCT}" which should be copied to /stackable - # from /stackable/nifi-${PRODUCT}-src/nifi-assembly/target/nifi-${PRODUCT}-bin/nifi-${PRODUCT} (see later steps) - # Therefore we add the suffix "-src" to be able to copy the binary and remove the unzipped sources afterwards. - mv nifi-${PRODUCT} nifi-${PRODUCT}-src && \ - # Apply patches - chmod +x patches/apply_patches.sh && \ - patches/apply_patches.sh ${PRODUCT} && \ - # Build NiFi - cd /stackable/nifi-${PRODUCT}-src/ && \ - mvn clean install -Dmaven.javadoc.skip=true -DskipTests --activate-profiles include-iceberg,include-hadoop-aws && \ - # Copy the binaries to the /stackable folder - mv /stackable/nifi-${PRODUCT}-src/nifi-assembly/target/nifi-${PRODUCT}-bin/nifi-${PRODUCT} /stackable/nifi-${PRODUCT} && \ - # Copy the SBOM as well - mv /stackable/nifi-${PRODUCT}-src/nifi-assembly/target/bom.json /stackable/nifi-${PRODUCT}/nifi-${PRODUCT}.cdx.json && \ - # Remove the unzipped sources - rm -rf /stackable/nifi-${PRODUCT}-src && \ - # Remove generated docs in binary - rm -rf /stackable/nifi-${PRODUCT}/docs ; \ - fi +RUN curl 'https://repo.stackable.tech/repository/m2/tech/stackable/nifi/stackable-bcrypt/1.0-SNAPSHOT/stackable-bcrypt-1.0-20240508.153334-1-jar-with-dependencies.jar' \ + # This used to be located in /bin/stackable-bcrypt.jar. We create a softlink for /bin/stackable-bcrypt.jar in the main container for backwards compatibility. + -o /stackable/stackable-bcrypt.jar && \ + # Get the source release from nexus + curl "https://repo.stackable.tech/repository/packages/nifi/nifi-${PRODUCT}-source-release.zip" -o "/stackable/nifi-${PRODUCT}-source-release.zip" && \ + unzip "nifi-${PRODUCT}-source-release.zip" && \ + # Clean up downloaded source after unzipping + rm -rf "nifi-${PRODUCT}-source-release.zip" && \ + # The NiFi "binary" ends up in a folder named "nifi-${PRODUCT}" which should be copied to /stackable + # from /stackable/nifi-${PRODUCT}-src/nifi-assembly/target/nifi-${PRODUCT}-bin/nifi-${PRODUCT} (see later steps) + # Therefore we add the suffix "-src" to be able to copy the binary and remove the unzipped sources afterwards. + mv nifi-${PRODUCT} nifi-${PRODUCT}-src && \ + # Apply patches + chmod +x patches/apply_patches.sh && \ + patches/apply_patches.sh ${PRODUCT} && \ + # Build NiFi + cd /stackable/nifi-${PRODUCT}-src/ && \ + mvn clean install -Dmaven.javadoc.skip=true -DskipTests --activate-profiles include-iceberg,include-hadoop-aws && \ + # Copy the binaries to the /stackable folder + mv /stackable/nifi-${PRODUCT}-src/nifi-assembly/target/nifi-${PRODUCT}-bin/nifi-${PRODUCT} /stackable/nifi-${PRODUCT} && \ + # Copy the SBOM as well + mv /stackable/nifi-${PRODUCT}-src/nifi-assembly/target/bom.json /stackable/nifi-${PRODUCT}/nifi-${PRODUCT}.cdx.json && \ + # Remove the unzipped sources + rm -rf /stackable/nifi-${PRODUCT}-src && \ + # Remove generated docs in binary + rm -rf /stackable/nifi-${PRODUCT}/docs FROM stackable/image/java-base AS final @@ -90,12 +62,12 @@ ARG RELEASE ARG STACKABLE_USER_UID LABEL name="Apache NiFi" \ - maintainer="info@stackable.tech" \ - vendor="Stackable GmbH" \ - version="${PRODUCT}" \ - release="${RELEASE}" \ - summary="The Stackable image for Apache NiFi." \ - description="This image is deployed by the Stackable Operator for Apache NiFi." + maintainer="info@stackable.tech" \ + vendor="Stackable GmbH" \ + version="${PRODUCT}" \ + release="${RELEASE}" \ + summary="The Stackable image for Apache NiFi." \ + description="This image is deployed by the Stackable Operator for Apache NiFi." COPY --chown=${STACKABLE_USER_UID}:0 --from=nifi-builder /stackable/nifi-${PRODUCT} /stackable/nifi-${PRODUCT}/ COPY --chown=${STACKABLE_USER_UID}:0 --from=nifi-builder /stackable/stackable-bcrypt.jar /stackable/stackable-bcrypt.jar diff --git a/nifi/stackable/patches/2.0.0-M4/001-NIFI-no-zip-assembly-2.0.0-M4.patch b/nifi/stackable/patches/2.0.0/001-NIFI-no-zip-assembly-2.0.0.patch similarity index 93% rename from nifi/stackable/patches/2.0.0-M4/001-NIFI-no-zip-assembly-2.0.0-M4.patch rename to nifi/stackable/patches/2.0.0/001-NIFI-no-zip-assembly-2.0.0.patch index 8a71fe868..f3e059e4e 100644 --- a/nifi/stackable/patches/2.0.0-M4/001-NIFI-no-zip-assembly-2.0.0-M4.patch +++ b/nifi/stackable/patches/2.0.0/001-NIFI-no-zip-assembly-2.0.0.patch @@ -1,5 +1,5 @@ diff --git a/nifi-assembly/pom.xml b/nifi-assembly/pom.xml -index 83eb8214f9..0764b3716d 100644 +index e980e507c6..cb19c89367 100644 --- a/nifi-assembly/pom.xml +++ b/nifi-assembly/pom.xml @@ -66,7 +66,6 @@ language governing permissions and limitations under the License. --> diff --git a/nifi/stackable/patches/2.0.0-M4/002-NIFI-no-host-header-check-2.0.0-M4.patch b/nifi/stackable/patches/2.0.0/002-NIFI-no-host-header-check-2.0.0.patch similarity index 98% rename from nifi/stackable/patches/2.0.0-M4/002-NIFI-no-host-header-check-2.0.0-M4.patch rename to nifi/stackable/patches/2.0.0/002-NIFI-no-host-header-check-2.0.0.patch index 9d1344c0d..e41ec137c 100644 --- a/nifi/stackable/patches/2.0.0-M4/002-NIFI-no-host-header-check-2.0.0-M4.patch +++ b/nifi/stackable/patches/2.0.0/002-NIFI-no-host-header-check-2.0.0.patch @@ -26,7 +26,7 @@ Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/HostHeaderHandler.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/HostHeaderHandler.java -index 97337d63e2..0f7a272de7 100644 +index 97337d63e2..12ce1d8646 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/HostHeaderHandler.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/HostHeaderHandler.java @@ -47,6 +47,7 @@ public class HostHeaderHandler extends Handler.Abstract { diff --git a/nifi/stackable/patches/2.0.0-M4/003-patch-cyclonedx-plugin.patch b/nifi/stackable/patches/2.0.0/003-patch-cyclonedx-plugin.patch similarity index 93% rename from nifi/stackable/patches/2.0.0-M4/003-patch-cyclonedx-plugin.patch rename to nifi/stackable/patches/2.0.0/003-patch-cyclonedx-plugin.patch index 6cfd9e15d..244cf94b2 100644 --- a/nifi/stackable/patches/2.0.0-M4/003-patch-cyclonedx-plugin.patch +++ b/nifi/stackable/patches/2.0.0/003-patch-cyclonedx-plugin.patch @@ -1,8 +1,8 @@ diff --git a/pom.xml b/pom.xml -index 6af359c..0be2176 100644 +index 3190206e88..7e754f691d 100644 --- a/pom.xml +++ b/pom.xml -@@ -954,6 +954,24 @@ +@@ -974,6 +974,24 @@ diff --git a/nifi/upload_new_nifi_version.sh b/nifi/upload_new_nifi_version.sh index fa631d4a7..1b053253e 100755 --- a/nifi/upload_new_nifi_version.sh +++ b/nifi/upload_new_nifi_version.sh @@ -32,7 +32,6 @@ trap cleanup EXIT cd "$WORK_DIR" || exit src_file="nifi-$VERSION-source-release.zip" -bin_file="nifi-$VERSION-bin.zip" download_url="https://archive.apache.org/dist/nifi/${VERSION}" echo "Downloading NiFi source (this can take a while, it is intentionally downloading from a slow mirror that contains all old versions)" @@ -40,14 +39,13 @@ curl --fail -LOs "${download_url}/${src_file}" curl --fail -LOs "${download_url}/${src_file}.asc" curl --fail -LOs "${download_url}/${src_file}.sha512" -echo "Downloading NiFi binary (this can take a while, it is intentionally downloading from a slow mirror that contains all old versions)" -curl --fail -LOs "${download_url}/${bin_file}" -curl --fail -LOs "${download_url}/${bin_file}.asc" -curl --fail -LOs "${download_url}/${bin_file}.sha512" - # It is probably redundant to check both the checksum and the signature but it's cheap and why not echo "Validating SHA512 Checksums" -if ! (sha512sum "$bin_file" | cut -d ' ' -f 1 | diff - "$bin_file.sha512" && sha512sum "$src_file" | cut -d ' ' -f 1 | diff - "$src_file.sha512"); then +# The 'echo -e "$(<"${src_file}".sha512)")' part removes possible new lines in the provided .sha512 file. +# This is due to the NiFi sha512 files sometimes ending on newline and sometimes dont. +# See https://archive.apache.org/dist/nifi/2.0.0/nifi-2.0.0-source-release.zip.sha512 vs +# https://archive.apache.org/dist/nifi/1.27.0/nifi-1.27.0-source-release.zip.sha512 +if ! (sha512sum "$src_file" | cut -d ' ' -f 1 | diff - <(echo -e "$(<"${src_file}".sha512)")); then echo "ERROR: One of the SHA512 sums does not match" exit 1 fi @@ -55,7 +53,7 @@ fi echo "Validating signatures" echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://archive.apache.org/dist/nifi/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using "curl https://archive.apache.org/dist/nifi/KEYS | gpg --import")' -if ! (gpg --verify "$bin_file.asc" "$bin_file" 2> /dev/null && gpg --verify "$src_file.asc" "$src_file" 2> /dev/null); then +if ! (gpg --verify "$src_file.asc" "$src_file" 2> /dev/null); then echo "ERROR: One of the signatures could not be verified" exit 1 fi @@ -66,10 +64,6 @@ curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}" 'https: curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}.asc" 'https://repo.stackable.tech/repository/packages/nifi/' || EXIT_STATUS=$? curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}.sha512" 'https://repo.stackable.tech/repository/packages/nifi/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}" 'https://repo.stackable.tech/repository/packages/nifi/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}.asc" 'https://repo.stackable.tech/repository/packages/nifi/' || EXIT_STATUS=$? -curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}.sha512" 'https://repo.stackable.tech/repository/packages/nifi/' || EXIT_STATUS=$? - if [ $EXIT_STATUS -ne 0 ]; then echo "ERROR: Upload failed" exit 1 diff --git a/nifi/versions.py b/nifi/versions.py index df2a23414..1500ef3ed 100644 --- a/nifi/versions.py +++ b/nifi/versions.py @@ -5,7 +5,7 @@ "java-devel": "11", # There is an error when trying to use the jdk 21 (since nifi 1.26.0) }, { - "product": "2.0.0-M4", + "product": "2.0.0", "java-base": "21", "java-devel": "21", },