Skip to content

Commit 82fb887

Browse files
maltesanderNickLarsenNZ
maltesander
and
NickLarsenNZ
authored
Add support for NiFi 2.0.0 (#917)
* add support for nifi 2.0.0 * adapt changelog * improve upload * remove test artefact * attempt to fix precommit * attempt fix 2 * adapt comment * fix typo --------- Co-authored-by: Nick <[email protected]>
1 parent b8a1ced commit 82fb887

File tree

7 files changed

+48
-79
lines changed

7 files changed

+48
-79
lines changed

‎CHANGELOG.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ All notable changes to this project will be documented in this file.
2424
- spark: Add HBase connector ([#878], [#882]).
2525
- hbase: hbase-entrypoint.sh script to start and gracefully stop services ([#898]).
2626
- tools: install yq command line tool for YAML manipulation ([#912]).
27+
- nifi: Add version 2.0.0 ([#917]).
2728

2829
### Changed
2930

@@ -56,6 +57,7 @@ All notable changes to this project will be documented in this file.
5657
- zookeeper: Remove `3.8.4` ([#851]).
5758
- nifi: Remove `1.21.0` and `1.25.0` ([#868]).
5859
- druid: Remove `28.0.1` ([#880]).
60+
- nifi: Removed binaries from upload nifi script ([#917]).
5961

6062
### Fixed
6163

@@ -102,6 +104,7 @@ All notable changes to this project will be documented in this file.
102104
[#912]: https://github.com/stackabletech/docker-images/pull/912
103105
[#913]: https://github.com/stackabletech/docker-images/pull/913
104106
[#914]: https://github.com/stackabletech/docker-images/pull/914
107+
[#917]: https://github.com/stackabletech/docker-images/pull/917
105108
[#920]: https://github.com/stackabletech/docker-images/pull/920
106109

107110
## [24.7.0] - 2024-07-24

‎nifi/Dockerfile‎

Lines changed: 34 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -19,69 +19,41 @@ RUN microdnf update && \
1919
#
2020
WORKDIR /tmp
2121
RUN if [[ "${PRODUCT}" == 2.* ]] ; then \
22-
curl "https://repo.stackable.tech/repository/packages/maven/apache-maven-${MAVEN_VERSION}-bin.tar.gz" | tar -xzC . && \
23-
ln -sf /tmp/apache-maven-${MAVEN_VERSION}/bin/mvn /usr/bin/mvn ; \
22+
curl "https://repo.stackable.tech/repository/packages/maven/apache-maven-${MAVEN_VERSION}-bin.tar.gz" | tar -xzC . && \
23+
ln -sf /tmp/apache-maven-${MAVEN_VERSION}/bin/mvn /usr/bin/mvn ; \
2424
fi
2525

2626
USER ${STACKABLE_USER_UID}
2727
WORKDIR /stackable
2828

2929
COPY --chown=${STACKABLE_USER_UID}:0 nifi/stackable/patches /stackable/patches
3030

31-
# NOTE: NiFi 1.21.0 source build does not work with the current arm64 git runners due to java heap issues:
32-
#
33-
# [ERROR] Failed to execute goal org.apache.maven.plugins:maven-assembly-plugin:3.5.0:single (make shared resource) on project nifi-registry-assembly:
34-
# Failed to create assembly: Error creating assembly archive bin: Problem creating zip: Execution exception: Java heap space
35-
#
36-
# Since this will be deprecated in the release 24.7 and then removed we copy the NiFi 1.21.0 binaries instead
37-
# of building from source. The if condition can be removed once 1.21.0 is no longer supported and only the
38-
# else branch is required to build from source.
39-
#
40-
RUN if [[ "${PRODUCT}" == "1.21.0" ]] ; then \
41-
curl 'https://repo.stackable.tech/repository/m2/tech/stackable/nifi/stackable-bcrypt/1.0-SNAPSHOT/stackable-bcrypt-1.0-20240508.153334-1-jar-with-dependencies.jar' \
42-
# This used to be located in /bin/stackable-bcrypt.jar. We create a softlink for /bin/stackable-bcrypt.jar in the main container for backwards compatibility.
43-
-o /stackable/stackable-bcrypt.jar && \
44-
# zip is different than tar and cannot be just piped, therefore the intermediate save and remove step to unzip
45-
curl https://repo.stackable.tech/repository/packages/nifi/nifi-${PRODUCT}-bin.zip -o /stackable/nifi-${PRODUCT}-bin.zip && \
46-
unzip /stackable/nifi-${PRODUCT}-bin.zip && \
47-
rm /stackable/nifi-${PRODUCT}-bin.zip && \
48-
# Remove generated docs in binary
49-
rm -rf /stackable/nifi-${PRODUCT}/docs && \
50-
# Add Iceberg extensions as they are not included by default and are important enough
51-
# They need to be build from source, as https://mvnrepository.com/artifact/org.apache.nifi/nifi-iceberg-processors-nar does not ship the org.apache.hadoop.fs.s3a.S3AFileSystem (see https://github.com/apache/nifi/pull/6368#issuecomment-1502175258)
52-
# See https://repo.stackable.tech/repository/packages/nifi/iceberg-nars/README.md for details on how to build them
53-
cd /stackable/nifi-${PRODUCT}/lib/ && \
54-
curl -O "https://repo.stackable.tech/repository/packages/nifi/iceberg-nars/nifi-iceberg-processors-nar-${PRODUCT}-with-aws.nar" && \
55-
curl -O "https://repo.stackable.tech/repository/packages/nifi/iceberg-nars/nifi-iceberg-services-nar-${PRODUCT}-with-aws.nar" && \
56-
curl -O "https://repo.stackable.tech/repository/packages/nifi/iceberg-nars/nifi-iceberg-services-api-nar-${PRODUCT}-with-aws.nar" ; \
57-
else \
58-
curl 'https://repo.stackable.tech/repository/m2/tech/stackable/nifi/stackable-bcrypt/1.0-SNAPSHOT/stackable-bcrypt-1.0-20240508.153334-1-jar-with-dependencies.jar' \
59-
# This used to be located in /bin/stackable-bcrypt.jar. We create a softlink for /bin/stackable-bcrypt.jar in the main container for backwards compatibility.
60-
-o /stackable/stackable-bcrypt.jar && \
61-
# Get the source release from nexus
62-
curl "https://repo.stackable.tech/repository/packages/nifi/nifi-${PRODUCT}-source-release.zip" -o "/stackable/nifi-${PRODUCT}-source-release.zip" && \
63-
unzip "nifi-${PRODUCT}-source-release.zip" && \
64-
# Clean up downloaded source after unzipping
65-
rm -rf "nifi-${PRODUCT}-source-release.zip" && \
66-
# The NiFi "binary" ends up in a folder named "nifi-${PRODUCT}" which should be copied to /stackable
67-
# from /stackable/nifi-${PRODUCT}-src/nifi-assembly/target/nifi-${PRODUCT}-bin/nifi-${PRODUCT} (see later steps)
68-
# Therefore we add the suffix "-src" to be able to copy the binary and remove the unzipped sources afterwards.
69-
mv nifi-${PRODUCT} nifi-${PRODUCT}-src && \
70-
# Apply patches
71-
chmod +x patches/apply_patches.sh && \
72-
patches/apply_patches.sh ${PRODUCT} && \
73-
# Build NiFi
74-
cd /stackable/nifi-${PRODUCT}-src/ && \
75-
mvn clean install -Dmaven.javadoc.skip=true -DskipTests --activate-profiles include-iceberg,include-hadoop-aws && \
76-
# Copy the binaries to the /stackable folder
77-
mv /stackable/nifi-${PRODUCT}-src/nifi-assembly/target/nifi-${PRODUCT}-bin/nifi-${PRODUCT} /stackable/nifi-${PRODUCT} && \
78-
# Copy the SBOM as well
79-
mv /stackable/nifi-${PRODUCT}-src/nifi-assembly/target/bom.json /stackable/nifi-${PRODUCT}/nifi-${PRODUCT}.cdx.json && \
80-
# Remove the unzipped sources
81-
rm -rf /stackable/nifi-${PRODUCT}-src && \
82-
# Remove generated docs in binary
83-
rm -rf /stackable/nifi-${PRODUCT}/docs ; \
84-
fi
31+
RUN curl 'https://repo.stackable.tech/repository/m2/tech/stackable/nifi/stackable-bcrypt/1.0-SNAPSHOT/stackable-bcrypt-1.0-20240508.153334-1-jar-with-dependencies.jar' \
32+
# This used to be located in /bin/stackable-bcrypt.jar. We create a softlink for /bin/stackable-bcrypt.jar in the main container for backwards compatibility.
33+
-o /stackable/stackable-bcrypt.jar && \
34+
# Get the source release from nexus
35+
curl "https://repo.stackable.tech/repository/packages/nifi/nifi-${PRODUCT}-source-release.zip" -o "/stackable/nifi-${PRODUCT}-source-release.zip" && \
36+
unzip "nifi-${PRODUCT}-source-release.zip" && \
37+
# Clean up downloaded source after unzipping
38+
rm -rf "nifi-${PRODUCT}-source-release.zip" && \
39+
# The NiFi "binary" ends up in a folder named "nifi-${PRODUCT}" which should be copied to /stackable
40+
# from /stackable/nifi-${PRODUCT}-src/nifi-assembly/target/nifi-${PRODUCT}-bin/nifi-${PRODUCT} (see later steps)
41+
# Therefore we add the suffix "-src" to be able to copy the binary and remove the unzipped sources afterwards.
42+
mv nifi-${PRODUCT} nifi-${PRODUCT}-src && \
43+
# Apply patches
44+
chmod +x patches/apply_patches.sh && \
45+
patches/apply_patches.sh ${PRODUCT} && \
46+
# Build NiFi
47+
cd /stackable/nifi-${PRODUCT}-src/ && \
48+
mvn clean install -Dmaven.javadoc.skip=true -DskipTests --activate-profiles include-iceberg,include-hadoop-aws && \
49+
# Copy the binaries to the /stackable folder
50+
mv /stackable/nifi-${PRODUCT}-src/nifi-assembly/target/nifi-${PRODUCT}-bin/nifi-${PRODUCT} /stackable/nifi-${PRODUCT} && \
51+
# Copy the SBOM as well
52+
mv /stackable/nifi-${PRODUCT}-src/nifi-assembly/target/bom.json /stackable/nifi-${PRODUCT}/nifi-${PRODUCT}.cdx.json && \
53+
# Remove the unzipped sources
54+
rm -rf /stackable/nifi-${PRODUCT}-src && \
55+
# Remove generated docs in binary
56+
rm -rf /stackable/nifi-${PRODUCT}/docs
8557

8658
FROM stackable/image/java-base AS final
8759

@@ -90,12 +62,12 @@ ARG RELEASE
9062
ARG STACKABLE_USER_UID
9163

9264
LABEL name="Apache NiFi" \
93-
maintainer="[email protected]" \
94-
vendor="Stackable GmbH" \
95-
version="${PRODUCT}" \
96-
release="${RELEASE}" \
97-
summary="The Stackable image for Apache NiFi." \
98-
description="This image is deployed by the Stackable Operator for Apache NiFi."
65+
maintainer="[email protected]" \
66+
vendor="Stackable GmbH" \
67+
version="${PRODUCT}" \
68+
release="${RELEASE}" \
69+
summary="The Stackable image for Apache NiFi." \
70+
description="This image is deployed by the Stackable Operator for Apache NiFi."
9971

10072
COPY --chown=${STACKABLE_USER_UID}:0 --from=nifi-builder /stackable/nifi-${PRODUCT} /stackable/nifi-${PRODUCT}/
10173
COPY --chown=${STACKABLE_USER_UID}:0 --from=nifi-builder /stackable/stackable-bcrypt.jar /stackable/stackable-bcrypt.jar

‎nifi/stackable/patches/2.0.0-M4/001-NIFI-no-zip-assembly-2.0.0-M4.patch‎ renamed to ‎nifi/stackable/patches/2.0.0/001-NIFI-no-zip-assembly-2.0.0.patch‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
diff --git a/nifi-assembly/pom.xml b/nifi-assembly/pom.xml
2-
index 83eb8214f9..0764b3716d 100644
2+
index e980e507c6..cb19c89367 100644
33
--- a/nifi-assembly/pom.xml
44
+++ b/nifi-assembly/pom.xml
55
@@ -66,7 +66,6 @@ language governing permissions and limitations under the License. -->

‎nifi/stackable/patches/2.0.0-M4/002-NIFI-no-host-header-check-2.0.0-M4.patch‎ renamed to ‎nifi/stackable/patches/2.0.0/002-NIFI-no-host-header-check-2.0.0.patch‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
2626
<+>UTF-8
2727
===================================================================
2828
diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/HostHeaderHandler.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/HostHeaderHandler.java
29-
index 97337d63e2..0f7a272de7 100644
29+
index 97337d63e2..12ce1d8646 100644
3030
--- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/HostHeaderHandler.java
3131
+++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/HostHeaderHandler.java
3232
@@ -47,6 +47,7 @@ public class HostHeaderHandler extends Handler.Abstract {

‎nifi/stackable/patches/2.0.0-M4/003-patch-cyclonedx-plugin.patch‎ renamed to ‎nifi/stackable/patches/2.0.0/003-patch-cyclonedx-plugin.patch‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
diff --git a/pom.xml b/pom.xml
2-
index 6af359c..0be2176 100644
2+
index 3190206e88..7e754f691d 100644
33
--- a/pom.xml
44
+++ b/pom.xml
5-
@@ -954,6 +954,24 @@
5+
@@ -974,6 +974,24 @@
66
</excludes>
77
</configuration>
88
</plugin>

‎nifi/upload_new_nifi_version.sh‎

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -32,30 +32,28 @@ trap cleanup EXIT
3232
cd "$WORK_DIR" || exit
3333

3434
src_file="nifi-$VERSION-source-release.zip"
35-
bin_file="nifi-$VERSION-bin.zip"
3635
download_url="https://archive.apache.org/dist/nifi/${VERSION}"
3736

3837
echo "Downloading NiFi source (this can take a while, it is intentionally downloading from a slow mirror that contains all old versions)"
3938
curl --fail -LOs "${download_url}/${src_file}"
4039
curl --fail -LOs "${download_url}/${src_file}.asc"
4140
curl --fail -LOs "${download_url}/${src_file}.sha512"
4241

43-
echo "Downloading NiFi binary (this can take a while, it is intentionally downloading from a slow mirror that contains all old versions)"
44-
curl --fail -LOs "${download_url}/${bin_file}"
45-
curl --fail -LOs "${download_url}/${bin_file}.asc"
46-
curl --fail -LOs "${download_url}/${bin_file}.sha512"
47-
4842
# It is probably redundant to check both the checksum and the signature but it's cheap and why not
4943
echo "Validating SHA512 Checksums"
50-
if ! (sha512sum "$bin_file" | cut -d ' ' -f 1 | diff - "$bin_file.sha512" && sha512sum "$src_file" | cut -d ' ' -f 1 | diff - "$src_file.sha512"); then
44+
# The 'echo -e "$(<"${src_file}".sha512)")' part removes possible new lines in the provided .sha512 file.
45+
# This is due to the NiFi sha512 files sometimes ending on newline and sometimes dont.
46+
# See https://archive.apache.org/dist/nifi/2.0.0/nifi-2.0.0-source-release.zip.sha512 vs
47+
# https://archive.apache.org/dist/nifi/1.27.0/nifi-1.27.0-source-release.zip.sha512
48+
if ! (sha512sum "$src_file" | cut -d ' ' -f 1 | diff - <(echo -e "$(<"${src_file}".sha512)")); then
5149
echo "ERROR: One of the SHA512 sums does not match"
5250
exit 1
5351
fi
5452

5553
echo "Validating signatures"
5654
echo '--> NOTE: Make sure you have downloaded and added the KEYS file (https://archive.apache.org/dist/nifi/KEYS) to GPG: https://www.apache.org/info/verification.html (e.g. by using "curl https://archive.apache.org/dist/nifi/KEYS | gpg --import")'
5755

58-
if ! (gpg --verify "$bin_file.asc" "$bin_file" 2> /dev/null && gpg --verify "$src_file.asc" "$src_file" 2> /dev/null); then
56+
if ! (gpg --verify "$src_file.asc" "$src_file" 2> /dev/null); then
5957
echo "ERROR: One of the signatures could not be verified"
6058
exit 1
6159
fi
@@ -66,10 +64,6 @@ curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}" 'https:
6664
curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}.asc" 'https://repo.stackable.tech/repository/packages/nifi/' || EXIT_STATUS=$?
6765
curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${src_file}.sha512" 'https://repo.stackable.tech/repository/packages/nifi/' || EXIT_STATUS=$?
6866

69-
curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}" 'https://repo.stackable.tech/repository/packages/nifi/' || EXIT_STATUS=$?
70-
curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}.asc" 'https://repo.stackable.tech/repository/packages/nifi/' || EXIT_STATUS=$?
71-
curl --fail -u "$NEXUS_USER:$NEXUS_PASSWORD" --upload-file "${bin_file}.sha512" 'https://repo.stackable.tech/repository/packages/nifi/' || EXIT_STATUS=$?
72-
7367
if [ $EXIT_STATUS -ne 0 ]; then
7468
echo "ERROR: Upload failed"
7569
exit 1

‎nifi/versions.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
"java-devel": "11", # There is an error when trying to use the jdk 21 (since nifi 1.26.0)
66
},
77
{
8-
"product": "2.0.0-M4",
8+
"product": "2.0.0",
99
"java-base": "21",
1010
"java-devel": "21",
1111
},

0 commit comments

Comments
 (0)