MINIFICPP-2514 Fix Python libloader

Fix and extend verify package to include aarch64

Closes #1919

Signed-off-by: Marton Szasz <[email protected]>

Author: Martin Zink

.github/workflows/verify-package.yml

@@ -6,9 +6,9 @@ env:
     -DENABLE_ELASTICSEARCH=OFF -DENABLE_GRAFANA_LOKI=ON -DENABLE_COUCHBASE=ON -DDOCKER_BUILD_ONLY=ON
   CCACHE_DIR: ${{ GITHUB.WORKSPACE }}/.ccache
 jobs:
-  rocky-build:
-    name: "Build package (Rocky 8)"
-    runs-on: ubuntu-22.04
+  rocky-x86-64-build:
+    name: "Build package (Rocky 8 x86_64)"
+    runs-on: ubuntu-24.04
     timeout-minutes: 180
     steps:
       - id: checkout
@@ -38,40 +38,76 @@ jobs:
           key: verify-package-ccache-${{github.ref}}-${{github.sha}}
       - uses: actions/upload-artifact@v4
         with:
-          name: minifi-tar
+          name: minifi-x86-64-tar
           path: build/nifi-minifi-cpp-*-bin-rockylinux.tar.gz
           if-no-files-found: error
 
-  docker-tests:
+  rocky-aarch64-build:
+    name: "Build package (Rocky 8 aarch64)"
+    runs-on: ubuntu-24.04-arm
+    timeout-minutes: 180
+    steps:
+      - id: checkout
+        uses: actions/checkout@v4
+      - name: cache restore
+        uses: actions/cache/restore@v4
+        with:
+          path: ~/.ccache
+          key: verify-package-arm-ccache-${{github.ref}}-${{github.sha}}
+          restore-keys: |
+            verify-package-arm-ccache-${{github.ref}}-
+            verify-package-arm-ccache-refs/heads/main-
+      - id: install_deps
+        run: |
+          sudo apt update
+          sudo apt install -y ccache cmake
+          echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
+      - id: build
+        run: |
+          if [ -d ~/.ccache ]; then mv ~/.ccache .; fi
+          mkdir build && cd build && cmake ${DOCKER_CMAKE_FLAGS} -DDOCKER_CCACHE_DUMP_LOCATION=${{ env.CCACHE_DIR }} .. && make rocky
+      - name: cache save
+        uses: actions/cache/save@v4
+        if: always()
+        with:
+          path: ${{ env.CCACHE_DIR }}
+          key: verify-package-arm-ccache-${{github.ref}}-${{github.sha}}
+      - uses: actions/upload-artifact@v4
+        with:
+          name: minifi-aarch64-tar
+          path: build/nifi-minifi-cpp-*-bin-rockylinux.tar.gz
+          if-no-files-found: error
+
+  docker-x86-64-tests:
     name: "${{ matrix.platform.name }}"
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 180
-    needs: rocky-build
+    needs: rocky-x86-64-build
     strategy:
       fail-fast: false
       matrix:
         platform: [
-          { name: "Rocky Linux 8",
+          { name: "Rocky Linux 8 x86-64",
             id: "rocky8",
             build_cmd: "make rocky8_from_rocky_package",
             verify_cmd: "make docker-verify-rocky8" },
-          { name: "Rocky Linux 9",
+          { name: "Rocky Linux 9 x86-64",
             id: "rocky9",
             build_cmd: "make rocky9_from_rocky_package",
             verify_cmd: "make docker-verify-rocky9" },
-          { name: "Ubuntu 22.04",
+          { name: "Ubuntu 22.04 x86-64",
             id: "ubuntu_jammy",
             build_cmd: "make jammy_from_rocky_package",
             verify_cmd: "make docker-verify-jammy" },
-          { name: "Ubuntu 24.04",
+          { name: "Ubuntu 24.04 x86-64",
             id: "ubuntu_noble",
             build_cmd: "make noble_from_rocky_package",
             verify_cmd: "make docker-verify-noble" },
-          { name: "Debian 11",
+          { name: "Debian 11 x86-64",
             id: "debian_bullseye",
             build_cmd: "make bullseye_from_rocky_package",
             verify_cmd: "make docker-verify-bullseye" },
-          { name: "Debian 12",
+          { name: "Debian 12 x86-64",
             id: "debian_bookworm",
             build_cmd: "make bookworm_from_rocky_package",
             verify_cmd: "make docker-verify-bookworm" },
@@ -82,7 +118,7 @@ jobs:
         uses: actions/checkout@v4
       - uses: actions/download-artifact@v4
         with:
-          name: minifi-tar
+          name: minifi-x86-64-tar
           path: build
       - id: install_deps
         run: |
@@ -104,55 +140,57 @@ jobs:
         if: failure()
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ matrix.platform.id }}_behavex_output
+          name: ${{ matrix.platform.id }}_x86_64_behavex_output
           path: build/behavex_output
-  docker-python-tests:
+
+
+  docker-aarch64-tests:
     name: "${{ matrix.platform.name }}"
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04-arm
     timeout-minutes: 180
-    needs: rocky-build
+    needs: rocky-aarch64-build
     strategy:
       fail-fast: false
       matrix:
         platform: [
-          { name: "Rocky Linux 8 Python",
+          { name: "Rocky Linux 8 aarch64",
             id: "rocky8",
-            build_cmd: "make rocky8_py_from_rocky_package",
-            verify_cmd: "make docker-verify-rocky8_py" },
-          { name: "Rocky Linux 9 Python",
+            build_cmd: "make rocky8_from_rocky_package",
+            verify_cmd: "make docker-verify-rocky8" },
+          { name: "Rocky Linux 9 aarch64",
             id: "rocky9",
-            build_cmd: "make rocky9_py_from_rocky_package",
-            verify_cmd: "make docker-verify-rocky9_py" },
-          { name: "Patchelf-ed Ubuntu 22.04 Python",
-            id: "patchelfed_ubuntu22.04",
-            build_cmd: "make patched_jammy_py_from_rocky_package",
-            verify_cmd: "make docker-verify-patched_jammy_py" },
-          { name: "Patchelf-ed Debian 11 Python",
-            id: "patchelfed_debian11",
-            build_cmd: "make patched_bullseye_py_from_rocky_package",
-            verify_cmd: "make docker-verify-patched_bullseye_py" },
-          { name: "Ubuntu 22.04 with Anaconda Python",
-            id: "ubuntu22.04_anaconda",
-            build_cmd: "make jammy_py_from_rocky_package && make conda_jammy_py_from_rocky_package",
-            verify_cmd: "make docker-verify-conda_jammy_py" },
-          { name: "Rocky Linux 9 in virtualenv Python",
-            id: "rocky9_venv",
-            build_cmd: "make rocky9_py_from_rocky_package && make venv_rocky9_py_from_rocky_package",
-            verify_cmd: "make docker-verify-venv_rocky9_py" }
+            build_cmd: "make rocky9_from_rocky_package",
+            verify_cmd: "make docker-verify-rocky9" },
+          { name: "Ubuntu 22.04 aarch64",
+            id: "ubuntu_jammy",
+            build_cmd: "make jammy_from_rocky_package",
+            verify_cmd: "make docker-verify-jammy" },
+          { name: "Ubuntu 24.04 aarch64",
+            id: "ubuntu_noble",
+            build_cmd: "make noble_from_rocky_package",
+            verify_cmd: "make docker-verify-noble" },
+          { name: "Debian 11 aarch64",
+            id: "debian_bullseye",
+            build_cmd: "make bullseye_from_rocky_package",
+            verify_cmd: "make docker-verify-bullseye" },
+          { name: "Debian 12 aarch64",
+            id: "debian_bookworm",
+            build_cmd: "make bookworm_from_rocky_package",
+            verify_cmd: "make docker-verify-bookworm" },
         ]
 
     steps:
       - id: checkout
         uses: actions/checkout@v4
       - uses: actions/download-artifact@v4
         with:
-          name: minifi-tar
+          name: minifi-aarch64-tar
           path: build
       - id: install_deps
         run: |
           sudo apt update
           sudo apt install -y python3-virtualenv
-      - run: cd build && cmake -DENABLE_PYTHON_SCRIPTING=ON -DDOCKER_BUILD_ONLY=ON .. \
+      - run: cd build && cmake ${DOCKER_CMAKE_FLAGS} .. \
           && ${{ matrix.platform.build_cmd }} && ${{ matrix.platform.verify_cmd }}
       - name: Test Reporter
         if: always()
@@ -168,5 +206,5 @@ jobs:
         if: failure()
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ matrix.platform.id }}_behavex_output
+          name: ${{ matrix.platform.id }}_aarch64_behavex_output
           path: build/behavex_output

cmake/VerifyPackageWithDocker.cmake

@@ -28,20 +28,20 @@ function(ADD_PACKAGE_VERIFY TAG_PREFIX)
         endif()
     endforeach()
 
-    set(DISABLED_TAGS "SKIP_CI,ENABLE_PYTHON_SCRIPTING")
+    set(DISABLED_TAGS "SKIP_CI,NEEDS_NUMPY")
 
     add_custom_target(
             docker-verify-${TAG_PREFIX}
             COMMAND ${CMAKE_SOURCE_DIR}/docker/DockerVerify.sh --image-tag-prefix ${TAG_PREFIX} ${MINIFI_VERSION_STR} ${ENABLED_TAGS} --tags_to_exclude=${DISABLED_TAGS} --parallel_processes=${DOCKER_VERIFY_THREADS})
 endfunction()
 
 
-CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(rockylinux:8 rocky8 "yum install -y wget")
-CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(rockylinux:9 rocky9 "yum install -y wget")
-CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(ubuntu:jammy jammy "apt update \\&\\& apt install -y wget")
-CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(ubuntu:noble noble "apt update \\&\\& apt install -y wget")
-CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(debian:bookworm bookworm "apt update \\&\\& apt install -y wget")
-CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(debian:bullseye bullseye "apt update \\&\\& apt install -y wget")
+CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(rockylinux:8 rocky8 "dnf install -y wget python3.12-devel python3.12-pip gcc gcc-c++")
+CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(rockylinux:9 rocky9 "dnf install -y wget python3-devel python3-pip gcc gcc-c++")
+CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(ubuntu:jammy jammy "apt update \\&\\& apt install -y wget python3-dev python3-venv python3-pip")
+CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(ubuntu:noble noble "apt update \\&\\& apt install -y wget python3-dev python3-venv python3-pip")
+CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(debian:bookworm bookworm "apt update \\&\\& apt install -y wget python3-dev python3-venv python3-pip")
+CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE(debian:bullseye bullseye "apt update \\&\\& apt install -y wget python3-dev python3-venv python3-pip")
 
 if (EXISTS ${CMAKE_SOURCE_DIR}/docker/test/integration/features)
     ADD_PACKAGE_VERIFY(rocky8)

docker/requirements.txt

@@ -2,10 +2,10 @@ pyjks==20.0.0
 shortuuid==1.0.11
 behavex==2.0.1
 docker==5.0.0
-kafka-python==2.0.2
-confluent-kafka==1.7.0
+kafka-python-ng==2.2.3
+confluent-kafka==2.8.0
 PyYAML==6.0.1
-m2crypto==0.41.0
+m2crypto==0.43.0
 watchdog==2.1.2
 pyopenssl==23.0.0
 azure-storage-blob==12.13.0

docker/test/integration/cluster/ImageStore.py

@@ -80,10 +80,10 @@ def __build_minifi_cpp_sql_image(self):
             so_location = "psqlodbca.so"
         elif "bullseye" in MinifiContainer.MINIFI_TAG_PREFIX or "bookworm" in MinifiContainer.MINIFI_TAG_PREFIX:
             install_sql_cmd = "apt -y install odbc-postgresql"
-            so_location = "/usr/lib/x86_64-linux-gnu/odbc/psqlodbca.so"
+            so_location = "/usr/lib/$(gcc -dumpmachine)/odbc/psqlodbca.so"
         elif "jammy" in MinifiContainer.MINIFI_TAG_PREFIX or "noble" in MinifiContainer.MINIFI_TAG_PREFIX:
             install_sql_cmd = "apt -y install odbc-postgresql"
-            so_location = "/usr/lib/x86_64-linux-gnu/odbc/psqlodbca.so"
+            so_location = "/usr/lib/$(gcc -dumpmachine)/odbc/psqlodbca.so"
         else:
             install_sql_cmd = "apk --update --no-cache add psqlodbc"
             so_location = "psqlodbca.so"
@@ -143,7 +143,7 @@ def __build_minifi_cpp_image_with_nifi_python_processors(self, python_option):
         parse_document_sed_cmd = 'sed -i "/class ProcessorDetails:/,/^$/{/^\\s*dependencies\\s*=/,/\\]\\s*$/d}" /opt/minifi/minifi-current/minifi-python/nifi_python_processors/ParseDocument.py && \\'
         chunk_document_sed_cmd = 'sed -i "/class ProcessorDetails:/,/^$/{/^\\s*dependencies\\s*=/,/\\]\\s*$/d}" /opt/minifi/minifi-current/minifi-python/nifi_python_processors/ChunkDocument.py && \\'
         if python_option == PythonOptions.SYSTEM_INSTALLED_PACKAGES:
-            if not MinifiContainer.MINIFI_TAG_PREFIX:
+            if not MinifiContainer.MINIFI_TAG_PREFIX or "bookworm" in MinifiContainer.MINIFI_TAG_PREFIX or "noble" in MinifiContainer.MINIFI_TAG_PREFIX:
                 additional_cmd = "RUN pip3 install --break-system-packages 'langchain<=0.17.0'"
             else:
                 additional_cmd = "RUN pip3 install 'langchain<=0.17.0'"

docker/test/integration/features/fetch_modbus_tcp.feature

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+@x86_x64_only
 @CORE
 Feature: Minifi C++ can act as a modbus tcp master
 

docker/test/integration/features/google_cloud_storage.feature

@@ -28,6 +28,7 @@ Feature: Sending data to Google Cloud Storage using PutGCSObject
     And a PutFile processor with the "Directory" property set to "/tmp/output"
     And the "success" relationship of the GetFile processor is connected to the PutGCSObject
     And the "success" relationship of the PutGCSObject processor is connected to the PutFile
+    And the "failure" relationship of the PutGCSObject processor is connected to the PutGCSObject
 
     When all instances start up
 

docker/test/integration/features/steps/steps.py

@@ -731,12 +731,13 @@ def step_impl(context, content, topic_name):
 
     producer = Producer({
         "bootstrap.servers": "localhost:29093",
-        "security.protocol": "ssl",
+        "security.protocol": "SSL",
         "ssl.ca.location": ca_cert_file.name,
         "ssl.certificate.location": client_cert_file.name,
         "ssl.key.location": client_key_file.name,
         "ssl.key.password": "",
-        "client.id": socket.gethostname()})
+        "client.id": socket.gethostname(),
+        "ssl.endpoint.identification.algorithm": "none"})
     producer.produce(topic_name, content.encode("utf-8"), callback=delivery_report)
     producer.flush(10)
 

extensions/python/pythonlibloader/PythonLibLoader.cpp

@@ -38,7 +38,7 @@ class PythonLibLoader {
       python_command = python_binary.value();
     }
     std::string command = python_command +
-      " -c \"import sysconfig, os, glob; print(min(glob.glob(os.path.join(sysconfig.get_config_var('LIBDIR'), f\\\"libpython{sysconfig.get_config_var('VERSION')}.so*\\\")), key=len, default=''))\"";
+      R"###( -c "import sysconfig, os, glob; v = sysconfig.get_config_vars(); lib_dir = v['LIBDIR']; ld_lib = v['LDLIBRARY']; so_paths = glob.glob(f'{lib_dir}/*{ld_lib}*'); so_paths.extend(glob.glob(f'{lib_dir}/*/*{ld_lib}*')); print(list(filter(os.path.exists, so_paths))[0])")###";
     auto lib_python_path = execCommand(command);
     if (lib_python_path.empty()) {
       logger_->log_error("Failed to find libpython path from specified python binary: {}", python_command);