build(deps): Migrate LibArchive dependency to task-based installation:

components/core/CMakeLists.txt

@@ -71,16 +71,6 @@ set(CLP_USE_STATIC_LIBS ON CACHE BOOL "Whether to link against static libraries"
 if (CLP_USE_STATIC_LIBS)
     if (APPLE)
         set(CLP_STATIC_LIBS_UNSUPPORTED_PLATFORM "macOS")
-    elseif (EXISTS "/etc/centos-release")
-        # NOTE:
-        # 1. We don't support static linking on any CentOS-based distro except manylinux_2_28 (which
-        #    shows up as "AlmaLinux").
-        # 2. A release called "AlmaLinux" doesn't guarantee we're running on a manylinux distro, but
-        #    we can improve this check when someone reports an issue.
-        file(READ "/etc/centos-release" CENTOS_RELEASE_CONTENT)
-        if(NOT "${CENTOS_RELEASE_CONTENT}" MATCHES "AlmaLinux")
-            set(CLP_STATIC_LIBS_UNSUPPORTED_PLATFORM "CentOS")
-        endif()
     endif()
 
     if (DEFINED CLP_STATIC_LIBS_UNSUPPORTED_PLATFORM)
@@ -155,6 +145,11 @@ if(CLP_NEED_BOOST)
     endif()
 endif()
 
+if(CLP_NEED_BZIP2)
+    find_package(BZip2 REQUIRED)
+    message(STATUS "Found BZip2")
+endif()
+
 if(CLP_NEED_CATCH2)
     find_package(Catch2 REQUIRED)
     if (Catch2_FOUND)
@@ -183,6 +178,14 @@ if(CLP_NEED_LOG_SURGEON)
     endif()
 endif()
 
+if(CLP_NEED_LZ4)
+    if(CLP_USE_STATIC_LIBS)
+        set(LZ4_USE_STATIC_LIBS ON)
+    endif()
+    find_package(LZ4 REQUIRED)
+    message(STATUS "Found LZ4 ${LZ4_VERSION}")
+endif()
+
 if(CLP_NEED_NLOHMANN_JSON)
     find_package(nlohmann_json REQUIRED)
     if(nlohmann_json_FOUND)
@@ -209,12 +212,8 @@ if(CLP_NEED_LIBARCHIVE)
     if(CLP_USE_STATIC_LIBS)
         set(LibArchive_USE_STATIC_LIBS ON)
     endif()
-    find_package(LibArchive REQUIRED)
-    if(LibArchive_FOUND)
-        message(STATUS "Found LibArchive ${LibArchive_VERSION}")
-    else()
-        message(FATAL_ERROR "Could not find ${CLP_LIBS_STRING} libraries for LibArchive")
-    endif()
+    find_package(LibArchive 3.8.1 REQUIRED)
+    message(STATUS "Found LibArchive ${LibArchive_VERSION}")
 endif()
 
 # Find and setup libcurl
@@ -288,6 +287,12 @@ if(CLP_NEED_YSTDLIB)
     add_subdirectory("${CLP_YSTDLIB_SOURCE_DIRECTORY}" "${CMAKE_BINARY_DIR}/ystdlib" EXCLUDE_FROM_ALL)
 endif()
 
+# Find and setup ZLIB
+if(CLP_NEED_ZLIB)
+    find_package(ZLIB REQUIRED)
+    message(STATUS "Found ZLIB ${ZLIB_VERSION_STRING}")
+endif()
+
 # Find and setup ZStd Library
 if(CLP_NEED_ZSTD)
     if(CLP_USE_STATIC_LIBS)
@@ -742,7 +747,7 @@ if(CLP_BUILD_TESTING)
             ${CLP_SQLITE3_INCLUDE_DIRECTORY}
             )
     target_link_libraries(unitTest
-            PRIVATE
+	    PRIVATE
             absl::flat_hash_map
             Boost::filesystem Boost::iostreams Boost::program_options Boost::regex Boost::url
             Catch2::Catch2
@@ -768,7 +773,6 @@ if(CLP_BUILD_TESTING)
             yaml-cpp
             ystdlib::containers
             ystdlib::error_handling
-            ${LIBLZMA_LIBRARIES}
             ZStd::ZStd
             )
     target_compile_features(unitTest

components/core/cmake/Modules/FindLibArchive.cmake

@@ -12,23 +12,14 @@
 # - Variables only for use within the script are prefixed with "libarchive_"
 # - Variables that should be externally visible are prefixed with "LibArchive_"
 
-set(libarchive_LIBNAME "archive")
-
 include(cmake/Modules/FindLibraryDependencies.cmake)
 
-# On macOS, libarchive installed through brew is not linked into prefix by default.
-# So it cannot be found by pkg-config and we need to manually find it.
-# For more details, see https://github.com/Homebrew/homebrew-core/issues/117642
-# Find and setup libarchive
-if(APPLE)
-    execute_process(COMMAND brew --prefix libarchive OUTPUT_VARIABLE libarchive_MACOS_PREFIX)
-    string(STRIP "${libarchive_MACOS_PREFIX}" libarchive_MACOS_PREFIX)
-    set(ENV{libarchive_PREV_CMAKE_PATH} "$ENV{CMAKE_PREFIX_PATH}")  # save it so we can revert it later
-    set(ENV{CMAKE_PREFIX_PATH} "${libarchive_MACOS_PREFIX};$ENV{CMAKE_PREFIX_PATH}")
-endif()
+set(libarchive_LIBNAME "archive")
+set(libarchive_PKGCONFIG_DIR "${LibArchive_ROOT}/lib/pkgconfig")
 
 # Run pkg-config
 find_package(PkgConfig)
+set(ENV{PKG_CONFIG_PATH} "${libarchive_PKGCONFIG_DIR};$ENV{PKG_CONFIG_PATH}")
 pkg_check_modules(libarchive_PKGCONF QUIET "lib${libarchive_LIBNAME}")
 
 # Set include directory
@@ -38,6 +29,7 @@ find_path(LibArchive_INCLUDE_DIR archive.h
         )
 
 # Handle static libraries
+set(LibArchive_USE_STATIC_LIBS ON)
 if(LibArchive_USE_STATIC_LIBS)
     # Save current value of CMAKE_FIND_LIBRARY_SUFFIXES
     set(libarchive_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
@@ -52,7 +44,7 @@ find_library(LibArchive_LIBRARY
         HINTS ${libarchive_PKGCONF_LIBDIR}
         PATH_SUFFIXES lib
         )
-if (LibArchive_LIBRARY)
+if(LibArchive_LIBRARY)
     # NOTE: This must be set for find_package_handle_standard_args to work
     set(LibArchive_FOUND ON)
 endif()
@@ -64,10 +56,16 @@ if(LibArchive_USE_STATIC_LIBS)
     # Restore original value of CMAKE_FIND_LIBRARY_SUFFIXES
     set(CMAKE_FIND_LIBRARY_SUFFIXES ${libarchive_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES})
     unset(libarchive_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES)
+else()
+    set(libarchive_DYNAMIC_LIBS "${libarchive_PKGCONF_STATIC_LIBRARIES}")
 endif()
 
 FindDynamicLibraryDependencies(libarchive "${libarchive_DYNAMIC_LIBS}")
 
+message(STATUS "libarchive_PKGCONF_STATIC_LIBRARIES = ${libarchive_PKGCONF_STATIC_LIBRARIES}")
+message(STATUS "libarchive_DYNAMIC_LIBS = ${libarchive_DYNAMIC_LIBS}")
+message(STATUS "libarchive_LIBRARY_DEPENDENCIES = ${libarchive_LIBRARY_DEPENDENCIES}")
+
 # Set version
 set(LibArchive_VERSION ${libarchive_PKGCONF_VERSION})
 
@@ -115,7 +113,3 @@ if(NOT TARGET LibArchive::LibArchive)
     endif()
 endif()
 
-if(APPLE)
-    # remove LibArchive-specific path
-    set(ENV{CMAKE_PREFIX_PATH} "$ENV{libarchive_PREV_CMAKE_PATH}")
-endif()

components/core/cmake/Modules/FindLibraryDependencies.cmake

@@ -10,6 +10,10 @@ macro(FindStaticLibraryDependencies fld_LIBNAME fld_PREFIX fld_STATIC_LIBS)
     # NOTE: libc, libm, libpthread, and librt should be dynamically linked
     set(fld_DYNAMIC_LIBS "c;m;pthread;rt")
 
+    if("${fld_PREFIX}" STREQUAL "libarchive")
+        set(fld_ALLOW_SHARED_LIBS_FALLBACK ON)
+    endif()
+
     # Get absolute path of dependent libraries
     foreach(fld_DEP_LIB ${fld_STATIC_LIBS})
         # Skip dynamic libs
@@ -31,6 +35,8 @@ macro(FindStaticLibraryDependencies fld_LIBNAME fld_PREFIX fld_STATIC_LIBS)
         if(${fld_PREFIX}_${fld_DEP_LIB}_LIBRARY)
             list(APPEND ${fld_PREFIX}_LIBRARY_DEPENDENCIES
                  "${${fld_PREFIX}_${fld_DEP_LIB}_LIBRARY}")
+        elseif(fld_ALLOW_SHARED_LIBS_FALLBACK)
+	    list(APPEND ${fld_PREFIX}_DYNAMIC_LIBS "${fld_DEP_LIB}")
         else()
             message(SEND_ERROR "Static ${fld_DEP_LIB} library not found")
         endif()

components/core/cmake/Options/options.cmake

@@ -133,11 +133,14 @@ function(set_clp_binaries_dependencies)
     set_clp_need_flags(
         CLP_NEED_ABSL
         CLP_NEED_BOOST
+        CLP_NEED_BZIP2
         CLP_NEED_CURL
         CLP_NEED_DATE
         CLP_NEED_FMT
         CLP_NEED_LIBARCHIVE
         CLP_NEED_LOG_SURGEON
+        CLP_NEED_LZ4
+        CLP_NEED_LZMA
         CLP_NEED_MARIADB
         CLP_NEED_MONGOCXX
         CLP_NEED_MSGPACKCXX
@@ -148,6 +151,7 @@ function(set_clp_binaries_dependencies)
         CLP_NEED_SQLITE
         CLP_NEED_YAMLCPP
         CLP_NEED_YSTDLIB
+        CLP_NEED_ZLIB
         CLP_NEED_ZSTD
     )
 endfunction()
@@ -167,11 +171,13 @@ function(set_clp_tests_dependencies)
     set_clp_need_flags(
         CLP_NEED_ABSL
         CLP_NEED_BOOST
+        CLP_NEED_BZIP2
         CLP_NEED_CATCH2
         CLP_NEED_DATE
         CLP_NEED_FMT
         CLP_NEED_LIBARCHIVE
         CLP_NEED_LOG_SURGEON
+        CLP_NEED_LZ4
         CLP_NEED_LZMA
         CLP_NEED_MARIADB
         CLP_NEED_MONGOCXX
@@ -182,6 +188,7 @@ function(set_clp_tests_dependencies)
         CLP_NEED_SQLITE
         CLP_NEED_YAMLCPP
         CLP_NEED_YSTDLIB
+        CLP_NEED_ZLIB
         CLP_NEED_ZSTD
     )
 endfunction()
@@ -450,12 +457,14 @@ function (convert_clp_dependency_properties_to_variables)
         CLP_NEED_ABSL
         CLP_NEED_ANTLR
         CLP_NEED_BOOST
+        CLP_NEED_BZIP2
         CLP_NEED_CATCH2
         CLP_NEED_CURL
         CLP_NEED_DATE
         CLP_NEED_FMT
         CLP_NEED_LOG_SURGEON
         CLP_NEED_LIBARCHIVE
+        CLP_NEED_LZ4
         CLP_NEED_LZMA
         CLP_NEED_MARIADB
         CLP_NEED_MONGOCXX
@@ -467,6 +476,7 @@ function (convert_clp_dependency_properties_to_variables)
         CLP_NEED_SQLITE
         CLP_NEED_YAMLCPP
         CLP_NEED_YSTDLIB
+        CLP_NEED_ZLIB
         CLP_NEED_ZSTD
     )
 

components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh

@@ -7,18 +7,20 @@ set -e
 set -u
 
 dnf install -y \
+    bzip2-devel \
     cmake \
     diffutils \
     gcc-c++ \
     git \
     java-11-openjdk \
     jq \
-    libarchive-devel \
     libcurl-devel \
     libzstd-devel \
+    lz4-devel \
     make \
     mariadb-connector-c-devel \
     openssl-devel \
+    python3-devel \
     python3-pip \
     unzip
 

components/core/tools/scripts/lib_install/macos/install-all.sh

@@ -40,7 +40,6 @@ brew install \
   coreutils \
   gcc \
   java11 \
-  libarchive \
   llvm@16 \
   lz4 \
   mariadb-connector-c \

components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh

@@ -17,6 +17,5 @@ lib_install_scripts_dir="${script_dir}/.."
 "${lib_install_scripts_dir}/liblzma.sh" 5.8.1
 "${lib_install_scripts_dir}/lz4.sh" 1.10.0
 "${lib_install_scripts_dir}/zstandard.sh" 1.5.7
-"${lib_install_scripts_dir}/libarchive.sh" 3.8.0
 
 "${lib_install_scripts_dir}/msgpack.sh" 7.0.0

components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh

@@ -4,6 +4,7 @@ set -eu
 set -o pipefail
 
 dnf install -y \
+    bzip2-devel \
     gcc-c++ \
     java-11-openjdk \
     jq \

components/core/tools/scripts/lib_install/musllinux_1_2/install-packages-from-source.sh

@@ -17,6 +17,5 @@ lib_install_scripts_dir="${script_dir}/.."
 "${lib_install_scripts_dir}/liblzma.sh" 5.8.1
 "${lib_install_scripts_dir}/lz4.sh" 1.10.0
 "${lib_install_scripts_dir}/zstandard.sh" 1.5.7
-"${lib_install_scripts_dir}/libarchive.sh" 3.8.0
 
 "${lib_install_scripts_dir}/msgpack.sh" 7.0.0