From 280ba24d21e1fcb37d5f449b453f80fcdcd9b99a Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 22 Jul 2025 18:12:30 +0000 Subject: [PATCH 01/20] Remove current libarchive installations from all os platforms --- .../install-prebuilt-packages.sh | 2 +- .../tools/scripts/lib_install/libarchive.sh | 95 ------------------- .../scripts/lib_install/macos/install-all.sh | 1 - .../install-packages-from-source.sh | 1 - .../install-packages-from-source.sh | 1 - .../install-packages-from-source.sh | 1 - .../ubuntu-jammy/install-prebuilt-packages.sh | 1 + 7 files changed, 2 insertions(+), 100 deletions(-) delete mode 100755 components/core/tools/scripts/lib_install/libarchive.sh diff --git a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh index 1d5b253a57..0619909e6d 100755 --- a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh @@ -13,12 +13,12 @@ dnf install -y \ git \ java-11-openjdk \ jq \ - libarchive-devel \ libcurl-devel \ libzstd-devel \ make \ mariadb-connector-c-devel \ openssl-devel \ + python3-devel \ python3-pip \ unzip diff --git a/components/core/tools/scripts/lib_install/libarchive.sh b/components/core/tools/scripts/lib_install/libarchive.sh deleted file mode 100755 index dd8c2b733e..0000000000 --- a/components/core/tools/scripts/lib_install/libarchive.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/bin/bash - -# Dependencies: -# - cmake -# - curl -# NOTE: Dependencies should be installed outside the script to allow the script to be largely distro-agnostic - -# Exit on any error -set -e - -cUsage="Usage: ${BASH_SOURCE[0]} [ <.deb output directory>]" -if [ "$#" -lt 1 ] ; then - echo $cUsage - exit -fi -version=$1 - -package_name=libarchive -temp_dir=/tmp/${package_name}-installation -deb_output_dir=${temp_dir} -if [[ "$#" -gt 1 ]] ; then - deb_output_dir="$(readlink -f "$2")" - if [ ! -d ${deb_output_dir} ] ; then - echo "${deb_output_dir} does not exist or is not a directory" - exit - fi -fi - -# Check if already installed -set +e -dpkg -l ${package_name} | grep ${version} -installed=$? -set -e -if [ $installed -eq 0 ] ; then - # Nothing to do - exit -fi - -echo "Checking for elevated privileges..." -install_cmd_args=() -if [ ${EUID:-$(id -u)} -ne 0 ] ; then - sudo echo "Script can elevate privileges." - install_cmd_args+=("sudo") -fi - -# Get number of cpu cores -num_cpus=$(grep -c ^processor /proc/cpuinfo) - -# Download -mkdir -p $temp_dir -cd $temp_dir -extracted_dir=${temp_dir}/${package_name}-${version} -if [ ! -e ${extracted_dir} ] ; then - tar_filename=${package_name}-${version}.tar.gz - if [ ! -e ${tar_filename} ] ; then - curl -fsSL https://www.libarchive.org/downloads/${tar_filename} -o ${tar_filename} - fi - - tar -xf ${tar_filename} -fi - -# Build -cd ${extracted_dir} -mkdir -p cmake-build-release -cd cmake-build-release -# NOTE: Disable Expat and OpenSSL so the static libarchive doesn't look for them at link time. -cmake -DENABLE_EXPAT=OFF -DENABLE_OPENSSL=OFF ../ -make -j${num_cpus} - -# Check if checkinstall is installed -set +e -command -v checkinstall -checkinstall_installed=$? -set -e - -# Install -if [ $checkinstall_installed -eq 0 ] ; then - install_cmd_args+=( - checkinstall - --default - --fstrans=no - --nodoc - --pkgname "${package_name}" - --pkgversion "${version}" - --provides "${package_name}" - --pakdir "${deb_output_dir}" - ) -fi -install_cmd_args+=( - make install -) -"${install_cmd_args[@]}" - -# Clean up -rm -rf $temp_dir diff --git a/components/core/tools/scripts/lib_install/macos/install-all.sh b/components/core/tools/scripts/lib_install/macos/install-all.sh index 57d77af764..b380e67027 100755 --- a/components/core/tools/scripts/lib_install/macos/install-all.sh +++ b/components/core/tools/scripts/lib_install/macos/install-all.sh @@ -40,7 +40,6 @@ brew install \ coreutils \ gcc \ java11 \ - libarchive \ llvm@16 \ lz4 \ mariadb-connector-c \ diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh index d31343c8ee..349cb9ce1c 100755 --- a/components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh @@ -17,6 +17,5 @@ lib_install_scripts_dir="${script_dir}/.." "${lib_install_scripts_dir}/liblzma.sh" 5.8.1 "${lib_install_scripts_dir}/lz4.sh" 1.10.0 "${lib_install_scripts_dir}/zstandard.sh" 1.5.7 -"${lib_install_scripts_dir}/libarchive.sh" 3.8.0 "${lib_install_scripts_dir}/msgpack.sh" 7.0.0 diff --git a/components/core/tools/scripts/lib_install/musllinux_1_2/install-packages-from-source.sh b/components/core/tools/scripts/lib_install/musllinux_1_2/install-packages-from-source.sh index e1f65b9bcd..fda3352b42 100755 --- a/components/core/tools/scripts/lib_install/musllinux_1_2/install-packages-from-source.sh +++ b/components/core/tools/scripts/lib_install/musllinux_1_2/install-packages-from-source.sh @@ -17,6 +17,5 @@ lib_install_scripts_dir="${script_dir}/.." "${lib_install_scripts_dir}/liblzma.sh" 5.8.1 "${lib_install_scripts_dir}/lz4.sh" 1.10.0 "${lib_install_scripts_dir}/zstandard.sh" 1.5.7 -"${lib_install_scripts_dir}/libarchive.sh" 3.8.0 "${lib_install_scripts_dir}/msgpack.sh" 7.0.0 diff --git a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-packages-from-source.sh b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-packages-from-source.sh index af1d2774a9..813c4277d9 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-packages-from-source.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-packages-from-source.sh @@ -12,7 +12,6 @@ lib_install_scripts_dir=$script_dir/.. # NOTE: boost must be installed first since the remaining packages depend on it "$lib_install_scripts_dir"/install-boost.sh 1.87.0 -"$lib_install_scripts_dir"/libarchive.sh 3.5.1 "$lib_install_scripts_dir"/liblzma.sh 5.8.1 "$lib_install_scripts_dir"/lz4.sh 1.8.2 "$lib_install_scripts_dir"/msgpack.sh 7.0.0 diff --git a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh index 03a3c26e8c..b13a4ab6f3 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh @@ -23,6 +23,7 @@ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ openjdk-11-jdk \ pkg-config \ python3 \ + python3-dev \ python3-pip \ python3-venv \ software-properties-common \ From 0f97d9b5b67fb58248f10d5c11713135f70ac801 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 22 Jul 2025 18:25:58 +0000 Subject: [PATCH 02/20] Use task to install a fixed version of libarchive --- components/core/CMakeLists.txt | 2 +- .../core/cmake/Modules/FindLibArchive.cmake | 121 ------------------ taskfiles/deps/main.yaml | 18 +++ 3 files changed, 19 insertions(+), 122 deletions(-) delete mode 100644 components/core/cmake/Modules/FindLibArchive.cmake diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index f71f7056ea..d463bdf69c 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -209,7 +209,7 @@ if(CLP_NEED_LIBARCHIVE) if(CLP_USE_STATIC_LIBS) set(LibArchive_USE_STATIC_LIBS ON) endif() - find_package(LibArchive REQUIRED) + find_package(LibArchive 3.8.1 REQUIRED) if(LibArchive_FOUND) message(STATUS "Found LibArchive ${LibArchive_VERSION}") else() diff --git a/components/core/cmake/Modules/FindLibArchive.cmake b/components/core/cmake/Modules/FindLibArchive.cmake deleted file mode 100644 index c257e88341..0000000000 --- a/components/core/cmake/Modules/FindLibArchive.cmake +++ /dev/null @@ -1,121 +0,0 @@ -# Try to find LibArchive -# NOTE: The FindLibArchive.cmake included with CMake has no support for static libraries, so we use our own. -# -# Set LibArchive_USE_STATIC_LIBS=ON to look for static libraries. -# -# Once done this will define: -# LibArchive_FOUND - Whether LibArchive was found on the system -# LibArchive_INCLUDE_DIR - The LibArchive include directories -# LibArchive_VERSION - The version of LibArchive installed on the system -# -# Conventions: -# - Variables only for use within the script are prefixed with "libarchive_" -# - Variables that should be externally visible are prefixed with "LibArchive_" - -set(libarchive_LIBNAME "archive") - -include(cmake/Modules/FindLibraryDependencies.cmake) - -# On macOS, libarchive installed through brew is not linked into prefix by default. -# So it cannot be found by pkg-config and we need to manually find it. -# For more details, see https://github.com/Homebrew/homebrew-core/issues/117642 -# Find and setup libarchive -if(APPLE) - execute_process(COMMAND brew --prefix libarchive OUTPUT_VARIABLE libarchive_MACOS_PREFIX) - string(STRIP "${libarchive_MACOS_PREFIX}" libarchive_MACOS_PREFIX) - set(ENV{libarchive_PREV_CMAKE_PATH} "$ENV{CMAKE_PREFIX_PATH}") # save it so we can revert it later - set(ENV{CMAKE_PREFIX_PATH} "${libarchive_MACOS_PREFIX};$ENV{CMAKE_PREFIX_PATH}") -endif() - -# Run pkg-config -find_package(PkgConfig) -pkg_check_modules(libarchive_PKGCONF QUIET "lib${libarchive_LIBNAME}") - -# Set include directory -find_path(LibArchive_INCLUDE_DIR archive.h - HINTS ${libarchive_PKGCONF_INCLUDEDIR} - PATH_SUFFIXES include - ) - -# Handle static libraries -if(LibArchive_USE_STATIC_LIBS) - # Save current value of CMAKE_FIND_LIBRARY_SUFFIXES - set(libarchive_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) - - # Temporarily change CMAKE_FIND_LIBRARY_SUFFIXES to static library suffix - set(CMAKE_FIND_LIBRARY_SUFFIXES .a) -endif() - -# Find library -find_library(LibArchive_LIBRARY - NAMES ${libarchive_LIBNAME} - HINTS ${libarchive_PKGCONF_LIBDIR} - PATH_SUFFIXES lib - ) -if (LibArchive_LIBRARY) - # NOTE: This must be set for find_package_handle_standard_args to work - set(LibArchive_FOUND ON) -endif() - -if(LibArchive_USE_STATIC_LIBS) - FindStaticLibraryDependencies(${libarchive_LIBNAME} libarchive - "${libarchive_PKGCONF_STATIC_LIBRARIES}") - - # Restore original value of CMAKE_FIND_LIBRARY_SUFFIXES - set(CMAKE_FIND_LIBRARY_SUFFIXES ${libarchive_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES}) - unset(libarchive_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES) -endif() - -FindDynamicLibraryDependencies(libarchive "${libarchive_DYNAMIC_LIBS}") - -# Set version -set(LibArchive_VERSION ${libarchive_PKGCONF_VERSION}) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(LibArchive - REQUIRED_VARS LibArchive_INCLUDE_DIR - VERSION_VAR LibArchive_VERSION - ) - -if(NOT TARGET LibArchive::LibArchive) - # Add library to build - if (LibArchive_FOUND) - if (LibArchive_USE_STATIC_LIBS) - add_library(LibArchive::LibArchive STATIC IMPORTED) - else() - # NOTE: We use UNKNOWN so that if the user doesn't have the SHARED - # libraries installed, we can still use the STATIC libraries - add_library(LibArchive::LibArchive UNKNOWN IMPORTED) - endif() - endif() - - # Set include directories for library - if(LibArchive_INCLUDE_DIR) - set_target_properties(LibArchive::LibArchive - PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${LibArchive_INCLUDE_DIR}" - ) - endif() - - # Set location of library - if(EXISTS "${LibArchive_LIBRARY}") - set_target_properties(LibArchive::LibArchive - PROPERTIES - IMPORTED_LINK_INTERFACE_LANGUAGES "C" - IMPORTED_LOCATION "${LibArchive_LIBRARY}" - ) - - # Add component's dependencies for linking - if(libarchive_LIBRARY_DEPENDENCIES) - set_target_properties(LibArchive::LibArchive - PROPERTIES - INTERFACE_LINK_LIBRARIES "${libarchive_LIBRARY_DEPENDENCIES}" - ) - endif() - endif() -endif() - -if(APPLE) - # remove LibArchive-specific path - set(ENV{CMAKE_PREFIX_PATH} "$ENV{libarchive_PREV_CMAKE_PATH}") -endif() \ No newline at end of file diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index cf47528703..accb616e27 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -65,6 +65,7 @@ tasks: - task: "catch2" - task: "date" - task: "fmt" + - task: "libarchive" - task: "log-surgeon" - task: "mongocxx" - task: "nlohmann_json" @@ -217,6 +218,23 @@ tasks: TARBALL_SHA256: "1250e4cc58bf06ee631567523f48848dc4596133e163f02615c97f78bab6c811" TARBALL_URL: "https://github.com/fmtlib/fmt/archive/refs/tags/10.2.1.tar.gz" + libarchive: + internal: true + run: "once" + cmds: + - task: "utils:install-remote-cmake-lib" + vars: + CMAKE_GEN_ARGS: + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_LIBDIR=lib" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + - "-DCMAKE_INSTALL_PREFIX={{.G_DEPS_CORE_DIR}}/libarchive-install" + - "-DENABLE_EXPAT=OFF" + - "-DENABLE_OPENSSL=OFF" + LIB_NAME: "LibArchive" + TARBALL_SHA256: "169b56714b620c3f8f07bf7975aba8573b9bd25e423e5c4f0cc9b52450a2403d" + TARBALL_URL: "https://github.com/libarchive/libarchive/releases/download/v3.8.1/libarchive-3.8.1.tar.gz" + log-surgeon: internal: true run: "once" From 0f686fde77edc6311bdd6fe77c015a699084c298 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 22 Jul 2025 19:39:14 +0000 Subject: [PATCH 03/20] Revert LibArchive static linking module --- .../core/cmake/Modules/FindLibArchive.cmake | 121 ++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 components/core/cmake/Modules/FindLibArchive.cmake diff --git a/components/core/cmake/Modules/FindLibArchive.cmake b/components/core/cmake/Modules/FindLibArchive.cmake new file mode 100644 index 0000000000..c257e88341 --- /dev/null +++ b/components/core/cmake/Modules/FindLibArchive.cmake @@ -0,0 +1,121 @@ +# Try to find LibArchive +# NOTE: The FindLibArchive.cmake included with CMake has no support for static libraries, so we use our own. +# +# Set LibArchive_USE_STATIC_LIBS=ON to look for static libraries. +# +# Once done this will define: +# LibArchive_FOUND - Whether LibArchive was found on the system +# LibArchive_INCLUDE_DIR - The LibArchive include directories +# LibArchive_VERSION - The version of LibArchive installed on the system +# +# Conventions: +# - Variables only for use within the script are prefixed with "libarchive_" +# - Variables that should be externally visible are prefixed with "LibArchive_" + +set(libarchive_LIBNAME "archive") + +include(cmake/Modules/FindLibraryDependencies.cmake) + +# On macOS, libarchive installed through brew is not linked into prefix by default. +# So it cannot be found by pkg-config and we need to manually find it. +# For more details, see https://github.com/Homebrew/homebrew-core/issues/117642 +# Find and setup libarchive +if(APPLE) + execute_process(COMMAND brew --prefix libarchive OUTPUT_VARIABLE libarchive_MACOS_PREFIX) + string(STRIP "${libarchive_MACOS_PREFIX}" libarchive_MACOS_PREFIX) + set(ENV{libarchive_PREV_CMAKE_PATH} "$ENV{CMAKE_PREFIX_PATH}") # save it so we can revert it later + set(ENV{CMAKE_PREFIX_PATH} "${libarchive_MACOS_PREFIX};$ENV{CMAKE_PREFIX_PATH}") +endif() + +# Run pkg-config +find_package(PkgConfig) +pkg_check_modules(libarchive_PKGCONF QUIET "lib${libarchive_LIBNAME}") + +# Set include directory +find_path(LibArchive_INCLUDE_DIR archive.h + HINTS ${libarchive_PKGCONF_INCLUDEDIR} + PATH_SUFFIXES include + ) + +# Handle static libraries +if(LibArchive_USE_STATIC_LIBS) + # Save current value of CMAKE_FIND_LIBRARY_SUFFIXES + set(libarchive_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) + + # Temporarily change CMAKE_FIND_LIBRARY_SUFFIXES to static library suffix + set(CMAKE_FIND_LIBRARY_SUFFIXES .a) +endif() + +# Find library +find_library(LibArchive_LIBRARY + NAMES ${libarchive_LIBNAME} + HINTS ${libarchive_PKGCONF_LIBDIR} + PATH_SUFFIXES lib + ) +if (LibArchive_LIBRARY) + # NOTE: This must be set for find_package_handle_standard_args to work + set(LibArchive_FOUND ON) +endif() + +if(LibArchive_USE_STATIC_LIBS) + FindStaticLibraryDependencies(${libarchive_LIBNAME} libarchive + "${libarchive_PKGCONF_STATIC_LIBRARIES}") + + # Restore original value of CMAKE_FIND_LIBRARY_SUFFIXES + set(CMAKE_FIND_LIBRARY_SUFFIXES ${libarchive_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES}) + unset(libarchive_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES) +endif() + +FindDynamicLibraryDependencies(libarchive "${libarchive_DYNAMIC_LIBS}") + +# Set version +set(LibArchive_VERSION ${libarchive_PKGCONF_VERSION}) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(LibArchive + REQUIRED_VARS LibArchive_INCLUDE_DIR + VERSION_VAR LibArchive_VERSION + ) + +if(NOT TARGET LibArchive::LibArchive) + # Add library to build + if (LibArchive_FOUND) + if (LibArchive_USE_STATIC_LIBS) + add_library(LibArchive::LibArchive STATIC IMPORTED) + else() + # NOTE: We use UNKNOWN so that if the user doesn't have the SHARED + # libraries installed, we can still use the STATIC libraries + add_library(LibArchive::LibArchive UNKNOWN IMPORTED) + endif() + endif() + + # Set include directories for library + if(LibArchive_INCLUDE_DIR) + set_target_properties(LibArchive::LibArchive + PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${LibArchive_INCLUDE_DIR}" + ) + endif() + + # Set location of library + if(EXISTS "${LibArchive_LIBRARY}") + set_target_properties(LibArchive::LibArchive + PROPERTIES + IMPORTED_LINK_INTERFACE_LANGUAGES "C" + IMPORTED_LOCATION "${LibArchive_LIBRARY}" + ) + + # Add component's dependencies for linking + if(libarchive_LIBRARY_DEPENDENCIES) + set_target_properties(LibArchive::LibArchive + PROPERTIES + INTERFACE_LINK_LIBRARIES "${libarchive_LIBRARY_DEPENDENCIES}" + ) + endif() + endif() +endif() + +if(APPLE) + # remove LibArchive-specific path + set(ENV{CMAKE_PREFIX_PATH} "$ENV{libarchive_PREV_CMAKE_PATH}") +endif() \ No newline at end of file From 88f1eed7206dfc61fec5292862c6d9461a0aa6f8 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 22 Jul 2025 17:16:14 -0400 Subject: [PATCH 04/20] Add missing target links to make ubuntu-jammy compile --- components/core/CMakeLists.txt | 39 ++++++++++++++++----- components/core/cmake/Options/options.cmake | 6 ++++ components/core/src/clp/clp/CMakeLists.txt | 13 ++++--- components/core/src/glt/glt/CMakeLists.txt | 9 ++--- 4 files changed, 49 insertions(+), 18 deletions(-) diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index d463bdf69c..2ab8863374 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -183,6 +183,16 @@ if(CLP_NEED_LOG_SURGEON) endif() endif() +if(CLP_NEED_LZ4) + if(CLP_USE_STATIC_LIBS) + set(LZ4_USE_STATIC_LIBS ON) + endif() + find_package(LZ4 REQUIRED) + if (LZ4_FOUND) + message(STATUS "Found LZ4 ${LZ4_VERSION}") + endif() +endif() + if(CLP_NEED_NLOHMANN_JSON) find_package(nlohmann_json REQUIRED) if(nlohmann_json_FOUND) @@ -288,6 +298,16 @@ if(CLP_NEED_YSTDLIB) add_subdirectory("${CLP_YSTDLIB_SOURCE_DIRECTORY}" "${CMAKE_BINARY_DIR}/ystdlib" EXCLUDE_FROM_ALL) endif() +# Find and setup ZLIB +if(CLP_NEED_ZLIB) + find_package(ZLIB REQUIRED) + if(ZLIB_FOUND) + message(STATUS "Found ZLIB ${ZLIB_VERSION_STRING}") + else() + message(FATAL_ERROR "Could not find ${CLP_LIBS_STRING} libraries for ZLIB") + endif() +endif() + # Find and setup ZStd Library if(CLP_NEED_ZSTD) if(CLP_USE_STATIC_LIBS) @@ -743,32 +763,33 @@ if(CLP_BUILD_TESTING) ) target_link_libraries(unitTest PRIVATE + ${CURL_LIBRARIES} + ${LIBLZMA_LIBRARIES} + ${MONGOCXX_TARGET} + ${sqlite_LIBRARY_DEPENDENCIES} + ${STD_FS_LIBS} absl::flat_hash_map Boost::filesystem Boost::iostreams Boost::program_options Boost::regex Boost::url Catch2::Catch2 - ${CURL_LIBRARIES} + clp::regex_utils + clp::string_utils clp_s::search::ast clp_s::search::kql clp_s::search::sql clp_s::timestamp_pattern date::date fmt::fmt - log_surgeon::log_surgeon LibArchive::LibArchive + LZ4::LZ4 + log_surgeon::log_surgeon MariaDBClient::MariaDBClient - ${MONGOCXX_TARGET} nlohmann_json::nlohmann_json + OpenSSL::Crypto simdjson::simdjson spdlog::spdlog - OpenSSL::Crypto - ${sqlite_LIBRARY_DEPENDENCIES} - ${STD_FS_LIBS} - clp::regex_utils - clp::string_utils yaml-cpp ystdlib::containers ystdlib::error_handling - ${LIBLZMA_LIBRARIES} ZStd::ZStd ) target_compile_features(unitTest diff --git a/components/core/cmake/Options/options.cmake b/components/core/cmake/Options/options.cmake index 40b1c610e7..cf793f6666 100644 --- a/components/core/cmake/Options/options.cmake +++ b/components/core/cmake/Options/options.cmake @@ -138,6 +138,8 @@ function(set_clp_binaries_dependencies) CLP_NEED_FMT CLP_NEED_LIBARCHIVE CLP_NEED_LOG_SURGEON + CLP_NEED_LZ4 + CLP_NEED_LZMA CLP_NEED_MARIADB CLP_NEED_MONGOCXX CLP_NEED_MSGPACKCXX @@ -148,6 +150,7 @@ function(set_clp_binaries_dependencies) CLP_NEED_SQLITE CLP_NEED_YAMLCPP CLP_NEED_YSTDLIB + CLP_NEED_ZLIB CLP_NEED_ZSTD ) endfunction() @@ -172,6 +175,7 @@ function(set_clp_tests_dependencies) CLP_NEED_FMT CLP_NEED_LIBARCHIVE CLP_NEED_LOG_SURGEON + CLP_NEED_LZ4 CLP_NEED_LZMA CLP_NEED_MARIADB CLP_NEED_MONGOCXX @@ -456,6 +460,7 @@ function (convert_clp_dependency_properties_to_variables) CLP_NEED_FMT CLP_NEED_LOG_SURGEON CLP_NEED_LIBARCHIVE + CLP_NEED_LZ4 CLP_NEED_LZMA CLP_NEED_MARIADB CLP_NEED_MONGOCXX @@ -467,6 +472,7 @@ function (convert_clp_dependency_properties_to_variables) CLP_NEED_SQLITE CLP_NEED_YAMLCPP CLP_NEED_YSTDLIB + CLP_NEED_ZLIB CLP_NEED_ZSTD ) diff --git a/components/core/src/clp/clp/CMakeLists.txt b/components/core/src/clp/clp/CMakeLists.txt index c9782c509f..6cf93034d8 100644 --- a/components/core/src/clp/clp/CMakeLists.txt +++ b/components/core/src/clp/clp/CMakeLists.txt @@ -174,20 +174,23 @@ if(CLP_BUILD_EXECUTABLES) ) target_link_libraries(clp PRIVATE + ${sqlite_LIBRARY_DEPENDENCIES} + ${STD_FS_LIBS} Boost::filesystem Boost::program_options + clp::string_utils date::date fmt::fmt - log_surgeon::log_surgeon - spdlog::spdlog - ${sqlite_LIBRARY_DEPENDENCIES} LibArchive::LibArchive + LibLZMA::LibLZMA + log_surgeon::log_surgeon + LZ4::LZ4 MariaDBClient::MariaDBClient nlohmann_json::nlohmann_json - ${STD_FS_LIBS} - clp::string_utils + spdlog::spdlog yaml-cpp ystdlib::containers ystdlib::error_handling + ZLIB::ZLIB ZStd::ZStd ) # Put the built executable at the root of the build directory diff --git a/components/core/src/glt/glt/CMakeLists.txt b/components/core/src/glt/glt/CMakeLists.txt index 8576f391f5..5d8e6d5f47 100644 --- a/components/core/src/glt/glt/CMakeLists.txt +++ b/components/core/src/glt/glt/CMakeLists.txt @@ -180,16 +180,17 @@ if(CLP_BUILD_EXECUTABLES) ) target_link_libraries(glt PRIVATE + ${sqlite_LIBRARY_DEPENDENCIES} + ${STD_FS_LIBS} Boost::filesystem Boost::iostreams Boost::program_options + clp::string_utils date::date fmt::fmt - spdlog::spdlog - ${sqlite_LIBRARY_DEPENDENCIES} LibArchive::LibArchive + LZ4::LZ4 MariaDBClient::MariaDBClient nlohmann_json::nlohmann_json - ${STD_FS_LIBS} - clp::string_utils + spdlog::spdlog yaml-cpp ystdlib::error_handling ZStd::ZStd From 2cacc5d9497719719d3c258df139a1c38e8eef2d Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 22 Jul 2025 17:20:26 -0400 Subject: [PATCH 05/20] Fix library name --- taskfiles/deps/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index accb616e27..23ed5a3aa9 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -228,7 +228,7 @@ tasks: - "-DCMAKE_BUILD_TYPE=Release" - "-DCMAKE_INSTALL_LIBDIR=lib" - "-DCMAKE_INSTALL_MESSAGE=LAZY" - - "-DCMAKE_INSTALL_PREFIX={{.G_DEPS_CORE_DIR}}/libarchive-install" + - "-DCMAKE_INSTALL_PREFIX={{.G_DEPS_CORE_DIR}}/LibArchive-install" - "-DENABLE_EXPAT=OFF" - "-DENABLE_OPENSSL=OFF" LIB_NAME: "LibArchive" From 2597a210adec9b7be202cb8d9fd51a9bd8af50ba Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Tue, 22 Jul 2025 17:39:21 -0400 Subject: [PATCH 06/20] Misc fixes --- .../lib_install/centos-stream-9/install-prebuilt-packages.sh | 1 + taskfiles/deps/main.yaml | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh index 0619909e6d..da96c6a18d 100755 --- a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh @@ -15,6 +15,7 @@ dnf install -y \ jq \ libcurl-devel \ libzstd-devel \ + lz4-devel \ make \ mariadb-connector-c-devel \ openssl-devel \ diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index 23ed5a3aa9..e5fe970494 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -233,7 +233,8 @@ tasks: - "-DENABLE_OPENSSL=OFF" LIB_NAME: "LibArchive" TARBALL_SHA256: "169b56714b620c3f8f07bf7975aba8573b9bd25e423e5c4f0cc9b52450a2403d" - TARBALL_URL: "https://github.com/libarchive/libarchive/releases/download/v3.8.1/libarchive-3.8.1.tar.gz" + TARBALL_URL: "https://github.com/libarchive/libarchive/releases/download/v3.8.1\ + /libarchive-3.8.1.tar.gz" log-surgeon: internal: true From 3a539732fb5fa6a60c4c9744d942583062d5e634 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 23 Jul 2025 03:04:42 -0400 Subject: [PATCH 07/20] Add missing deps (bzip2) --- components/core/CMakeLists.txt | 8 ++++++++ components/core/cmake/Options/options.cmake | 3 +++ components/core/src/clp/clp/CMakeLists.txt | 1 + components/core/src/glt/glt/CMakeLists.txt | 1 + .../centos-stream-9/install-prebuilt-packages.sh | 1 + .../lib_install/ubuntu-jammy/install-prebuilt-packages.sh | 1 + 6 files changed, 15 insertions(+) diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index 2ab8863374..67d3d6f822 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -155,6 +155,13 @@ if(CLP_NEED_BOOST) endif() endif() +if(CLP_NEED_BZIP2) + find_package(BZip2 REQUIRED) + if (BZip2_FOUND) + message(STATUS "Found BZip2") + endif() +endif() + if(CLP_NEED_CATCH2) find_package(Catch2 REQUIRED) if (Catch2_FOUND) @@ -780,6 +787,7 @@ if(CLP_BUILD_TESTING) date::date fmt::fmt LibArchive::LibArchive + LibLZMA::LibLZMA LZ4::LZ4 log_surgeon::log_surgeon MariaDBClient::MariaDBClient diff --git a/components/core/cmake/Options/options.cmake b/components/core/cmake/Options/options.cmake index cf793f6666..a13de0184d 100644 --- a/components/core/cmake/Options/options.cmake +++ b/components/core/cmake/Options/options.cmake @@ -133,6 +133,7 @@ function(set_clp_binaries_dependencies) set_clp_need_flags( CLP_NEED_ABSL CLP_NEED_BOOST + CLP_NEED_BZIP2 CLP_NEED_CURL CLP_NEED_DATE CLP_NEED_FMT @@ -170,6 +171,7 @@ function(set_clp_tests_dependencies) set_clp_need_flags( CLP_NEED_ABSL CLP_NEED_BOOST + CLP_NEED_BZIP2 CLP_NEED_CATCH2 CLP_NEED_DATE CLP_NEED_FMT @@ -454,6 +456,7 @@ function (convert_clp_dependency_properties_to_variables) CLP_NEED_ABSL CLP_NEED_ANTLR CLP_NEED_BOOST + CLP_NEED_BZIP2 CLP_NEED_CATCH2 CLP_NEED_CURL CLP_NEED_DATE diff --git a/components/core/src/clp/clp/CMakeLists.txt b/components/core/src/clp/clp/CMakeLists.txt index 6cf93034d8..5e2ffc5184 100644 --- a/components/core/src/clp/clp/CMakeLists.txt +++ b/components/core/src/clp/clp/CMakeLists.txt @@ -181,6 +181,7 @@ if(CLP_BUILD_EXECUTABLES) date::date fmt::fmt LibArchive::LibArchive + ${BZIP2_LIBRARIES} LibLZMA::LibLZMA log_surgeon::log_surgeon LZ4::LZ4 diff --git a/components/core/src/glt/glt/CMakeLists.txt b/components/core/src/glt/glt/CMakeLists.txt index 5d8e6d5f47..07159631d3 100644 --- a/components/core/src/glt/glt/CMakeLists.txt +++ b/components/core/src/glt/glt/CMakeLists.txt @@ -187,6 +187,7 @@ if(CLP_BUILD_EXECUTABLES) date::date fmt::fmt LibArchive::LibArchive + LibLZMA::LibLZMA LZ4::LZ4 MariaDBClient::MariaDBClient nlohmann_json::nlohmann_json diff --git a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh index da96c6a18d..ed98f49933 100755 --- a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh @@ -7,6 +7,7 @@ set -e set -u dnf install -y \ + bzip2-devel \ cmake \ diffutils \ gcc-c++ \ diff --git a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh index b13a4ab6f3..4fb7baecc9 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh @@ -8,6 +8,7 @@ set -u apt-get update DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ + libbz2-dev \ ca-certificates \ checkinstall \ cmake \ From 3ff86690bf2150bc58f6cc2b6f8d4fd447b2d163 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 23 Jul 2025 04:53:55 -0400 Subject: [PATCH 08/20] Add missing bzip2 for manylinux --- .../lib_install/manylinux_2_28/install-prebuilt-packages.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh index 57ea2caab0..b4a5b88171 100755 --- a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh @@ -4,6 +4,7 @@ set -eu set -o pipefail dnf install -y \ + bzip2-devel \ gcc-c++ \ java-11-openjdk \ jq \ From 8a2407b44cbaf375ac80eef32a530e1dd23864ff Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 23 Jul 2025 06:01:33 -0400 Subject: [PATCH 09/20] Address coderabbit ai review --- components/core/CMakeLists.txt | 1 - components/core/cmake/Options/options.cmake | 1 + components/core/src/clp/clp/CMakeLists.txt | 3 ++- taskfiles/deps/main.yaml | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index 67d3d6f822..7bc0690e69 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -787,7 +787,6 @@ if(CLP_BUILD_TESTING) date::date fmt::fmt LibArchive::LibArchive - LibLZMA::LibLZMA LZ4::LZ4 log_surgeon::log_surgeon MariaDBClient::MariaDBClient diff --git a/components/core/cmake/Options/options.cmake b/components/core/cmake/Options/options.cmake index a13de0184d..6d89b1098c 100644 --- a/components/core/cmake/Options/options.cmake +++ b/components/core/cmake/Options/options.cmake @@ -188,6 +188,7 @@ function(set_clp_tests_dependencies) CLP_NEED_SQLITE CLP_NEED_YAMLCPP CLP_NEED_YSTDLIB + CLP_NEED_ZLIB CLP_NEED_ZSTD ) endfunction() diff --git a/components/core/src/clp/clp/CMakeLists.txt b/components/core/src/clp/clp/CMakeLists.txt index 5e2ffc5184..35bab97b6b 100644 --- a/components/core/src/clp/clp/CMakeLists.txt +++ b/components/core/src/clp/clp/CMakeLists.txt @@ -181,7 +181,8 @@ if(CLP_BUILD_EXECUTABLES) date::date fmt::fmt LibArchive::LibArchive - ${BZIP2_LIBRARIES} + # BZip2 must be linked after static LibArchive for musllinux + BZip2::BZip2 LibLZMA::LibLZMA log_surgeon::log_surgeon LZ4::LZ4 diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index e5fe970494..965a4382d5 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -232,7 +232,7 @@ tasks: - "-DENABLE_EXPAT=OFF" - "-DENABLE_OPENSSL=OFF" LIB_NAME: "LibArchive" - TARBALL_SHA256: "169b56714b620c3f8f07bf7975aba8573b9bd25e423e5c4f0cc9b52450a2403d" + TARBALL_SHA256: "bde832a5e3344dc723cfe9cc37f8e54bde04565bfe6f136bc1bd31ab352e9fab" TARBALL_URL: "https://github.com/libarchive/libarchive/releases/download/v3.8.1\ /libarchive-3.8.1.tar.gz" From 3233f2fcf92aa7479732b90f21b0dafdd383c963 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 23 Jul 2025 20:44:44 +0000 Subject: [PATCH 10/20] Replace broken LibLZMA target with old-fashioned LZMA libraries variable --- components/core/src/clp/clp/CMakeLists.txt | 2 +- components/core/src/glt/glt/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/core/src/clp/clp/CMakeLists.txt b/components/core/src/clp/clp/CMakeLists.txt index 35bab97b6b..0df121732b 100644 --- a/components/core/src/clp/clp/CMakeLists.txt +++ b/components/core/src/clp/clp/CMakeLists.txt @@ -174,6 +174,7 @@ if(CLP_BUILD_EXECUTABLES) ) target_link_libraries(clp PRIVATE + ${LIBLZMA_LIBRARIES} ${sqlite_LIBRARY_DEPENDENCIES} ${STD_FS_LIBS} Boost::filesystem Boost::program_options @@ -183,7 +184,6 @@ if(CLP_BUILD_EXECUTABLES) LibArchive::LibArchive # BZip2 must be linked after static LibArchive for musllinux BZip2::BZip2 - LibLZMA::LibLZMA log_surgeon::log_surgeon LZ4::LZ4 MariaDBClient::MariaDBClient diff --git a/components/core/src/glt/glt/CMakeLists.txt b/components/core/src/glt/glt/CMakeLists.txt index 07159631d3..8551ca9191 100644 --- a/components/core/src/glt/glt/CMakeLists.txt +++ b/components/core/src/glt/glt/CMakeLists.txt @@ -180,6 +180,7 @@ if(CLP_BUILD_EXECUTABLES) ) target_link_libraries(glt PRIVATE + ${LIBLZMA_LIBRARIES} ${sqlite_LIBRARY_DEPENDENCIES} ${STD_FS_LIBS} Boost::filesystem Boost::iostreams Boost::program_options @@ -187,7 +188,6 @@ if(CLP_BUILD_EXECUTABLES) date::date fmt::fmt LibArchive::LibArchive - LibLZMA::LibLZMA LZ4::LZ4 MariaDBClient::MariaDBClient nlohmann_json::nlohmann_json From f61890bb160e3803594880427bd8cebd15294f94 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 23 Jul 2025 20:49:50 +0000 Subject: [PATCH 11/20] Optimize package find logging statement --- components/core/CMakeLists.txt | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index 7bc0690e69..4dad688738 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -157,9 +157,7 @@ endif() if(CLP_NEED_BZIP2) find_package(BZip2 REQUIRED) - if (BZip2_FOUND) - message(STATUS "Found BZip2") - endif() + message(STATUS "Found BZip2") endif() if(CLP_NEED_CATCH2) @@ -195,9 +193,7 @@ if(CLP_NEED_LZ4) set(LZ4_USE_STATIC_LIBS ON) endif() find_package(LZ4 REQUIRED) - if (LZ4_FOUND) - message(STATUS "Found LZ4 ${LZ4_VERSION}") - endif() + message(STATUS "Found LZ4 ${LZ4_VERSION}") endif() if(CLP_NEED_NLOHMANN_JSON) @@ -227,11 +223,7 @@ if(CLP_NEED_LIBARCHIVE) set(LibArchive_USE_STATIC_LIBS ON) endif() find_package(LibArchive 3.8.1 REQUIRED) - if(LibArchive_FOUND) - message(STATUS "Found LibArchive ${LibArchive_VERSION}") - else() - message(FATAL_ERROR "Could not find ${CLP_LIBS_STRING} libraries for LibArchive") - endif() + message(STATUS "Found LibArchive ${LibArchive_VERSION}") endif() # Find and setup libcurl @@ -308,11 +300,7 @@ endif() # Find and setup ZLIB if(CLP_NEED_ZLIB) find_package(ZLIB REQUIRED) - if(ZLIB_FOUND) - message(STATUS "Found ZLIB ${ZLIB_VERSION_STRING}") - else() - message(FATAL_ERROR "Could not find ${CLP_LIBS_STRING} libraries for ZLIB") - endif() + message(STATUS "Found ZLIB ${ZLIB_VERSION_STRING}") endif() # Find and setup ZStd Library From e78c0359ac9d31fd0cf2a2ceafbe6360a45ed9e6 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 23 Jul 2025 23:19:32 +0000 Subject: [PATCH 12/20] Use existing helper to find dep modules for libarchive --- components/core/CMakeLists.txt | 10 +++++++--- components/core/src/clp/clp/CMakeLists.txt | 6 +----- components/core/src/glt/glt/CMakeLists.txt | 3 +-- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index 4dad688738..a15ec0ea3b 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -12,6 +12,8 @@ validate_compiler_versions() # Include options for CLP build include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/Options/options.cmake") +include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/FindLibraryDependencies.cmake") + validate_and_setup_all_clp_dependency_flags() convert_clp_dependency_properties_to_variables() @@ -224,6 +226,10 @@ if(CLP_NEED_LIBARCHIVE) endif() find_package(LibArchive 3.8.1 REQUIRED) message(STATUS "Found LibArchive ${LibArchive_VERSION}") + set(LibArchive_STATIC_LIBS "lzma") + FindStaticLibraryDependencies(archive LibArchive "${LibArchive_STATIC_LIBS}") + set(LibArchive_DYNAMIC_LIBS "bz2;lz4;z") + FindDynamicLibraryDependencies(LibArchive "${LibArchive_DYNAMIC_LIBS}") endif() # Find and setup libcurl @@ -346,7 +352,6 @@ endif() # sqlite dependencies if(CLP_NEED_SQLITE) set(sqlite_DYNAMIC_LIBS "dl;m;pthread") - include(cmake/Modules/FindLibraryDependencies.cmake) FindDynamicLibraryDependencies(sqlite "${sqlite_DYNAMIC_LIBS}") endif() @@ -759,7 +764,6 @@ if(CLP_BUILD_TESTING) target_link_libraries(unitTest PRIVATE ${CURL_LIBRARIES} - ${LIBLZMA_LIBRARIES} ${MONGOCXX_TARGET} ${sqlite_LIBRARY_DEPENDENCIES} ${STD_FS_LIBS} @@ -775,7 +779,7 @@ if(CLP_BUILD_TESTING) date::date fmt::fmt LibArchive::LibArchive - LZ4::LZ4 + ${LibArchive_LIBRARY_DEPENDENCIES} log_surgeon::log_surgeon MariaDBClient::MariaDBClient nlohmann_json::nlohmann_json diff --git a/components/core/src/clp/clp/CMakeLists.txt b/components/core/src/clp/clp/CMakeLists.txt index 0df121732b..c795f21af8 100644 --- a/components/core/src/clp/clp/CMakeLists.txt +++ b/components/core/src/clp/clp/CMakeLists.txt @@ -174,7 +174,6 @@ if(CLP_BUILD_EXECUTABLES) ) target_link_libraries(clp PRIVATE - ${LIBLZMA_LIBRARIES} ${sqlite_LIBRARY_DEPENDENCIES} ${STD_FS_LIBS} Boost::filesystem Boost::program_options @@ -182,17 +181,14 @@ if(CLP_BUILD_EXECUTABLES) date::date fmt::fmt LibArchive::LibArchive - # BZip2 must be linked after static LibArchive for musllinux - BZip2::BZip2 + ${LibArchive_LIBRARY_DEPENDENCIES} log_surgeon::log_surgeon - LZ4::LZ4 MariaDBClient::MariaDBClient nlohmann_json::nlohmann_json spdlog::spdlog yaml-cpp ystdlib::containers ystdlib::error_handling - ZLIB::ZLIB ZStd::ZStd ) # Put the built executable at the root of the build directory diff --git a/components/core/src/glt/glt/CMakeLists.txt b/components/core/src/glt/glt/CMakeLists.txt index 8551ca9191..32e5469397 100644 --- a/components/core/src/glt/glt/CMakeLists.txt +++ b/components/core/src/glt/glt/CMakeLists.txt @@ -180,7 +180,6 @@ if(CLP_BUILD_EXECUTABLES) ) target_link_libraries(glt PRIVATE - ${LIBLZMA_LIBRARIES} ${sqlite_LIBRARY_DEPENDENCIES} ${STD_FS_LIBS} Boost::filesystem Boost::iostreams Boost::program_options @@ -188,7 +187,7 @@ if(CLP_BUILD_EXECUTABLES) date::date fmt::fmt LibArchive::LibArchive - LZ4::LZ4 + ${LibArchive_LIBRARY_DEPENDENCIES} MariaDBClient::MariaDBClient nlohmann_json::nlohmann_json spdlog::spdlog From 5fdbc1227ae77fe8d52fa14a76aac49859c4ab01 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 24 Jul 2025 13:16:04 +0000 Subject: [PATCH 13/20] Make Pkgconf work in FindLibArchive --- components/core/CMakeLists.txt | 5 ----- .../core/cmake/Modules/FindLibArchive.cmake | 20 ++++--------------- components/core/src/clp/clp/CMakeLists.txt | 1 - components/core/src/glt/glt/CMakeLists.txt | 1 - 4 files changed, 4 insertions(+), 23 deletions(-) diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index a15ec0ea3b..0b1d8d6880 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -226,10 +226,6 @@ if(CLP_NEED_LIBARCHIVE) endif() find_package(LibArchive 3.8.1 REQUIRED) message(STATUS "Found LibArchive ${LibArchive_VERSION}") - set(LibArchive_STATIC_LIBS "lzma") - FindStaticLibraryDependencies(archive LibArchive "${LibArchive_STATIC_LIBS}") - set(LibArchive_DYNAMIC_LIBS "bz2;lz4;z") - FindDynamicLibraryDependencies(LibArchive "${LibArchive_DYNAMIC_LIBS}") endif() # Find and setup libcurl @@ -779,7 +775,6 @@ if(CLP_BUILD_TESTING) date::date fmt::fmt LibArchive::LibArchive - ${LibArchive_LIBRARY_DEPENDENCIES} log_surgeon::log_surgeon MariaDBClient::MariaDBClient nlohmann_json::nlohmann_json diff --git a/components/core/cmake/Modules/FindLibArchive.cmake b/components/core/cmake/Modules/FindLibArchive.cmake index c257e88341..7665f8d8ca 100644 --- a/components/core/cmake/Modules/FindLibArchive.cmake +++ b/components/core/cmake/Modules/FindLibArchive.cmake @@ -13,22 +13,13 @@ # - Variables that should be externally visible are prefixed with "LibArchive_" set(libarchive_LIBNAME "archive") +set(libarchive_PKGCONFIG_DIR "${LibArchive_ROOT}/lib/pkgconfig") include(cmake/Modules/FindLibraryDependencies.cmake) -# On macOS, libarchive installed through brew is not linked into prefix by default. -# So it cannot be found by pkg-config and we need to manually find it. -# For more details, see https://github.com/Homebrew/homebrew-core/issues/117642 -# Find and setup libarchive -if(APPLE) - execute_process(COMMAND brew --prefix libarchive OUTPUT_VARIABLE libarchive_MACOS_PREFIX) - string(STRIP "${libarchive_MACOS_PREFIX}" libarchive_MACOS_PREFIX) - set(ENV{libarchive_PREV_CMAKE_PATH} "$ENV{CMAKE_PREFIX_PATH}") # save it so we can revert it later - set(ENV{CMAKE_PREFIX_PATH} "${libarchive_MACOS_PREFIX};$ENV{CMAKE_PREFIX_PATH}") -endif() - # Run pkg-config find_package(PkgConfig) +set(ENV{PKG_CONFIG_PATH} "${libarchive_PKGCONFIG_DIR};$ENV{PKG_CONFIG_PATH}") pkg_check_modules(libarchive_PKGCONF QUIET "lib${libarchive_LIBNAME}") # Set include directory @@ -77,6 +68,8 @@ find_package_handle_standard_args(LibArchive VERSION_VAR LibArchive_VERSION ) +message(STATUS "libarchive_PKGCONF_STATIC_LIBRARIES = ${libarchive_PKGCONF_STATIC_LIBRARIES}") +message(STATUS "libarchive_LIBRARY_DEPENDENCIES = ${libarchive_LIBRARY_DEPENDENCIES}") if(NOT TARGET LibArchive::LibArchive) # Add library to build if (LibArchive_FOUND) @@ -114,8 +107,3 @@ if(NOT TARGET LibArchive::LibArchive) endif() endif() endif() - -if(APPLE) - # remove LibArchive-specific path - set(ENV{CMAKE_PREFIX_PATH} "$ENV{libarchive_PREV_CMAKE_PATH}") -endif() \ No newline at end of file diff --git a/components/core/src/clp/clp/CMakeLists.txt b/components/core/src/clp/clp/CMakeLists.txt index c795f21af8..bd2fefe55c 100644 --- a/components/core/src/clp/clp/CMakeLists.txt +++ b/components/core/src/clp/clp/CMakeLists.txt @@ -181,7 +181,6 @@ if(CLP_BUILD_EXECUTABLES) date::date fmt::fmt LibArchive::LibArchive - ${LibArchive_LIBRARY_DEPENDENCIES} log_surgeon::log_surgeon MariaDBClient::MariaDBClient nlohmann_json::nlohmann_json diff --git a/components/core/src/glt/glt/CMakeLists.txt b/components/core/src/glt/glt/CMakeLists.txt index 32e5469397..c95f8b84aa 100644 --- a/components/core/src/glt/glt/CMakeLists.txt +++ b/components/core/src/glt/glt/CMakeLists.txt @@ -187,7 +187,6 @@ if(CLP_BUILD_EXECUTABLES) date::date fmt::fmt LibArchive::LibArchive - ${LibArchive_LIBRARY_DEPENDENCIES} MariaDBClient::MariaDBClient nlohmann_json::nlohmann_json spdlog::spdlog From cb47228c01b1fa119fdfa543064693605e9854cc Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 24 Jul 2025 14:04:23 +0000 Subject: [PATCH 14/20] restore cmakelists order to reduce clutter --- components/core/CMakeLists.txt | 21 +++++++++---------- .../core/cmake/Modules/FindLibArchive.cmake | 17 ++++++++++----- components/core/src/clp/clp/CMakeLists.txt | 10 ++++----- components/core/src/glt/glt/CMakeLists.txt | 8 +++---- 4 files changed, 31 insertions(+), 25 deletions(-) diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index 0b1d8d6880..0ca6cc82dc 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -12,8 +12,6 @@ validate_compiler_versions() # Include options for CLP build include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/Options/options.cmake") -include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/FindLibraryDependencies.cmake") - validate_and_setup_all_clp_dependency_flags() convert_clp_dependency_properties_to_variables() @@ -348,6 +346,7 @@ endif() # sqlite dependencies if(CLP_NEED_SQLITE) set(sqlite_DYNAMIC_LIBS "dl;m;pthread") + include(cmake/Modules/FindLibraryDependencies.cmake) FindDynamicLibraryDependencies(sqlite "${sqlite_DYNAMIC_LIBS}") endif() @@ -758,29 +757,29 @@ if(CLP_BUILD_TESTING) ${CLP_SQLITE3_INCLUDE_DIRECTORY} ) target_link_libraries(unitTest - PRIVATE - ${CURL_LIBRARIES} - ${MONGOCXX_TARGET} - ${sqlite_LIBRARY_DEPENDENCIES} - ${STD_FS_LIBS} + PRIVATE absl::flat_hash_map Boost::filesystem Boost::iostreams Boost::program_options Boost::regex Boost::url Catch2::Catch2 - clp::regex_utils - clp::string_utils + ${CURL_LIBRARIES} clp_s::search::ast clp_s::search::kql clp_s::search::sql clp_s::timestamp_pattern date::date fmt::fmt - LibArchive::LibArchive log_surgeon::log_surgeon + LibArchive::LibArchive MariaDBClient::MariaDBClient + ${MONGOCXX_TARGET} nlohmann_json::nlohmann_json - OpenSSL::Crypto simdjson::simdjson spdlog::spdlog + OpenSSL::Crypto + ${sqlite_LIBRARY_DEPENDENCIES} + ${STD_FS_LIBS} + clp::regex_utils + clp::string_utils yaml-cpp ystdlib::containers ystdlib::error_handling diff --git a/components/core/cmake/Modules/FindLibArchive.cmake b/components/core/cmake/Modules/FindLibArchive.cmake index 7665f8d8ca..35bd63d01c 100644 --- a/components/core/cmake/Modules/FindLibArchive.cmake +++ b/components/core/cmake/Modules/FindLibArchive.cmake @@ -12,11 +12,11 @@ # - Variables only for use within the script are prefixed with "libarchive_" # - Variables that should be externally visible are prefixed with "LibArchive_" +include(cmake/Modules/FindLibraryDependencies.cmake) + set(libarchive_LIBNAME "archive") set(libarchive_PKGCONFIG_DIR "${LibArchive_ROOT}/lib/pkgconfig") -include(cmake/Modules/FindLibraryDependencies.cmake) - # Run pkg-config find_package(PkgConfig) set(ENV{PKG_CONFIG_PATH} "${libarchive_PKGCONFIG_DIR};$ENV{PKG_CONFIG_PATH}") @@ -43,7 +43,7 @@ find_library(LibArchive_LIBRARY HINTS ${libarchive_PKGCONF_LIBDIR} PATH_SUFFIXES lib ) -if (LibArchive_LIBRARY) +if(LibArchive_LIBRARY) # NOTE: This must be set for find_package_handle_standard_args to work set(LibArchive_FOUND ON) endif() @@ -57,8 +57,16 @@ if(LibArchive_USE_STATIC_LIBS) unset(libarchive_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES) endif() +if(APPLE) + set(libarchive_DYNAMIC_LIBS "${libarchive_PKGCONF_STATIC_LIBRARIES}") +endif() + FindDynamicLibraryDependencies(libarchive "${libarchive_DYNAMIC_LIBS}") +message(STATUS "libarchive_PKGCONF_STATIC_LIBRARIES = ${libarchive_PKGCONF_STATIC_LIBRARIES}") +message(STATUS "libarchive_DYNAMIC_LIBS = ${libarchive_DYNAMIC_LIBS}") +message(STATUS "libarchive_LIBRARY_DEPENDENCIES = ${libarchive_LIBRARY_DEPENDENCIES}") + # Set version set(LibArchive_VERSION ${libarchive_PKGCONF_VERSION}) @@ -68,8 +76,6 @@ find_package_handle_standard_args(LibArchive VERSION_VAR LibArchive_VERSION ) -message(STATUS "libarchive_PKGCONF_STATIC_LIBRARIES = ${libarchive_PKGCONF_STATIC_LIBRARIES}") -message(STATUS "libarchive_LIBRARY_DEPENDENCIES = ${libarchive_LIBRARY_DEPENDENCIES}") if(NOT TARGET LibArchive::LibArchive) # Add library to build if (LibArchive_FOUND) @@ -107,3 +113,4 @@ if(NOT TARGET LibArchive::LibArchive) endif() endif() endif() + diff --git a/components/core/src/clp/clp/CMakeLists.txt b/components/core/src/clp/clp/CMakeLists.txt index bd2fefe55c..c9782c509f 100644 --- a/components/core/src/clp/clp/CMakeLists.txt +++ b/components/core/src/clp/clp/CMakeLists.txt @@ -174,17 +174,17 @@ if(CLP_BUILD_EXECUTABLES) ) target_link_libraries(clp PRIVATE - ${sqlite_LIBRARY_DEPENDENCIES} - ${STD_FS_LIBS} Boost::filesystem Boost::program_options - clp::string_utils date::date fmt::fmt - LibArchive::LibArchive log_surgeon::log_surgeon + spdlog::spdlog + ${sqlite_LIBRARY_DEPENDENCIES} + LibArchive::LibArchive MariaDBClient::MariaDBClient nlohmann_json::nlohmann_json - spdlog::spdlog + ${STD_FS_LIBS} + clp::string_utils yaml-cpp ystdlib::containers ystdlib::error_handling diff --git a/components/core/src/glt/glt/CMakeLists.txt b/components/core/src/glt/glt/CMakeLists.txt index c95f8b84aa..8576f391f5 100644 --- a/components/core/src/glt/glt/CMakeLists.txt +++ b/components/core/src/glt/glt/CMakeLists.txt @@ -180,16 +180,16 @@ if(CLP_BUILD_EXECUTABLES) ) target_link_libraries(glt PRIVATE - ${sqlite_LIBRARY_DEPENDENCIES} - ${STD_FS_LIBS} Boost::filesystem Boost::iostreams Boost::program_options - clp::string_utils date::date fmt::fmt + spdlog::spdlog + ${sqlite_LIBRARY_DEPENDENCIES} LibArchive::LibArchive MariaDBClient::MariaDBClient nlohmann_json::nlohmann_json - spdlog::spdlog + ${STD_FS_LIBS} + clp::string_utils yaml-cpp ystdlib::error_handling ZStd::ZStd From 8f5d04dcd387fe15ac1ab63d2180eca0f3a930a0 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Thu, 24 Jul 2025 17:09:02 +0000 Subject: [PATCH 15/20] Remove centos restriction on avoiding static libs --- components/core/CMakeLists.txt | 10 ---------- components/core/cmake/Modules/FindLibArchive.cmake | 5 ++--- .../core/cmake/Modules/FindLibraryDependencies.cmake | 6 ++++++ 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index 0ca6cc82dc..10148c32d9 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -71,16 +71,6 @@ set(CLP_USE_STATIC_LIBS ON CACHE BOOL "Whether to link against static libraries" if (CLP_USE_STATIC_LIBS) if (APPLE) set(CLP_STATIC_LIBS_UNSUPPORTED_PLATFORM "macOS") - elseif (EXISTS "/etc/centos-release") - # NOTE: - # 1. We don't support static linking on any CentOS-based distro except manylinux_2_28 (which - # shows up as "AlmaLinux"). - # 2. A release called "AlmaLinux" doesn't guarantee we're running on a manylinux distro, but - # we can improve this check when someone reports an issue. - file(READ "/etc/centos-release" CENTOS_RELEASE_CONTENT) - if(NOT "${CENTOS_RELEASE_CONTENT}" MATCHES "AlmaLinux") - set(CLP_STATIC_LIBS_UNSUPPORTED_PLATFORM "CentOS") - endif() endif() if (DEFINED CLP_STATIC_LIBS_UNSUPPORTED_PLATFORM) diff --git a/components/core/cmake/Modules/FindLibArchive.cmake b/components/core/cmake/Modules/FindLibArchive.cmake index 35bd63d01c..577e1954e0 100644 --- a/components/core/cmake/Modules/FindLibArchive.cmake +++ b/components/core/cmake/Modules/FindLibArchive.cmake @@ -29,6 +29,7 @@ find_path(LibArchive_INCLUDE_DIR archive.h ) # Handle static libraries +set(LibArchive_USE_STATIC_LIBS ON) if(LibArchive_USE_STATIC_LIBS) # Save current value of CMAKE_FIND_LIBRARY_SUFFIXES set(libarchive_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) @@ -55,9 +56,7 @@ if(LibArchive_USE_STATIC_LIBS) # Restore original value of CMAKE_FIND_LIBRARY_SUFFIXES set(CMAKE_FIND_LIBRARY_SUFFIXES ${libarchive_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES}) unset(libarchive_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES) -endif() - -if(APPLE) +else() set(libarchive_DYNAMIC_LIBS "${libarchive_PKGCONF_STATIC_LIBRARIES}") endif() diff --git a/components/core/cmake/Modules/FindLibraryDependencies.cmake b/components/core/cmake/Modules/FindLibraryDependencies.cmake index 769fcf9b06..bbe67931a6 100644 --- a/components/core/cmake/Modules/FindLibraryDependencies.cmake +++ b/components/core/cmake/Modules/FindLibraryDependencies.cmake @@ -10,6 +10,10 @@ macro(FindStaticLibraryDependencies fld_LIBNAME fld_PREFIX fld_STATIC_LIBS) # NOTE: libc, libm, libpthread, and librt should be dynamically linked set(fld_DYNAMIC_LIBS "c;m;pthread;rt") + if("${fld_PREFIX}" STREQUAL "libarchive") + set(fld_ALLOW_SHARED_LIBS_FALLBACK ON) + endif() + # Get absolute path of dependent libraries foreach(fld_DEP_LIB ${fld_STATIC_LIBS}) # Skip dynamic libs @@ -31,6 +35,8 @@ macro(FindStaticLibraryDependencies fld_LIBNAME fld_PREFIX fld_STATIC_LIBS) if(${fld_PREFIX}_${fld_DEP_LIB}_LIBRARY) list(APPEND ${fld_PREFIX}_LIBRARY_DEPENDENCIES "${${fld_PREFIX}_${fld_DEP_LIB}_LIBRARY}") + elseif(fld_ALLOW_SHARED_LIBS_FALLBACK) + list(APPEND ${fld_PREFIX}_DYNAMIC_LIBS "${fld_DEP_LIB}") else() message(SEND_ERROR "Static ${fld_DEP_LIB} library not found") endif() From 5046d0d5d6ed6a821a2555c65a4c886a018b76b2 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 13 Aug 2025 03:58:50 -0400 Subject: [PATCH 16/20] Improve LibArchive find module --- .../core/cmake/Modules/FindLibArchive.cmake | 69 +++++++++++-------- 1 file changed, 39 insertions(+), 30 deletions(-) diff --git a/components/core/cmake/Modules/FindLibArchive.cmake b/components/core/cmake/Modules/FindLibArchive.cmake index 577e1954e0..fcf57b8740 100644 --- a/components/core/cmake/Modules/FindLibArchive.cmake +++ b/components/core/cmake/Modules/FindLibArchive.cmake @@ -1,40 +1,51 @@ # Try to find LibArchive -# NOTE: The FindLibArchive.cmake included with CMake has no support for static libraries, so we use our own. +# NOTE: The FindLibArchive.cmake included with CMake has no support for static libraries, so we use +# our own. # # Set LibArchive_USE_STATIC_LIBS=ON to look for static libraries. # -# Once done this will define: -# LibArchive_FOUND - Whether LibArchive was found on the system -# LibArchive_INCLUDE_DIR - The LibArchive include directories -# LibArchive_VERSION - The version of LibArchive installed on the system +# Once done, this will define: +# LibArchive_FOUND - Whether the library was found on the system +# LibArchive_INCLUDE_DIR - The library include directories +# LibArchive_LIBRARY - The path to the library file +# +# And will define the following if the package configuration file provides relevant information: +# LibArchive_VERSION - The version of library installed on the system +# LibArchive_LIBRARY_DEPENDENCIES - Any additional modules required to link with the library # # Conventions: # - Variables only for use within the script are prefixed with "libarchive_" # - Variables that should be externally visible are prefixed with "LibArchive_" +include(FindPackageHandleStandardArgs) + include(cmake/Modules/FindLibraryDependencies.cmake) +set(libarchive_HEADER "archive.h") set(libarchive_LIBNAME "archive") -set(libarchive_PKGCONFIG_DIR "${LibArchive_ROOT}/lib/pkgconfig") +set(libarchive_LOCAL_PREFIX "libarchive") +set(libarchive_PKGCONFIG_NAME "libarchive") + +if(DEFINED LibArchive_ROOT) + set(libarchive_PKGCONFIG_DIR "${LibArchive_ROOT}/lib/pkgconfig") + set(ENV{libarchive_ORIG_PKG_CONFIG_PATH} "$ENV{PKG_CONFIG_PATH}") + set(ENV{PKG_CONFIG_PATH} "${libarchive_PKGCONFIG_DIR};$ENV{PKG_CONFIG_PATH}") +endif() # Run pkg-config find_package(PkgConfig) -set(ENV{PKG_CONFIG_PATH} "${libarchive_PKGCONFIG_DIR};$ENV{PKG_CONFIG_PATH}") -pkg_check_modules(libarchive_PKGCONF QUIET "lib${libarchive_LIBNAME}") +pkg_check_modules(libarchive_PKGCONF QUIET "${libarchive_PKGCONFIG_NAME}") # Set include directory -find_path(LibArchive_INCLUDE_DIR archive.h +find_path(LibArchive_INCLUDE_DIR ${libarchive_HEADER} HINTS ${libarchive_PKGCONF_INCLUDEDIR} PATH_SUFFIXES include ) # Handle static libraries -set(LibArchive_USE_STATIC_LIBS ON) if(LibArchive_USE_STATIC_LIBS) - # Save current value of CMAKE_FIND_LIBRARY_SUFFIXES - set(libarchive_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) - # Temporarily change CMAKE_FIND_LIBRARY_SUFFIXES to static library suffix + set(libarchive_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) set(CMAKE_FIND_LIBRARY_SUFFIXES .a) endif() @@ -44,20 +55,15 @@ find_library(LibArchive_LIBRARY HINTS ${libarchive_PKGCONF_LIBDIR} PATH_SUFFIXES lib ) -if(LibArchive_LIBRARY) - # NOTE: This must be set for find_package_handle_standard_args to work - set(LibArchive_FOUND ON) -endif() +# Find dependencies if(LibArchive_USE_STATIC_LIBS) - FindStaticLibraryDependencies(${libarchive_LIBNAME} libarchive + FindStaticLibraryDependencies(${libarchive_LIBNAME} ${libarchive_LOCAL_PREFIX} "${libarchive_PKGCONF_STATIC_LIBRARIES}") # Restore original value of CMAKE_FIND_LIBRARY_SUFFIXES set(CMAKE_FIND_LIBRARY_SUFFIXES ${libarchive_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES}) unset(libarchive_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES) -else() - set(libarchive_DYNAMIC_LIBS "${libarchive_PKGCONF_STATIC_LIBRARIES}") endif() FindDynamicLibraryDependencies(libarchive "${libarchive_DYNAMIC_LIBS}") @@ -69,22 +75,20 @@ message(STATUS "libarchive_LIBRARY_DEPENDENCIES = ${libarchive_LIBRARY_DEPENDENC # Set version set(LibArchive_VERSION ${libarchive_PKGCONF_VERSION}) -include(FindPackageHandleStandardArgs) +# Set up find_package() call find_package_handle_standard_args(LibArchive - REQUIRED_VARS LibArchive_INCLUDE_DIR + REQUIRED_VARS LibArchive_LIBRARY LibArchive_INCLUDE_DIR VERSION_VAR LibArchive_VERSION ) if(NOT TARGET LibArchive::LibArchive) # Add library to build - if (LibArchive_FOUND) - if (LibArchive_USE_STATIC_LIBS) - add_library(LibArchive::LibArchive STATIC IMPORTED) - else() - # NOTE: We use UNKNOWN so that if the user doesn't have the SHARED - # libraries installed, we can still use the STATIC libraries - add_library(LibArchive::LibArchive UNKNOWN IMPORTED) - endif() + if(LibArchive_USE_STATIC_LIBS) + add_library(LibArchive::LibArchive STATIC IMPORTED) + else() + # NOTE: We use UNKNOWN so that if the user doesn't have the SHARED + # libraries installed, we can still use the STATIC libraries + add_library(LibArchive::LibArchive UNKNOWN IMPORTED) endif() # Set include directories for library @@ -113,3 +117,8 @@ if(NOT TARGET LibArchive::LibArchive) endif() endif() +# Restore original value of PKG_CONFIG_PATH +if(DEFINED ENV{libarchive_ORIG_PKG_CONFIG_PATH}) + set(ENV{PKG_CONFIG_PATH} "$ENV{libarchive_ORIG_PKG_CONFIG_PATH}") + unset(ENV{libarchive_ORIG_PKG_CONFIG_PATH}) +endif() From e65b43301d03ed8db9a1d38e2edf02c888874dda Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 13 Aug 2025 04:04:02 -0400 Subject: [PATCH 17/20] Revert changes to find library deps --- components/core/cmake/Modules/FindLibraryDependencies.cmake | 6 ------ 1 file changed, 6 deletions(-) diff --git a/components/core/cmake/Modules/FindLibraryDependencies.cmake b/components/core/cmake/Modules/FindLibraryDependencies.cmake index bbe67931a6..769fcf9b06 100644 --- a/components/core/cmake/Modules/FindLibraryDependencies.cmake +++ b/components/core/cmake/Modules/FindLibraryDependencies.cmake @@ -10,10 +10,6 @@ macro(FindStaticLibraryDependencies fld_LIBNAME fld_PREFIX fld_STATIC_LIBS) # NOTE: libc, libm, libpthread, and librt should be dynamically linked set(fld_DYNAMIC_LIBS "c;m;pthread;rt") - if("${fld_PREFIX}" STREQUAL "libarchive") - set(fld_ALLOW_SHARED_LIBS_FALLBACK ON) - endif() - # Get absolute path of dependent libraries foreach(fld_DEP_LIB ${fld_STATIC_LIBS}) # Skip dynamic libs @@ -35,8 +31,6 @@ macro(FindStaticLibraryDependencies fld_LIBNAME fld_PREFIX fld_STATIC_LIBS) if(${fld_PREFIX}_${fld_DEP_LIB}_LIBRARY) list(APPEND ${fld_PREFIX}_LIBRARY_DEPENDENCIES "${${fld_PREFIX}_${fld_DEP_LIB}_LIBRARY}") - elseif(fld_ALLOW_SHARED_LIBS_FALLBACK) - list(APPEND ${fld_PREFIX}_DYNAMIC_LIBS "${fld_DEP_LIB}") else() message(SEND_ERROR "Static ${fld_DEP_LIB} library not found") endif() From d54a1ea730961af0f61368b0833f9135d199a630 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 13 Aug 2025 04:14:02 -0400 Subject: [PATCH 18/20] Remove unnecessary find_package calls --- components/core/CMakeLists.txt | 23 +++------------------ components/core/cmake/Options/options.cmake | 12 ++--------- 2 files changed, 5 insertions(+), 30 deletions(-) diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index 10148c32d9..1e8699acc1 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -145,11 +145,6 @@ if(CLP_NEED_BOOST) endif() endif() -if(CLP_NEED_BZIP2) - find_package(BZip2 REQUIRED) - message(STATUS "Found BZip2") -endif() - if(CLP_NEED_CATCH2) find_package(Catch2 REQUIRED) if (Catch2_FOUND) @@ -178,14 +173,6 @@ if(CLP_NEED_LOG_SURGEON) endif() endif() -if(CLP_NEED_LZ4) - if(CLP_USE_STATIC_LIBS) - set(LZ4_USE_STATIC_LIBS ON) - endif() - find_package(LZ4 REQUIRED) - message(STATUS "Found LZ4 ${LZ4_VERSION}") -endif() - if(CLP_NEED_NLOHMANN_JSON) find_package(nlohmann_json REQUIRED) if(nlohmann_json_FOUND) @@ -212,6 +199,7 @@ if(CLP_NEED_LIBARCHIVE) if(CLP_USE_STATIC_LIBS) set(LibArchive_USE_STATIC_LIBS ON) endif() + # TODO: Add why we need 3.8.1 find_package(LibArchive 3.8.1 REQUIRED) message(STATUS "Found LibArchive ${LibArchive_VERSION}") endif() @@ -287,12 +275,6 @@ if(CLP_NEED_YSTDLIB) add_subdirectory("${CLP_YSTDLIB_SOURCE_DIRECTORY}" "${CMAKE_BINARY_DIR}/ystdlib" EXCLUDE_FROM_ALL) endif() -# Find and setup ZLIB -if(CLP_NEED_ZLIB) - find_package(ZLIB REQUIRED) - message(STATUS "Found ZLIB ${ZLIB_VERSION_STRING}") -endif() - # Find and setup ZStd Library if(CLP_NEED_ZSTD) if(CLP_USE_STATIC_LIBS) @@ -747,7 +729,7 @@ if(CLP_BUILD_TESTING) ${CLP_SQLITE3_INCLUDE_DIRECTORY} ) target_link_libraries(unitTest - PRIVATE + PRIVATE absl::flat_hash_map Boost::filesystem Boost::iostreams Boost::program_options Boost::regex Boost::url Catch2::Catch2 @@ -773,6 +755,7 @@ if(CLP_BUILD_TESTING) yaml-cpp ystdlib::containers ystdlib::error_handling + ${LIBLZMA_LIBRARIES} ZStd::ZStd ) target_compile_features(unitTest diff --git a/components/core/cmake/Options/options.cmake b/components/core/cmake/Options/options.cmake index 6d89b1098c..a558b36f89 100644 --- a/components/core/cmake/Options/options.cmake +++ b/components/core/cmake/Options/options.cmake @@ -133,14 +133,11 @@ function(set_clp_binaries_dependencies) set_clp_need_flags( CLP_NEED_ABSL CLP_NEED_BOOST - CLP_NEED_BZIP2 CLP_NEED_CURL CLP_NEED_DATE CLP_NEED_FMT CLP_NEED_LIBARCHIVE CLP_NEED_LOG_SURGEON - CLP_NEED_LZ4 - CLP_NEED_LZMA CLP_NEED_MARIADB CLP_NEED_MONGOCXX CLP_NEED_MSGPACKCXX @@ -151,7 +148,6 @@ function(set_clp_binaries_dependencies) CLP_NEED_SQLITE CLP_NEED_YAMLCPP CLP_NEED_YSTDLIB - CLP_NEED_ZLIB CLP_NEED_ZSTD ) endfunction() @@ -171,13 +167,11 @@ function(set_clp_tests_dependencies) set_clp_need_flags( CLP_NEED_ABSL CLP_NEED_BOOST - CLP_NEED_BZIP2 CLP_NEED_CATCH2 CLP_NEED_DATE CLP_NEED_FMT CLP_NEED_LIBARCHIVE CLP_NEED_LOG_SURGEON - CLP_NEED_LZ4 CLP_NEED_LZMA CLP_NEED_MARIADB CLP_NEED_MONGOCXX @@ -188,7 +182,6 @@ function(set_clp_tests_dependencies) CLP_NEED_SQLITE CLP_NEED_YAMLCPP CLP_NEED_YSTDLIB - CLP_NEED_ZLIB CLP_NEED_ZSTD ) endfunction() @@ -256,6 +249,7 @@ function(set_clp_s_clp_dependencies_dependencies) CLP_NEED_BOOST CLP_NEED_CURL CLP_NEED_FMT + CLP_NEED_LOG_SURGEON CLP_NEED_MSGPACKCXX CLP_NEED_NLOHMANN_JSON CLP_NEED_OPENSSL @@ -318,6 +312,7 @@ endfunction() function(set_clp_s_search_dependencies) set_clp_need_flags( CLP_NEED_ABSL + CLP_NEED_LOG_SURGEON CLP_NEED_SIMDJSON CLP_NEED_SPDLOG ) @@ -457,14 +452,12 @@ function (convert_clp_dependency_properties_to_variables) CLP_NEED_ABSL CLP_NEED_ANTLR CLP_NEED_BOOST - CLP_NEED_BZIP2 CLP_NEED_CATCH2 CLP_NEED_CURL CLP_NEED_DATE CLP_NEED_FMT CLP_NEED_LOG_SURGEON CLP_NEED_LIBARCHIVE - CLP_NEED_LZ4 CLP_NEED_LZMA CLP_NEED_MARIADB CLP_NEED_MONGOCXX @@ -476,7 +469,6 @@ function (convert_clp_dependency_properties_to_variables) CLP_NEED_SQLITE CLP_NEED_YAMLCPP CLP_NEED_YSTDLIB - CLP_NEED_ZLIB CLP_NEED_ZSTD ) From 2bb246b7b2a509f0b3366bc6f73861fba5fc0bba Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 13 Aug 2025 04:29:13 -0400 Subject: [PATCH 19/20] Revert many/musllinux dep scripts and update ubuntujammy/centos/macos to install everything --- .../lib_install/centos-stream-9/install-prebuilt-packages.sh | 4 +++- .../core/tools/scripts/lib_install/macos/install-all.sh | 3 +++ .../manylinux_2_28/install-packages-from-source.sh | 1 + .../lib_install/manylinux_2_28/install-prebuilt-packages.sh | 1 - .../lib_install/musllinux_1_2/install-packages-from-source.sh | 1 + .../lib_install/ubuntu-jammy/install-packages-from-source.sh | 1 + .../lib_install/ubuntu-jammy/install-prebuilt-packages.sh | 3 ++- 7 files changed, 11 insertions(+), 3 deletions(-) diff --git a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh index ed98f49933..f30a67c7a9 100755 --- a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh @@ -14,6 +14,7 @@ dnf install -y \ git \ java-11-openjdk \ jq \ + libarchive-devel \ libcurl-devel \ libzstd-devel \ lz4-devel \ @@ -22,7 +23,8 @@ dnf install -y \ openssl-devel \ python3-devel \ python3-pip \ - unzip + unzip \ + zlib-devel # Determine architecture for `task` release to install rpm_arch=$(rpm --eval "%{_arch}") diff --git a/components/core/tools/scripts/lib_install/macos/install-all.sh b/components/core/tools/scripts/lib_install/macos/install-all.sh index b380e67027..042dbf0a82 100755 --- a/components/core/tools/scripts/lib_install/macos/install-all.sh +++ b/components/core/tools/scripts/lib_install/macos/install-all.sh @@ -37,14 +37,17 @@ rm -rf "$formula_dir" brew install \ boost \ + bzip2 \ coreutils \ gcc \ java11 \ + libarchive \ llvm@16 \ lz4 \ mariadb-connector-c \ msgpack-cxx \ xz \ + zlib \ zstd # Install pkg-config if it isn't already installed diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh index 349cb9ce1c..d31343c8ee 100755 --- a/components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-packages-from-source.sh @@ -17,5 +17,6 @@ lib_install_scripts_dir="${script_dir}/.." "${lib_install_scripts_dir}/liblzma.sh" 5.8.1 "${lib_install_scripts_dir}/lz4.sh" 1.10.0 "${lib_install_scripts_dir}/zstandard.sh" 1.5.7 +"${lib_install_scripts_dir}/libarchive.sh" 3.8.0 "${lib_install_scripts_dir}/msgpack.sh" 7.0.0 diff --git a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh index b4a5b88171..57ea2caab0 100755 --- a/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/manylinux_2_28/install-prebuilt-packages.sh @@ -4,7 +4,6 @@ set -eu set -o pipefail dnf install -y \ - bzip2-devel \ gcc-c++ \ java-11-openjdk \ jq \ diff --git a/components/core/tools/scripts/lib_install/musllinux_1_2/install-packages-from-source.sh b/components/core/tools/scripts/lib_install/musllinux_1_2/install-packages-from-source.sh index fda3352b42..e1f65b9bcd 100755 --- a/components/core/tools/scripts/lib_install/musllinux_1_2/install-packages-from-source.sh +++ b/components/core/tools/scripts/lib_install/musllinux_1_2/install-packages-from-source.sh @@ -17,5 +17,6 @@ lib_install_scripts_dir="${script_dir}/.." "${lib_install_scripts_dir}/liblzma.sh" 5.8.1 "${lib_install_scripts_dir}/lz4.sh" 1.10.0 "${lib_install_scripts_dir}/zstandard.sh" 1.5.7 +"${lib_install_scripts_dir}/libarchive.sh" 3.8.0 "${lib_install_scripts_dir}/msgpack.sh" 7.0.0 diff --git a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-packages-from-source.sh b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-packages-from-source.sh index 813c4277d9..af1d2774a9 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-packages-from-source.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-packages-from-source.sh @@ -12,6 +12,7 @@ lib_install_scripts_dir=$script_dir/.. # NOTE: boost must be installed first since the remaining packages depend on it "$lib_install_scripts_dir"/install-boost.sh 1.87.0 +"$lib_install_scripts_dir"/libarchive.sh 3.5.1 "$lib_install_scripts_dir"/liblzma.sh 5.8.1 "$lib_install_scripts_dir"/lz4.sh 1.8.2 "$lib_install_scripts_dir"/msgpack.sh 7.0.0 diff --git a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh index 4fb7baecc9..eb20aef19f 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh @@ -28,7 +28,8 @@ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ python3-pip \ python3-venv \ software-properties-common \ - unzip + unzip \ + zlib1g-dev # Install `task` # NOTE: We lock `task` to a version < 3.43 to avoid https://github.com/y-scope/clp/issues/872 From a21c9cefc9534199039e98a88e35245b785292a1 Mon Sep 17 00:00:00 2001 From: Bingran Hu Date: Wed, 13 Aug 2025 05:45:13 -0400 Subject: [PATCH 20/20] revert libarchive script --- .../tools/scripts/lib_install/libarchive.sh | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100755 components/core/tools/scripts/lib_install/libarchive.sh diff --git a/components/core/tools/scripts/lib_install/libarchive.sh b/components/core/tools/scripts/lib_install/libarchive.sh new file mode 100755 index 0000000000..dd8c2b733e --- /dev/null +++ b/components/core/tools/scripts/lib_install/libarchive.sh @@ -0,0 +1,95 @@ +#!/bin/bash + +# Dependencies: +# - cmake +# - curl +# NOTE: Dependencies should be installed outside the script to allow the script to be largely distro-agnostic + +# Exit on any error +set -e + +cUsage="Usage: ${BASH_SOURCE[0]} [ <.deb output directory>]" +if [ "$#" -lt 1 ] ; then + echo $cUsage + exit +fi +version=$1 + +package_name=libarchive +temp_dir=/tmp/${package_name}-installation +deb_output_dir=${temp_dir} +if [[ "$#" -gt 1 ]] ; then + deb_output_dir="$(readlink -f "$2")" + if [ ! -d ${deb_output_dir} ] ; then + echo "${deb_output_dir} does not exist or is not a directory" + exit + fi +fi + +# Check if already installed +set +e +dpkg -l ${package_name} | grep ${version} +installed=$? +set -e +if [ $installed -eq 0 ] ; then + # Nothing to do + exit +fi + +echo "Checking for elevated privileges..." +install_cmd_args=() +if [ ${EUID:-$(id -u)} -ne 0 ] ; then + sudo echo "Script can elevate privileges." + install_cmd_args+=("sudo") +fi + +# Get number of cpu cores +num_cpus=$(grep -c ^processor /proc/cpuinfo) + +# Download +mkdir -p $temp_dir +cd $temp_dir +extracted_dir=${temp_dir}/${package_name}-${version} +if [ ! -e ${extracted_dir} ] ; then + tar_filename=${package_name}-${version}.tar.gz + if [ ! -e ${tar_filename} ] ; then + curl -fsSL https://www.libarchive.org/downloads/${tar_filename} -o ${tar_filename} + fi + + tar -xf ${tar_filename} +fi + +# Build +cd ${extracted_dir} +mkdir -p cmake-build-release +cd cmake-build-release +# NOTE: Disable Expat and OpenSSL so the static libarchive doesn't look for them at link time. +cmake -DENABLE_EXPAT=OFF -DENABLE_OPENSSL=OFF ../ +make -j${num_cpus} + +# Check if checkinstall is installed +set +e +command -v checkinstall +checkinstall_installed=$? +set -e + +# Install +if [ $checkinstall_installed -eq 0 ] ; then + install_cmd_args+=( + checkinstall + --default + --fstrans=no + --nodoc + --pkgname "${package_name}" + --pkgversion "${version}" + --provides "${package_name}" + --pakdir "${deb_output_dir}" + ) +fi +install_cmd_args+=( + make install +) +"${install_cmd_args[@]}" + +# Clean up +rm -rf $temp_dir