diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index 57715c7f5a..ecbb0a6fdc 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -297,16 +297,14 @@ if(CLP_NEED_YSTDLIB) add_subdirectory("${CLP_YSTDLIB_SOURCE_DIRECTORY}" "${CMAKE_BINARY_DIR}/ystdlib" EXCLUDE_FROM_ALL) endif() -# Find and setup ZStd Library if(CLP_NEED_ZSTD) + # v1.4.8 is the lowest version available in the package managers of the OSes we support. + find_package(zstd 1.4.8 REQUIRED) + message(STATUS "Found zstd ${zstd_VERSION}") if(CLP_USE_STATIC_LIBS) - set(ZStd_USE_STATIC_LIBS ON) - endif() - find_package(ZStd 1.4.4 REQUIRED) - if(ZStd_FOUND) - message(STATUS "Found ZStd ${ZStd_VERSION}") + set(zstd_TARGET zstd::libzstd_static) else() - message(FATAL_ERROR "Could not find ${CLP_LIBS_STRING} libraries for ZStd") + set(zstd_TARGET zstd::libzstd_shared) endif() endif() @@ -772,7 +770,7 @@ if(CLP_BUILD_TESTING) ystdlib::containers ystdlib::error_handling ${LIBLZMA_LIBRARIES} - ZStd::ZStd + ${zstd_TARGET} ) target_compile_features(unitTest PRIVATE cxx_std_20 diff --git a/components/core/cmake/Modules/FindLZ4.cmake b/components/core/cmake/Modules/FindLZ4.cmake deleted file mode 100644 index 66ad14ff0d..0000000000 --- a/components/core/cmake/Modules/FindLZ4.cmake +++ /dev/null @@ -1,103 +0,0 @@ -# Try to find LZ4 -# -# Set LZ4_USE_STATIC_LIBS=ON to look for static libraries. -# -# Once done this will define: -# LZ4_FOUND - Whether LZ4 was found on the system -# LZ4_INCLUDE_DIR - The LZ4 include directories -# LZ4_VERSION - The version of LZ4 installed on the system -# -# Conventions: -# - Variables only for use within the script are prefixed with "lz4_" -# - Variables that should be externally visible are prefixed with "LZ4_" - -set(lz4_LIBNAME "lz4") - -include(cmake/Modules/FindLibraryDependencies.cmake) - -# Run pkg-config -find_package(PkgConfig) -pkg_check_modules(lz4_PKGCONF QUIET "lib${lz4_LIBNAME}") - -# Set include directory -find_path(LZ4_INCLUDE_DIR lz4.h - HINTS ${lz4_PKGCONF_INCLUDEDIR} - PATH_SUFFIXES include - ) - -# Handle static libraries -if(LZ4_USE_STATIC_LIBS) - # Save current value of CMAKE_FIND_LIBRARY_SUFFIXES - set(lz4_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) - - # Temporarily change CMAKE_FIND_LIBRARY_SUFFIXES to static library suffix - set(CMAKE_FIND_LIBRARY_SUFFIXES .a) -endif() - -# Find library -find_library(LZ4_LIBRARY - NAMES ${lz4_LIBNAME} - HINTS ${lz4_PKGCONF_LIBDIR} - PATH_SUFFIXES lib - ) -if (LZ4_LIBRARY) - # NOTE: This must be set for find_package_handle_standard_args to work - set(LZ4_FOUND ON) -endif() - -if(LZ4_USE_STATIC_LIBS) - FindStaticLibraryDependencies(${lz4_LIBNAME} lz4 "${lz4_PKGCONF_STATIC_LIBRARIES}") - - # Restore original value of CMAKE_FIND_LIBRARY_SUFFIXES - set(CMAKE_FIND_LIBRARY_SUFFIXES ${lz4_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES}) - unset(lz4_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES) -endif() - -FindDynamicLibraryDependencies(lz4 "${lz4_DYNAMIC_LIBS}") - -# Set version -set(LZ4_VERSION ${lz4_PKGCONF_VERSION}) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(LZ4 - REQUIRED_VARS LZ4_INCLUDE_DIR - VERSION_VAR LZ4_VERSION - ) - -if(NOT TARGET LZ4::LZ4) - # Add library to build - if (LZ4_FOUND) - if (LZ4_USE_STATIC_LIBS) - add_library(LZ4::LZ4 STATIC IMPORTED) - else() - # NOTE: We use UNKNOWN so that if the user doesn't have the SHARED - # libraries installed, we can still use the STATIC libraries - add_library(LZ4::LZ4 UNKNOWN IMPORTED) - endif() - endif() - - # Set include directories for library - if(LZ4_INCLUDE_DIR) - set_target_properties(LZ4::LZ4 - PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${LZ4_INCLUDE_DIR}" - ) - endif() - - # Set location of library - if(EXISTS "${LZ4_LIBRARY}") - set_target_properties(LZ4::LZ4 - PROPERTIES - IMPORTED_LINK_INTERFACE_LANGUAGES "C" - IMPORTED_LOCATION "${LZ4_LIBRARY}" - ) - - # Add component's dependencies for linking - if(lz4_LIBRARY_DEPENDENCIES) - set_target_properties(LZ4::LZ4 - PROPERTIES - INTERFACE_LINK_LIBRARIES "${lz4_LIBRARY_DEPENDENCIES}" - ) - endif() - endif() -endif() diff --git a/components/core/cmake/Modules/FindZStd.cmake b/components/core/cmake/Modules/FindZStd.cmake deleted file mode 100644 index 503dfebfac..0000000000 --- a/components/core/cmake/Modules/FindZStd.cmake +++ /dev/null @@ -1,106 +0,0 @@ -# Try to find ZStd -# -# Set ZStd_USE_STATIC_LIBS=ON to look for static libraries. -# -# Once done this will define: -# ZStd_FOUND - Whether ZStd was found on the system -# ZStd_INCLUDE_DIR - The ZStd include directories -# ZStd_VERSION - The version of ZStd installed on the system -# -# Conventions: -# - Variables only for use within the script are prefixed with "zstd_" -# - Variables that should be externally visible are prefixed with "ZStd_" - -set(zstd_LIBNAME "zstd") - -include(cmake/Modules/FindLibraryDependencies.cmake) - -# Run pkg-config -find_package(PkgConfig) -pkg_check_modules(zstd_PKGCONF QUIET lib${zstd_LIBNAME}) - -# Set include directory -find_path(ZStd_INCLUDE_DIR zstd.h - HINTS ${zstd_PKGCONF_INCLUDEDIR} - PATH_SUFFIXES include - ) - -# Handle static libraries -if(ZStd_USE_STATIC_LIBS) - # Save current value of CMAKE_FIND_LIBRARY_SUFFIXES - set(zstd_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) - - # Temporarily change CMAKE_FIND_LIBRARY_SUFFIXES to static library suffix - set(CMAKE_FIND_LIBRARY_SUFFIXES .a) -endif() - -# Find library -find_library(ZStd_LIBRARY - NAMES zstd - HINTS ${zstd_PKGCONF_LIBDIR} - PATH_SUFFIXES lib - ) -if (ZStd_LIBRARY) - # NOTE: This must be set for find_package_handle_standard_args to work - set(ZStd_FOUND ON) -endif() - -if(ZStd_USE_STATIC_LIBS) - FindStaticLibraryDependencies(${zstd_LIBNAME} zstd "${zstd_PKGCONF_STATIC_LIBRARIES}") - - # Restore original value of CMAKE_FIND_LIBRARY_SUFFIXES - set(CMAKE_FIND_LIBRARY_SUFFIXES ${zstd_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES}) - unset(zstd_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES) -endif() - -# Add pthread manually since zstd's pkgconfig doesn't include it -list(APPEND zstd_DYNAMIC_LIBS "pthread") - -FindDynamicLibraryDependencies(zstd "${zstd_DYNAMIC_LIBS}") - -# Set version -set(ZStd_VERSION ${zstd_PKGCONF_VERSION}) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(ZStd - REQUIRED_VARS ZStd_INCLUDE_DIR - VERSION_VAR ZStd_VERSION - ) - -if(NOT TARGET ZStd::ZStd) - # Add library to build - if (ZStd_FOUND) - if (ZStd_USE_STATIC_LIBS) - add_library(ZStd::ZStd STATIC IMPORTED) - else() - # NOTE: We use UNKNOWN so that if the user doesn't have the SHARED - # libraries installed, we can still use the STATIC libraries - add_library(ZStd::ZStd UNKNOWN IMPORTED) - endif() - endif() - - # Set include directories for library - if(ZStd_INCLUDE_DIR) - set_target_properties(ZStd::ZStd - PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${ZStd_INCLUDE_DIR}" - ) - endif() - - # Set location of library - if(EXISTS "${ZStd_LIBRARY}") - set_target_properties(ZStd::ZStd - PROPERTIES - IMPORTED_LINK_INTERFACE_LANGUAGES "C" - IMPORTED_LOCATION "${ZStd_LIBRARY}" - ) - - # Add component's dependencies for linking - if(zstd_LIBRARY_DEPENDENCIES) - set_target_properties(ZStd::ZStd - PROPERTIES - INTERFACE_LINK_LIBRARIES "${zstd_LIBRARY_DEPENDENCIES}" - ) - endif() - endif() -endif() diff --git a/components/core/src/clp/clg/CMakeLists.txt b/components/core/src/clp/clg/CMakeLists.txt index d4443536d7..fd69eb2b8a 100644 --- a/components/core/src/clp/clg/CMakeLists.txt +++ b/components/core/src/clp/clg/CMakeLists.txt @@ -150,7 +150,7 @@ if(CLP_BUILD_EXECUTABLES) yaml-cpp ystdlib::containers ystdlib::error_handling - ZStd::ZStd + ${zstd_TARGET} ) # Put the built executable at the root of the build directory set_target_properties( diff --git a/components/core/src/clp/clo/CMakeLists.txt b/components/core/src/clp/clo/CMakeLists.txt index fef53eb8de..d53fe21d1f 100644 --- a/components/core/src/clp/clo/CMakeLists.txt +++ b/components/core/src/clp/clo/CMakeLists.txt @@ -178,7 +178,7 @@ if(CLP_BUILD_EXECUTABLES) clp::string_utils ystdlib::containers ystdlib::error_handling - ZStd::ZStd + ${zstd_TARGET} ) # Put the built executable at the root of the build directory set_target_properties( diff --git a/components/core/src/clp/clp/CMakeLists.txt b/components/core/src/clp/clp/CMakeLists.txt index e9b35c296e..c468725ed7 100644 --- a/components/core/src/clp/clp/CMakeLists.txt +++ b/components/core/src/clp/clp/CMakeLists.txt @@ -189,7 +189,7 @@ if(CLP_BUILD_EXECUTABLES) yaml-cpp ystdlib::containers ystdlib::error_handling - ZStd::ZStd + ${zstd_TARGET} ) # Put the built executable at the root of the build directory set_target_properties( diff --git a/components/core/src/clp/make_dictionaries_readable/CMakeLists.txt b/components/core/src/clp/make_dictionaries_readable/CMakeLists.txt index 65bf33e03e..c979b9453b 100644 --- a/components/core/src/clp/make_dictionaries_readable/CMakeLists.txt +++ b/components/core/src/clp/make_dictionaries_readable/CMakeLists.txt @@ -49,7 +49,7 @@ if(CLP_BUILD_EXECUTABLES) spdlog::spdlog clp::string_utils ystdlib::containers - ZStd::ZStd + ${zstd_TARGET} ) # Put the built executable at the root of the build directory set_target_properties( diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index 076ef8cbfe..46cbda49d3 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -124,7 +124,7 @@ if(CLP_BUILD_CLP_S_CLP_DEPENDENCIES) OpenSSL::Crypto spdlog::spdlog ystdlib::error_handling - ZStd::ZStd + ${zstd_TARGET} ) endif() @@ -204,7 +204,7 @@ if(CLP_BUILD_CLP_S_IO) clp_s::clp_dependencies fmt::fmt spdlog::spdlog - ZStd::ZStd + ${zstd_TARGET} ) endif() diff --git a/components/core/src/clp_s/indexer/CMakeLists.txt b/components/core/src/clp_s/indexer/CMakeLists.txt index ba5e83e44d..43e4e88354 100644 --- a/components/core/src/clp_s/indexer/CMakeLists.txt +++ b/components/core/src/clp_s/indexer/CMakeLists.txt @@ -124,7 +124,7 @@ if(CLP_BUILD_EXECUTABLES) spdlog::spdlog yaml-cpp ystdlib::containers - ZStd::ZStd + ${zstd_TARGET} ) # Put the built executable at the root of the build directory set_target_properties( diff --git a/components/core/src/glt/glt/CMakeLists.txt b/components/core/src/glt/glt/CMakeLists.txt index 8576f391f5..6936dab8ea 100644 --- a/components/core/src/glt/glt/CMakeLists.txt +++ b/components/core/src/glt/glt/CMakeLists.txt @@ -192,7 +192,7 @@ if(CLP_BUILD_EXECUTABLES) clp::string_utils yaml-cpp ystdlib::error_handling - ZStd::ZStd + ${zstd_TARGET} ) # Put the built executable at the root of the build directory set_target_properties( diff --git a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh index 1d5b253a57..f91035b61c 100755 --- a/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/centos-stream-9/install-prebuilt-packages.sh @@ -16,6 +16,7 @@ dnf install -y \ libarchive-devel \ libcurl-devel \ libzstd-devel \ + lz4-devel \ make \ mariadb-connector-c-devel \ openssl-devel \ diff --git a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-packages-from-source.sh b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-packages-from-source.sh index af1d2774a9..161058ae9f 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-packages-from-source.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-packages-from-source.sh @@ -14,6 +14,6 @@ lib_install_scripts_dir=$script_dir/.. "$lib_install_scripts_dir"/libarchive.sh 3.5.1 "$lib_install_scripts_dir"/liblzma.sh 5.8.1 -"$lib_install_scripts_dir"/lz4.sh 1.8.2 +"$lib_install_scripts_dir"/lz4.sh 1.10.0 "$lib_install_scripts_dir"/msgpack.sh 7.0.0 -"$lib_install_scripts_dir"/zstandard.sh 1.4.9 +"$lib_install_scripts_dir"/zstandard.sh 1.5.7 diff --git a/docs/src/dev-guide/components-core/index.md b/docs/src/dev-guide/components-core/index.md index 2d0b768bea..20db3ed9a8 100644 --- a/docs/src/dev-guide/components-core/index.md +++ b/docs/src/dev-guide/components-core/index.md @@ -42,6 +42,7 @@ The task will download, build, and install (within the build directory) the foll | [fmt](https://github.com/fmtlib/fmt) | v10.2.1 | | [json](https://github.com/nlohmann/json.git) | v3.11.3 | | [log-surgeon](https://github.com/y-scope/log-surgeon) | f801a3f | +| [lz4](https://github.com/lz4/lz4) | v1.10.0 | | [mongo-cxx-driver](https://github.com/mongodb/mongo-cxx-driver) | r3.10.2 | | [simdjson](https://github.com/simdjson/simdjson) | v3.13.0 | | [spdlog](https://github.com/gabime/spdlog) | v1.14.1 | @@ -50,6 +51,7 @@ The task will download, build, and install (within the build directory) the foll | [yaml-cpp](https://github.com/jbeder/yaml-cpp.git) | v0.7.0 | | [yscope-log-viewer](https://github.com/y-scope/yscope-log-viewer.git) | 969ff35 | | [ystdlib-cpp](https://github.com/y-scope/ystdlib-cpp.git) | d80cf86 | +| [zstd](https://github.com/facebook/zstd) | v1.5.7 | ### Environment diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index e4cbd258c6..cdb5905a3b 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -67,6 +67,7 @@ tasks: - task: "date" - task: "fmt" - task: "log-surgeon" + - task: "lz4" - task: "microsoft.gsl" - task: "mongocxx" - task: "nlohmann_json" @@ -76,6 +77,7 @@ tasks: - task: "utfcpp" - task: "yaml-cpp" - task: "ystdlib" + - task: "zstd" absl: internal: true @@ -238,6 +240,23 @@ tasks: TARBALL_SHA256: "d91a7469db437162d518eb0f9175268084b478b5003163d0f6233abbbce9b0f8" TARBALL_URL: "https://github.com/y-scope/log-surgeon/archive/dfd5c79.tar.gz" + lz4: + internal: true + run: "once" + cmds: + - task: "utils:install-remote-cmake-lib" + vars: + CMAKE_GEN_ARGS: + - "-DBUILD_SHARED_LIBS=ON" + - "-DBUILD_STATIC_LIBS=ON" + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + - "-DLZ4_BUILD_CLI=OFF" + CMAKE_SOURCE_DIR: "build/cmake" + LIB_NAME: "lz4" + TARBALL_SHA256: "537512904744b35e232912055ccf8ec66d768639ff3abe5788d90d792ec5f48b" + TARBALL_URL: "https://github.com/lz4/lz4/releases/download/v1.10.0/lz4-1.10.0.tar.gz" + microsoft.gsl: internal: true run: "once" @@ -413,3 +432,22 @@ tasks: echo "set( CLP_YSTDLIB_SOURCE_DIRECTORY \"{{.YSTDLIB_OUTPUT_DIR}}\" )" > "{{.G_DEPS_CORE_CMAKE_SETTINGS_DIR}}/{{.LIB_NAME}}.cmake" + + zstd: + internal: true + run: "once" + cmds: + - task: "utils:install-remote-cmake-lib" + vars: + CMAKE_GEN_ARGS: + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" + - "-DZSTD_BUILD_CONTRIB=OFF" + - "-DZSTD_BUILD_PROGRAMS=OFF" + - "-DZSTD_BUILD_SHARED=ON" + - "-DZSTD_BUILD_STATIC=ON" + - "-DZSTD_BUILD_TESTS=OFF" + CMAKE_SOURCE_DIR: "build/cmake" + LIB_NAME: "zstd" + TARBALL_SHA256: "eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3" + TARBALL_URL: "https://github.com/facebook/zstd/releases/download/v1.5.7/zstd-1.5.7.tar.gz" diff --git a/taskfiles/deps/utils.yaml b/taskfiles/deps/utils.yaml index a80fe953a7..4ab651b7a7 100644 --- a/taskfiles/deps/utils.yaml +++ b/taskfiles/deps/utils.yaml @@ -46,11 +46,15 @@ tasks: # @param {string} TARBALL_URL # @param {string[]} [CMAKE_GEN_ARGS] Any additional arguments to pass to the CMake generate # command. + # @param {string} [CMAKE_SOURCE_DIR=.] The path, within the tarball extraction directory, + # containing the project's top level CMakeLists.txt. install-remote-cmake-lib: internal: true vars: CMAKE_GEN_ARGS: ref: "default (list) .CMAKE_GEN_ARGS" + CMAKE_SOURCE_DIR: >- + {{default "." .CMAKE_SOURCE_DIR}} INSTALL_PREFIX: "{{.G_DEPS_CORE_DIR}}/{{.LIB_NAME}}-install" INSTALL_DIR_CHECKSUM_FILE: "{{.G_DEPS_CORE_CHECKSUMS_DIR}}/{{.LIB_NAME}}.md5" deps: @@ -68,6 +72,7 @@ tasks: CMAKE_JOBS: "{{.G_CORE_MAX_PARALLELISM_PER_BUILD_TASK}}" CMAKE_PACKAGE_NAME: "{{.LIB_NAME}}" CMAKE_SETTINGS_DIR: "{{.G_DEPS_CORE_CMAKE_SETTINGS_DIR}}" + CMAKE_SOURCE_DIR: "{{.CMAKE_SOURCE_DIR}}" TAR_SHA256: "{{.TARBALL_SHA256}}" TAR_URL: "{{.TARBALL_URL}}" WORK_DIR: "{{.G_DEPS_CORE_DIR}}"