Merge remote-tracking branch 'upstream/main' into add_deserializer

Author: Alex-PLACET

CMakeLists.txt

@@ -108,11 +108,11 @@ set(SPARROW_IPC_HEADERS
     ${SPARROW_IPC_INCLUDE_DIR}/sparrow_ipc/chunk_memory_serializer.hpp
     ${SPARROW_IPC_INCLUDE_DIR}/sparrow_ipc/config/config.hpp
     ${SPARROW_IPC_INCLUDE_DIR}/sparrow_ipc/config/sparrow_ipc_version.hpp
+    ${SPARROW_IPC_INCLUDE_DIR}/sparrow_ipc/compression.hpp
     ${SPARROW_IPC_INCLUDE_DIR}/sparrow_ipc/deserialize_fixedsizebinary_array.hpp
     ${SPARROW_IPC_INCLUDE_DIR}/sparrow_ipc/deserialize_primitive_array.hpp
     ${SPARROW_IPC_INCLUDE_DIR}/sparrow_ipc/deserialize_utils.hpp
     ${SPARROW_IPC_INCLUDE_DIR}/sparrow_ipc/deserialize_variable_size_binary_array.hpp
-    ${SPARROW_IPC_INCLUDE_DIR}/sparrow_ipc/deserialize_variable_size_binary_array.hpp
     ${SPARROW_IPC_INCLUDE_DIR}/sparrow_ipc/deserialize.hpp
     ${SPARROW_IPC_INCLUDE_DIR}/sparrow_ipc/deserializer.hpp
     ${SPARROW_IPC_INCLUDE_DIR}/sparrow_ipc/encapsulated_message.hpp
@@ -123,6 +123,7 @@ set(SPARROW_IPC_HEADERS
     ${SPARROW_IPC_INCLUDE_DIR}/sparrow_ipc/serialize_utils.hpp
     ${SPARROW_IPC_INCLUDE_DIR}/sparrow_ipc/serialize.hpp
     ${SPARROW_IPC_INCLUDE_DIR}/sparrow_ipc/serializer.hpp
+    ${SPARROW_IPC_INCLUDE_DIR}/sparrow_ipc/stream_file_serializer.hpp
     ${SPARROW_IPC_INCLUDE_DIR}/sparrow_ipc/utils.hpp
 )
 
@@ -133,6 +134,8 @@ set(SPARROW_IPC_SRC
     ${SPARROW_IPC_SOURCE_DIR}/arrow_interface/arrow_schema.cpp
     ${SPARROW_IPC_SOURCE_DIR}/arrow_interface/arrow_schema/private_data.cpp
     ${SPARROW_IPC_SOURCE_DIR}/chunk_memory_serializer.cpp
+    ${SPARROW_IPC_SOURCE_DIR}/compression.cpp
+    ${SPARROW_IPC_SOURCE_DIR}/compression_impl.hpp
     ${SPARROW_IPC_SOURCE_DIR}/deserialize_fixedsizebinary_array.cpp
     ${SPARROW_IPC_SOURCE_DIR}/deserialize_utils.cpp
     ${SPARROW_IPC_SOURCE_DIR}/deserialize.cpp
@@ -142,6 +145,7 @@ set(SPARROW_IPC_SRC
     ${SPARROW_IPC_SOURCE_DIR}/serialize_utils.cpp
     ${SPARROW_IPC_SOURCE_DIR}/serialize.cpp
     ${SPARROW_IPC_SOURCE_DIR}/serializer.cpp
+    ${SPARROW_IPC_SOURCE_DIR}/stream_file_serializer.cpp
     ${SPARROW_IPC_SOURCE_DIR}/utils.cpp
 )
 
@@ -254,6 +258,9 @@ target_link_libraries(sparrow-ipc
     PUBLIC
         sparrow::sparrow
         flatbuffers::flatbuffers
+    PRIVATE
+        lz4::lz4
+        zstd::libzstd
         )
 
 # Ensure generated headers are available when building sparrow-ipc
@@ -319,6 +326,25 @@ if (TARGET flatbuffers)
     endif()
 endif()
 
+if (TARGET lz4)
+    get_target_property(is_imported lz4 IMPORTED)
+    if(NOT is_imported)
+        # This means `lz4` was fetched using FetchContent
+        # We need to export `lz4` target explicitly
+        list(APPEND SPARROW_IPC_EXPORTED_TARGETS lz4)
+    endif()
+endif()
+
+if (TARGET lz4_static)
+    get_target_property(is_imported lz4_static IMPORTED)
+    if(NOT is_imported)
+        # `lz4_static` is needed as this is the actual library
+        # and `lz4` is an interface pointing to it.
+        # If `lz4_shared` is used instead for some reason, modify this accordingly
+        list(APPEND SPARROW_IPC_EXPORTED_TARGETS lz4_static)
+    endif()
+endif()
+
 install(TARGETS ${SPARROW_IPC_EXPORTED_TARGETS}
     EXPORT ${PROJECT_NAME}-targets)
 

cmake/Findlz4.cmake

@@ -0,0 +1,42 @@
+# Find LZ4 library and headers
+
+# This module defines:
+#  LZ4_FOUND          - True if lz4 is found
+#  LZ4_INCLUDE_DIRS   - LZ4 include directories
+#  LZ4_LIBRARIES      - Libraries needed to use LZ4
+#  LZ4_VERSION        - LZ4 version number
+#
+
+find_package(PkgConfig)
+if(PKG_CONFIG_FOUND)
+    pkg_check_modules(LZ4 QUIET liblz4)
+    if(NOT LZ4_FOUND)
+        message(STATUS "Did not find 'liblz4.pc', trying 'lz4.pc'")
+        pkg_check_modules(LZ4 QUIET lz4)
+    endif()
+endif()
+
+find_path(LZ4_INCLUDE_DIR lz4.h)
+#      HINTS ${LZ4_INCLUDEDIR} ${LZ4_INCLUDE_DIRS})
+find_library(LZ4_LIBRARY NAMES lz4 liblz4)
+#     HINTS ${LZ4_LIBDIR} ${LZ4_LIBRARY_DIRS})
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(lz4 DEFAULT_MSG
+    LZ4_LIBRARY LZ4_INCLUDE_DIR)
+mark_as_advanced(LZ4_INCLUDE_DIR LZ4_LIBRARY)
+
+set(LZ4_LIBRARIES ${LZ4_LIBRARY})
+set(LZ4_INCLUDE_DIRS ${LZ4_INCLUDE_DIR})
+
+if(LZ4_FOUND AND NOT TARGET lz4::lz4)
+    add_library(lz4::lz4 UNKNOWN IMPORTED)
+    set_target_properties(lz4::lz4 PROPERTIES
+        IMPORTED_LOCATION "${LZ4_LIBRARIES}"
+        INTERFACE_INCLUDE_DIRECTORIES "${LZ4_INCLUDE_DIRS}")
+    if (NOT TARGET LZ4::LZ4 AND TARGET lz4::lz4)
+      add_library(LZ4::LZ4 ALIAS lz4::lz4)
+    endif ()
+endif()
+
+#TODO add version?

cmake/external_dependencies.cmake

@@ -11,8 +11,8 @@ endif()
 
 function(find_package_or_fetch)
     set(options)
-    set(oneValueArgs CONAN_PKG_NAME PACKAGE_NAME GIT_REPOSITORY TAG)
-    set(multiValueArgs)
+    set(oneValueArgs CONAN_PKG_NAME PACKAGE_NAME GIT_REPOSITORY TAG SOURCE_SUBDIR)
+    set(multiValueArgs CMAKE_ARGS)
     cmake_parse_arguments(PARSE_ARGV 0 arg
         "${options}" "${oneValueArgs}" "${multiValueArgs}"
     )
@@ -26,21 +26,43 @@ function(find_package_or_fetch)
         find_package(${actual_pkg_name} ${FIND_PACKAGE_OPTIONS})
     endif()
 
-    if(FETCH_DEPENDENCIES_WITH_CMAKE STREQUAL "ON" OR FETCH_DEPENDENCIES_WITH_CMAKE STREQUAL "MISSING")
-        if(NOT ${actual_pkg_name}_FOUND)
-            message(STATUS "📦 Fetching ${arg_PACKAGE_NAME}")
-            FetchContent_Declare(
-                ${arg_PACKAGE_NAME}
-                GIT_SHALLOW TRUE
-                GIT_REPOSITORY ${arg_GIT_REPOSITORY}
-                GIT_TAG ${arg_TAG}
-                GIT_PROGRESS TRUE
-                SYSTEM
-                EXCLUDE_FROM_ALL)
-            FetchContent_MakeAvailable(${arg_PACKAGE_NAME})
-            message(STATUS "\t✅ Fetched ${arg_PACKAGE_NAME}")
-        else()
+    if(arg_GIT_REPOSITORY)
+        if(FETCH_DEPENDENCIES_WITH_CMAKE STREQUAL "ON" OR FETCH_DEPENDENCIES_WITH_CMAKE STREQUAL "MISSING")
+            if(NOT ${actual_pkg_name}_FOUND)
+                message(STATUS "📦 Fetching ${arg_PACKAGE_NAME}")
+                # Apply CMAKE_ARGS before fetching
+                foreach(cmake_arg ${arg_CMAKE_ARGS})
+                    string(REGEX MATCH "^([^=]+)=(.*)$" _ ${cmake_arg})
+                    if(CMAKE_MATCH_1)
+                        set(${CMAKE_MATCH_1} ${CMAKE_MATCH_2} CACHE BOOL "" FORCE)
+                    endif()
+                endforeach()
+                set(fetch_args
+                    ${arg_PACKAGE_NAME}
+                    GIT_SHALLOW TRUE
+                    GIT_REPOSITORY ${arg_GIT_REPOSITORY}
+                    GIT_TAG ${arg_TAG}
+                    GIT_PROGRESS TRUE
+                    SYSTEM
+                    EXCLUDE_FROM_ALL)
+                if(arg_SOURCE_SUBDIR)
+                    list(APPEND fetch_args SOURCE_SUBDIR ${arg_SOURCE_SUBDIR})
+                endif()
+                FetchContent_Declare(${fetch_args})
+                FetchContent_MakeAvailable(${arg_PACKAGE_NAME})
+                message(STATUS "\t✅ Fetched ${arg_PACKAGE_NAME}")
+            else()
+                message(STATUS "📦 ${actual_pkg_name} found here: ${${actual_pkg_name}_DIR}")
+            endif()
+        endif()
+    else()
+        # No GIT_REPOSITORY provided - only find_package is attempted
+        if(${actual_pkg_name}_FOUND)
             message(STATUS "📦 ${actual_pkg_name} found here: ${${actual_pkg_name}_DIR}")
+        elseif(FETCH_DEPENDENCIES_WITH_CMAKE STREQUAL "OFF")
+            message(FATAL_ERROR "Could not find ${actual_pkg_name} and no GIT_REPOSITORY provided for fetching")
+        else()
+            message(WARNING "Could not find ${actual_pkg_name} and no GIT_REPOSITORY provided for fetching")
         endif()
     endif()
 endfunction()
@@ -52,14 +74,17 @@ endif()
 find_package_or_fetch(
     PACKAGE_NAME sparrow
     GIT_REPOSITORY https://github.com/man-group/sparrow.git
-    TAG 1.2.0
+    TAG 1.3.0
 )
 unset(CREATE_JSON_READER_TARGET)
 
 if(NOT TARGET sparrow::sparrow)
     add_library(sparrow::sparrow ALIAS sparrow)
 endif()
 if(${SPARROW_IPC_BUILD_TESTS})
+    find_package_or_fetch(
+        PACKAGE_NAME sparrow-json-reader
+    )
     if(NOT TARGET sparrow::json_reader)
         add_library(sparrow::json_reader ALIAS json_reader)
     endif()
@@ -79,6 +104,50 @@ if(NOT TARGET flatbuffers::flatbuffers)
 endif()
 unset(FLATBUFFERS_BUILD_TESTS CACHE)
 
+# Fetching lz4
+# Disable bundled mode to allow shared libraries if needed
+# lz4 is built as static by default if bundled
+# set(LZ4_BUNDLED_MODE OFF CACHE BOOL "" FORCE)
+# set(BUILD_SHARED_LIBS ON CACHE BOOL "" FORCE)
+find_package_or_fetch(
+    PACKAGE_NAME lz4
+    GIT_REPOSITORY https://github.com/lz4/lz4.git
+    TAG v1.10.0
+    SOURCE_SUBDIR build/cmake
+    CMAKE_ARGS
+        "LZ4_BUILD_CLI=OFF"
+        "LZ4_BUILD_LEGACY_LZ4C=OFF"
+)
+
+if(NOT TARGET lz4::lz4)
+    add_library(lz4::lz4 ALIAS lz4)
+endif()
+
+find_package_or_fetch(
+    PACKAGE_NAME zstd
+    GIT_REPOSITORY https://github.com/facebook/zstd.git
+    TAG v1.5.7
+    SOURCE_SUBDIR build/cmake
+    CMAKE_ARGS
+        "ZSTD_BUILD_PROGRAMS=OFF"
+)
+
+if(NOT TARGET zstd::libzstd)
+    if(SPARROW_IPC_BUILD_SHARED)
+        if(TARGET zstd::libzstd_shared) # Linux case
+            add_library(zstd::libzstd ALIAS zstd::libzstd_shared)
+        elseif(TARGET libzstd_shared) # Windows case
+            add_library(zstd::libzstd ALIAS libzstd_shared)
+        endif()
+    else()
+        if(TARGET zstd::libzstd_static) # Linux case
+            add_library(zstd::libzstd ALIAS zstd::libzstd_static)
+        elseif(TARGET libzstd_static) # Windows case
+            add_library(zstd::libzstd ALIAS libzstd_static)
+        endif()
+    endif()
+endif()
+
 if(SPARROW_IPC_BUILD_TESTS)
     find_package_or_fetch(
         PACKAGE_NAME doctest
@@ -109,10 +178,18 @@ if(SPARROW_IPC_BUILD_TESTS)
     )
     message(STATUS "\t✅ Fetched arrow-testing")
 
-    # Iterate over all the files in the arrow-testing-data source directiory. When it's a gz, extract in place.
-    file(GLOB_RECURSE arrow_testing_data_targz_files CONFIGURE_DEPENDS
+    # Fetch all the files in the cpp-21.0.0 directory
+    file(GLOB_RECURSE arrow_testing_data_targz_files_cpp_21 CONFIGURE_DEPENDS
         "${arrow-testing_SOURCE_DIR}/data/arrow-ipc-stream/integration/cpp-21.0.0/*.json.gz"
     )
+    # Fetch all the files in the 2.0.0-compression directory
+    file(GLOB_RECURSE arrow_testing_data_targz_files_compression CONFIGURE_DEPENDS
+        "${arrow-testing_SOURCE_DIR}/data/arrow-ipc-stream/integration/2.0.0-compression/*.json.gz"
+    )
+
+    # Combine lists of files
+    list(APPEND arrow_testing_data_targz_files ${arrow_testing_data_targz_files_cpp_21} ${arrow_testing_data_targz_files_compression})
+    # Iterate over all the files in the arrow-testing-data source directory. When it's a gz, extract in place.
     foreach(file_path IN LISTS arrow_testing_data_targz_files)
             cmake_path(GET file_path PARENT_PATH parent_dir)
             cmake_path(GET file_path STEM filename)
@@ -128,5 +205,4 @@ if(SPARROW_IPC_BUILD_TESTS)
                 endif()
             endif()
     endforeach()
-
 endif()

conanfile.py

@@ -45,6 +45,8 @@ def configure(self):
     def requirements(self):
         self.requires("sparrow/1.0.0")
         self.requires(f"flatbuffers/{self._flatbuffers_version}")
+        self.requires("lz4/1.9.4")
+        self.requires("zstd/1.5.7")
         if self.options.get_safe("build_tests"):
             self.test_requires("doctest/2.4.12")
 

environment-dev.yml

@@ -8,8 +8,12 @@ dependencies:
   - cxx-compiler
   # Libraries dependencies
   - flatbuffers
+  - lz4-c
+  - zstd
   - nlohmann_json
-  - sparrow-devel >=1.1.2
+  - sparrow-devel
+  - sparrow-json-reader
+  # Testing dependencies
   - doctest
   # Documentation dependencies
   - doxygen

examples/CMakeLists.txt

@@ -31,16 +31,40 @@ add_dependencies(write_and_read_streams generate_flatbuffers_headers)
 
 # Optional: Copy to build directory for easy execution
 if(WIN32)
+    set(ZSTD_DLL_TARGET "")
+    if(TARGET libzstd_shared AND SPARROW_IPC_BUILD_SHARED) # Building deps from src case: shared target without namespace
+        set(ZSTD_DLL_TARGET libzstd_shared)
+    elseif(TARGET libzstd_static AND NOT SPARROW_IPC_BUILD_SHARED) # Building deps from src case: static target without namespace
+        set(ZSTD_DLL_TARGET libzstd_static)
+    endif()
+
     # On Windows, copy required DLLs
-    add_custom_command(
-        TARGET write_and_read_streams POST_BUILD
+    set(DLL_COPY_COMMANDS "") # Initialize a list to hold all copy commands
+    # Add unconditional copy commands
+    list(APPEND DLL_COPY_COMMANDS
         COMMAND ${CMAKE_COMMAND} -E copy_if_different
             "$<TARGET_FILE:sparrow::sparrow>"
             "$<TARGET_FILE_DIR:write_and_read_streams>"
         COMMAND ${CMAKE_COMMAND} -E copy_if_different
             "$<TARGET_FILE:sparrow-ipc>"
             "$<TARGET_FILE_DIR:write_and_read_streams>"
-        COMMENT "Copying sparrow and sparrow-ipc DLLs to example executable directory"
+    )
+
+    # Conditionally add ZSTD copy command
+    if(ZSTD_DLL_TARGET)
+        list(APPEND DLL_COPY_COMMANDS
+            COMMAND ${CMAKE_COMMAND} -E copy_if_different
+                "$<TARGET_FILE:${ZSTD_DLL_TARGET}>"
+                "$<TARGET_FILE_DIR:write_and_read_streams>"
+        )
+    else()
+        message(WARNING "ZSTD DLL will not be copied for examples.")
+    endif()
+
+    add_custom_command(
+        TARGET write_and_read_streams POST_BUILD
+        ${DLL_COPY_COMMANDS}
+        COMMENT "Copying required DLLs to example executable directory"
     )
 endif()