Add examples

Author: Alex-PLACET

.github/workflows/linux.yml

@@ -59,6 +59,14 @@ jobs:
         working-directory: build
         run: cmake --build . --target run_example
 
+      - name: Build deserializer example
+        working-directory: build
+        run: cmake --build . --target deserializer_example
+
+      - name: Run deserializer example
+        working-directory: build
+        run: cmake --build . --target run_deserializer_example
+
       - name: Install
         working-directory: build
         run: cmake --install .
@@ -103,6 +111,14 @@ jobs:
         working-directory: build
         run: cmake --build . --target run_example
 
+      - name: Build deserializer example
+        working-directory: build
+        run: cmake --build . --target deserializer_example
+
+      - name: Run deserializer example
+        working-directory: build
+        run: cmake --build . --target run_deserializer_example
+
       - name: Install
         working-directory: build
         run: sudo cmake --install .

.github/workflows/osx.yml

@@ -64,6 +64,14 @@ jobs:
         working-directory: build
         run: cmake --build . --target run_example
 
+      - name: Build deserializer example
+        working-directory: build
+        run: cmake --build . --target deserializer_example
+
+      - name: Run deserializer example
+        working-directory: build
+        run: cmake --build . --target run_deserializer_example
+
       - name: Install
         working-directory: build
         run: cmake --install .
@@ -113,6 +121,14 @@ jobs:
         working-directory: build
         run: cmake --build . --target run_example
 
+      - name: Build deserializer example
+        working-directory: build
+        run: cmake --build . --target deserializer_example
+
+      - name: Run deserializer example
+        working-directory: build
+        run: cmake --build . --target run_deserializer_example
+
       - name: Install
         working-directory: build
         run: sudo cmake --install .

.github/workflows/windows.yml

@@ -64,6 +64,14 @@ jobs:
         working-directory: build
         run: cmake --build . --config ${{ matrix.build_type }} --target run_example
 
+      - name: Build deserializer example
+        working-directory: build
+        run: cmake --build . --config ${{ matrix.build_type }} --target deserializer_example
+
+      - name: Run deserializer example
+        working-directory: build
+        run: cmake --build . --config ${{ matrix.build_type }} --target run_deserializer_example
+
       - name: Install
         working-directory: build
         run: cmake --install . --config ${{ matrix.build_type }}
@@ -113,6 +121,14 @@ jobs:
         working-directory: build
         run: cmake --build . --config ${{ matrix.build_type }} --target run_example
 
+      - name: Build deserializer example
+        working-directory: build
+        run: cmake --build . --config ${{ matrix.build_type }} --target deserializer_example
+
+      - name: Run deserializer example
+        working-directory: build
+        run: cmake --build . --config ${{ matrix.build_type }} --target run_deserializer_example
+
       - name: Install
         working-directory: build
         run: cmake --install . --config ${{ matrix.build_type }}

README.md

@@ -117,16 +117,20 @@ void stream_record_batches(std::ostream& os, record_batch_source& source)
 namespace sp = sparrow;
 namespace sp_ipc = sparrow_ipc;
 
-std::vector<sp::record_batch> deserialize_stream_to_batches(const std::vector<uint8_t>& stream_data)
+std::vector<sp::record_batch> deserialize_stream_example(const std::vector<uint8_t>& stream_data)
 {
+    // Deserialize the entire stream at once
     auto batches = sp_ipc::deserialize_stream(stream_data);
     return batches;
 }
 ```
 
 #### Using the deserializer class
 
+The deserializer class allows you to accumulate record batches into an existing container as you deserialize data:
+
 ```cpp
+#include <iostream>
 #include <span>
 #include <vector>
 #include <sparrow_ipc/deserializer.hpp>
@@ -135,22 +139,56 @@ std::vector<sp::record_batch> deserialize_stream_to_batches(const std::vector<ui
 namespace sp = sparrow;
 namespace sp_ipc = sparrow_ipc;
 
-void deserialize_incremental_stream(const std::vector<std::vector<uint8_t>>& stream_chunks)
+void deserializer_basic_example(const std::vector<uint8_t>& stream_data)
 {
+    // Create a container to hold the deserialized batches
     std::vector<sp::record_batch> batches;
+
+    // Create a deserializer that will append to our container
     sp_ipc::deserializer deser(batches);
 
-    // Deserialize chunks incrementally as they arrive
-    for (const auto& chunk : stream_chunks)
+    // Deserialize the stream data
+    deser.deserialize(std::span<const uint8_t>(stream_data));
+
+    // Process the accumulated batches
+    for (const auto& batch : batches)
     {
-        deser << std::span<const uint8_t>(chunk);
+        std::cout << "Batch with " << batch.nb_rows() << " rows and " << batch.nb_columns() << " columns\n";
     }
+}
+```
 
-    // Process accumulated batches
-    for (const auto& batch : batches)
+#### Incremental deserialization
+
+The deserializer class is particularly useful for streaming scenarios where data arrives in chunks:
+
+```cpp
+#include <iostream>
+#include <span>
+#include <vector>
+#include <sparrow_ipc/deserializer.hpp>
+#include <sparrow/record_batch.hpp>
+
+namespace sp = sparrow;
+namespace sp_ipc = sparrow_ipc;
+
+void deserializer_incremental_example(const std::vector<std::vector<uint8_t>>& stream_chunks)
+{
+    // Container to accumulate all deserialized batches
+    std::vector<sp::record_batch> batches;
+
+    // Create a deserializer
+    sp_ipc::deserializer deser(batches);
+
+    // Deserialize chunks as they arrive using the streaming operator
+    for (const auto& chunk : stream_chunks)
     {
-        // Process each batch...
+        deser << std::span<const uint8_t>(chunk);
+        std::cout << "After chunk: " << batches.size() << " batches accumulated\n";
     }
+
+    // All batches are now available in the container
+    std::cout << "Total batches deserialized: " << batches.size() << "\n";
 }
 ```
 

examples/CMakeLists.txt

@@ -11,8 +11,18 @@ target_link_libraries(write_and_read_streams
         arrow-testing-data 
 )
 
+# Create executable for the deserializer_example
+add_executable(deserializer_example deserializer_example.cpp)
+
+# Link against sparrow-ipc and its dependencies
+target_link_libraries(deserializer_example
+    PRIVATE
+        sparrow-ipc
+        sparrow::sparrow
+)
+
 # Set C++ standard to match the main project
-set_target_properties(write_and_read_streams
+set_target_properties(write_and_read_streams deserializer_example
     PROPERTIES
         CXX_STANDARD 20
         CXX_STANDARD_REQUIRED ON
@@ -26,8 +36,15 @@ target_include_directories(write_and_read_streams
         ${CMAKE_BINARY_DIR}/generated
 )
 
+target_include_directories(deserializer_example
+    PRIVATE
+        ${CMAKE_SOURCE_DIR}/include
+        ${CMAKE_BINARY_DIR}/generated
+)
+
 # Ensure generated flatbuffer headers are available
 add_dependencies(write_and_read_streams generate_flatbuffers_headers)
+add_dependencies(deserializer_example generate_flatbuffers_headers)
 
 # Optional: Copy to build directory for easy execution
 if(WIN32)
@@ -38,7 +55,7 @@ if(WIN32)
         set(ZSTD_DLL_TARGET libzstd_static)
     endif()
 
-    # On Windows, copy required DLLs
+    # On Windows, copy required DLLs for write_and_read_streams
     set(DLL_COPY_COMMANDS "") # Initialize a list to hold all copy commands
     # Add unconditional copy commands
     list(APPEND DLL_COPY_COMMANDS
@@ -66,6 +83,31 @@ if(WIN32)
         ${DLL_COPY_COMMANDS}
         COMMENT "Copying required DLLs to example executable directory"
     )
+
+    # On Windows, copy required DLLs for deserializer_example
+    set(DLL_COPY_COMMANDS_DESER "") # Initialize a list to hold all copy commands
+    list(APPEND DLL_COPY_COMMANDS_DESER
+        COMMAND ${CMAKE_COMMAND} -E copy_if_different
+            "$<TARGET_FILE:sparrow::sparrow>"
+            "$<TARGET_FILE_DIR:deserializer_example>"
+        COMMAND ${CMAKE_COMMAND} -E copy_if_different
+            "$<TARGET_FILE:sparrow-ipc>"
+            "$<TARGET_FILE_DIR:deserializer_example>"
+    )
+
+    if(ZSTD_DLL_TARGET)
+        list(APPEND DLL_COPY_COMMANDS_DESER
+            COMMAND ${CMAKE_COMMAND} -E copy_if_different
+                "$<TARGET_FILE:${ZSTD_DLL_TARGET}>"
+                "$<TARGET_FILE_DIR:deserializer_example>"
+        )
+    endif()
+
+    add_custom_command(
+        TARGET deserializer_example POST_BUILD
+        ${DLL_COPY_COMMANDS_DESER}
+        COMMENT "Copying required DLLs to deserializer_example executable directory"
+    )
 endif()
 
 # Create a custom target to easily run the example
@@ -77,3 +119,13 @@ add_custom_target(run_example
 )
 
 set_target_properties(run_example PROPERTIES FOLDER "Examples")
+
+# Create a custom target to run the deserializer example
+add_custom_target(run_deserializer_example
+    COMMAND deserializer_example
+    DEPENDS deserializer_example
+    COMMENT "Running deserializer_example"
+    USES_TERMINAL
+)
+
+set_target_properties(run_deserializer_example PROPERTIES FOLDER "Examples")