Add read of ipc file in example

Author: Alex-PLACET

cmake/external_dependencies.cmake

@@ -52,7 +52,7 @@ endif()
 find_package_or_fetch(
     PACKAGE_NAME sparrow
     GIT_REPOSITORY https://github.com/man-group/sparrow.git
-    TAG 1.1.2
+    TAG 1.2.0
 )
 unset(CREATE_JSON_READER_TARGET)
 
@@ -111,7 +111,7 @@ if(SPARROW_IPC_BUILD_TESTS)
 
     # Iterate over all the files in the arrow-testing-data source directiory. When it's a gz, extract in place.
     file(GLOB_RECURSE arrow_testing_data_targz_files CONFIGURE_DEPENDS
-        "${arrow-testing_SOURCE_DIR}/data/arrow-ipc-stream/integration/1.0.0-littleendian/*.json.gz"
+        "${arrow-testing_SOURCE_DIR}/data/arrow-ipc-stream/integration/cpp-21.0.0/*.json.gz"
     )
     foreach(file_path IN LISTS arrow_testing_data_targz_files)
             cmake_path(GET file_path PARENT_PATH parent_dir)

examples/CMakeLists.txt

@@ -8,6 +8,7 @@ target_link_libraries(write_and_read_streams
     PRIVATE
         sparrow-ipc
         sparrow::sparrow
+        arrow-testing-data 
 )
 
 # Set C++ standard to match the main project

examples/write_and_read_streams.cpp

@@ -1,5 +1,7 @@
 #include <algorithm>
 #include <cstdlib>
+#include <filesystem>
+#include <fstream>
 #include <iostream>
 #include <random>
 #include <vector>
@@ -10,6 +12,11 @@
 
 #include <sparrow/record_batch.hpp>
 
+const std::filesystem::path arrow_testing_data_dir = ARROW_TESTING_DATA_DIR;
+const std::filesystem::path tests_resources_files_path = arrow_testing_data_dir / "data" / "arrow-ipc-stream"
+                                                         / "integration" / "cpp-21.0.0";
+
+
 namespace sp = sparrow;
 
 // Random number generator
@@ -23,34 +30,26 @@ std::mt19937 gen(rd());
 sp::record_batch create_random_record_batch(size_t num_rows)
 {
     std::uniform_int_distribution<int32_t> int_dist(0, 1000);
-    std::uniform_real_distribution<float> float_dist(-100.0f, 100.0f);
-    std::uniform_int_distribution<int> bool_dist(0, 1);
+    
 
     // Create integer column with random values
     std::vector<int32_t> int_values;
     int_values.reserve(num_rows);
-    for (size_t i = 0; i < num_rows; ++i)
-    {
-        int_values.push_back(int_dist(gen));
-    }
+    std::generate_n(std::back_inserter(int_values), num_rows, [&]() { return int_dist(gen); });
     auto int_array = sp::primitive_array<int32_t>(std::move(int_values));
 
     // Create float column with random values
+    std::uniform_real_distribution<float> float_dist(-100.0f, 100.0f);
     std::vector<float> float_values;
     float_values.reserve(num_rows);
-    for (size_t i = 0; i < num_rows; ++i)
-    {
-        float_values.push_back(float_dist(gen));
-    }
+    std::generate_n(std::back_inserter(float_values), num_rows, [&]() { return float_dist(gen); });
     auto float_array = sp::primitive_array<float>(std::move(float_values));
 
     // Create boolean column with random values
+    std::uniform_int_distribution<int> bool_dist(0, 1);
     std::vector<bool> bool_values;
     bool_values.reserve(num_rows);
-    for (size_t i = 0; i < num_rows; ++i)
-    {
-        bool_values.push_back(static_cast<bool>(bool_dist(gen)));
-    }
+    std::generate_n(std::back_inserter(bool_values), num_rows, [&]() { return static_cast<bool>(bool_dist(gen)); });
     auto bool_array = sp::primitive_array<bool>(std::move(bool_values));
 
     // Create string column with random values
@@ -59,11 +58,7 @@ sp::record_batch create_random_record_batch(size_t num_rows)
     const std::vector<std::string> sample_strings =
         {"alpha", "beta", "gamma", "delta", "epsilon", "zeta", "eta", "theta", "iota", "kappa"};
     std::uniform_int_distribution<size_t> str_dist(0, sample_strings.size() - 1);
-
-    for (size_t i = 0; i < num_rows; ++i)
-    {
-        string_values.push_back(sample_strings[str_dist(gen)] + "_" + std::to_string(i));
-    }
+    std::generate_n(std::back_inserter(string_values), num_rows, [&]() { return sample_strings[str_dist(gen)] + "_" + std::to_string(string_values.size()); });
     auto string_array = sp::string_array(std::move(string_values));
 
     // Create record batch with named columns (same schema for all batches)
@@ -277,6 +272,50 @@ int main()
             std::cerr << "   ✗ Schema inconsistency detected!\n";
         }
 
+        // Step 8: Read and display a primitive stream file from test resources
+        std::cout << "\n8. Reading a primitive stream file from test resources...\n";
+
+        const std::filesystem::path primitive_stream_file = tests_resources_files_path / "generated_primitive.stream";
+        
+        if (std::filesystem::exists(primitive_stream_file))
+        {
+            std::cout << "   Reading file: " << primitive_stream_file << "\n";
+            
+            // Read the stream file
+            std::ifstream stream_file(primitive_stream_file, std::ios::in | std::ios::binary);
+            if (!stream_file.is_open())
+            {
+                std::cerr << "   ERROR: Could not open stream file!\n";
+            }
+            else
+            {
+                const std::vector<uint8_t> file_stream_data(
+                    (std::istreambuf_iterator<char>(stream_file)),
+                    (std::istreambuf_iterator<char>())
+                );
+                stream_file.close();
+                
+                std::cout << "   File size: " << file_stream_data.size() << " bytes\n";
+                
+                // Deserialize the stream
+                auto file_batches = sparrow_ipc::deserialize_stream(file_stream_data);
+                
+                std::cout << "   Deserialized " << file_batches.size() << " record batch(es) from file\n";
+                
+                // Display the first batch
+                if (!file_batches.empty())
+                {
+                    std::cout << "   First batch from file:\n";
+                    std::cout << std::format("{}\n", file_batches[0]);
+                }
+            }
+        }
+        else
+        {
+            std::cout << "   Note: Test resource file not found at " << primitive_stream_file << "\n";
+            std::cout << "   This is expected if test data is not available.\n";
+        }
+
         std::cout << "\n=== Example completed successfully! ===\n";
     }
     catch (const std::exception& e)

include/sparrow_ipc/flatbuffer_utils.hpp

@@ -88,26 +88,6 @@ namespace sparrow_ipc
         ::flatbuffers::Vector<::flatbuffers::Offset<org::apache::arrow::flatbuf::Field>>>
     create_children(flatbuffers::FlatBufferBuilder& builder, const sparrow::record_batch& record_batch);
 
-    /**
-     * @brief Creates a FlatBuffers vector of Field objects from an ArrowSchema's children.
-     *
-     * This function iterates through all children of the given ArrowSchema and converts
-     * each child to a FlatBuffers Field object. The resulting fields are collected into
-     * a FlatBuffers vector.
-     *
-     * @param builder Reference to the FlatBufferBuilder used for creating FlatBuffers objects
-     * @param arrow_schema The ArrowSchema containing the children to convert
-     *
-     * @return A FlatBuffers offset to a vector of Field objects, or 0 if no children exist
-     *
-     * @throws std::invalid_argument If any child pointer in the ArrowSchema is null
-     *
-     * @note The function reserves space for all children upfront for performance optimization
-     * @note Returns 0 (null offset) when the schema has no children, otherwise returns a valid vector offset
-     */
-    [[nodiscard]] ::flatbuffers::Offset<
-        ::flatbuffers::Vector<::flatbuffers::Offset<org::apache::arrow::flatbuf::Field>>>
-    create_children(flatbuffers::FlatBufferBuilder& builder, const sparrow::record_batch& record_batch);
 
     /**
      * @brief Creates a FlatBuffers vector of Field objects from an ArrowSchema's children.

tests/test_de_serialization_with_files.cpp

@@ -20,11 +20,11 @@
 const std::filesystem::path arrow_testing_data_dir = ARROW_TESTING_DATA_DIR;
 
 const std::filesystem::path tests_resources_files_path = arrow_testing_data_dir / "data" / "arrow-ipc-stream"
-                                                         / "integration" / "1.0.0-littleendian";
+                                                         / "integration" / "cpp-21.0.0";
 
 const std::vector<std::filesystem::path> files_paths_to_test = {
     tests_resources_files_path / "generated_primitive",
-    tests_resources_files_path / "generated_primitive_large_offsets",
+    // tests_resources_files_path / "generated_primitive_large_offsets",
     tests_resources_files_path / "generated_primitive_zerolength",
     // tests_resources_files_path / "generated_primitive_no_batches"
 };