wip

Author: Alex-PLACET

integration_tests/arrow_file_to_stream.cpp

@@ -4,10 +4,7 @@
 #include <iostream>
 #include <vector>
 
-#include <sparrow_ipc/deserialize.hpp>
-#include <sparrow_ipc/memory_output_stream.hpp>
-#include <sparrow_ipc/serializer.hpp>
-#include <sparrow_ipc/stream_file_serializer.hpp>
+#include "integration_tools.hpp"
 
 /**
  * @brief Reads an Arrow IPC file and outputs the serialized Arrow IPC stream to a file.
@@ -36,37 +33,52 @@ int main(int argc, char* argv[])
 
     try
     {
-        // Read the Arrow file
+        if (!std::filesystem::exists(input_path))
+        {
+            std::cerr << "Error: Input file not found: " << input_path << "\n";
+            return EXIT_FAILURE;
+        }
+
         std::ifstream input_file(input_path, std::ios::binary);
-        if (!input_file)
+        if (!input_file.is_open())
         {
             std::cerr << "Error: Could not open input file: " << input_path << "\n";
             return EXIT_FAILURE;
         }
-        
+
         const std::vector<uint8_t> file_data(
             (std::istreambuf_iterator<char>(input_file)),
-            (std::istreambuf_iterator<char>())
+            std::istreambuf_iterator<char>()
         );
         input_file.close();
 
-        const auto batches = sparrow_ipc::deserialize_file(file_data);
-        
-        std::vector<uint8_t> stream_data;
-        sparrow_ipc::memory_output_stream mem_stream(stream_data);
-        sparrow_ipc::serializer serializer(mem_stream);
-        serializer << batches << sparrow_ipc::end_stream;
-        
+        if (file_data.empty())
+        {
+            std::cerr << "Error: Input file is empty.\n";
+            return EXIT_FAILURE;
+        }
+
+        const std::vector<uint8_t> stream_data = integration_tools::file_to_stream(file_data);
+
         std::ofstream output_file(output_path, std::ios::binary);
-        if (!output_file)
+        if (!output_file.is_open())
         {
             std::cerr << "Error: Could not open output file: " << output_path << "\n";
             return EXIT_FAILURE;
         }
-        
-        output_file.write(reinterpret_cast<const char*>(stream_data.data()), static_cast<std::streamsize>(stream_data.size()));
+
+        output_file.write(
+            reinterpret_cast<const char*>(stream_data.data()),
+            static_cast<std::streamsize>(stream_data.size())
+        );
         output_file.close();
 
+        if (!output_file.good())
+        {
+            std::cerr << "Error: Failed to write to output file: " << output_path << "\n";
+            return EXIT_FAILURE;
+        }
+
         return EXIT_SUCCESS;
     }
     catch (const std::exception& e)

integration_tests/arrow_json_to_file.cpp

@@ -34,14 +34,15 @@ int main(int argc, char* argv[])
     try
     {
         const std::vector<uint8_t> file_data = integration_tools::json_file_to_arrow_file(json_path);
+
         std::ofstream output_file(output_path, std::ios::out | std::ios::binary);
         if (!output_file.is_open())
         {
             std::cerr << "Error: Could not open output file: " << output_path << "\n";
             return EXIT_FAILURE;
         }
 
-        output_file.write(reinterpret_cast<const char*>(file_data.data()), file_data.size());
+        output_file.write(reinterpret_cast<const char*>(file_data.data()), static_cast<std::streamsize>(file_data.size()));
         output_file.close();
 
         if (!output_file.good())

integration_tests/arrow_stream_to_file.cpp

@@ -2,7 +2,6 @@
 #include <filesystem>
 #include <fstream>
 #include <iostream>
-#include <iterator>
 #include <vector>
 
 #include "integration_tools.hpp"
@@ -22,7 +21,6 @@
  */
 int main(int argc, char* argv[])
 {
-    // Check command-line arguments
     if (argc != 3)
     {
         std::cerr << "Usage: " << argv[0] << " <input_file_path> <output_file_path>\n";
@@ -60,9 +58,7 @@ int main(int argc, char* argv[])
             return EXIT_FAILURE;
         }
 
-        const std::vector<uint8_t> output_file_data = integration_tools::stream_to_file(
-            std::span<const uint8_t>(input_stream_data)
-        );
+        const std::vector<uint8_t> output_file_data = integration_tools::stream_to_file(input_stream_data);
 
         std::ofstream output_file(output_path, std::ios::out | std::ios::binary);
         if (!output_file.is_open())
@@ -71,7 +67,10 @@ int main(int argc, char* argv[])
             return EXIT_FAILURE;
         }
 
-        output_file.write(reinterpret_cast<const char*>(output_file_data.data()), output_file_data.size());
+        output_file.write(
+            reinterpret_cast<const char*>(output_file_data.data()),
+            static_cast<std::streamsize>(output_file_data.size())
+        );
         output_file.close();
 
         if (!output_file.good())

integration_tests/include/integration_tools.hpp

@@ -52,6 +52,15 @@ namespace integration_tools
      */
     std::vector<uint8_t> stream_to_file(std::span<const uint8_t> input_stream_data);
 
+    /**
+     * @brief Reads an Arrow IPC file and re-serializes it to stream format.
+     *
+     * @param input_file_data Binary Arrow IPC file data
+     * @return Vector of bytes containing the re-serialized Arrow IPC stream format
+     * @throws std::runtime_error if the file cannot be deserialized
+     */
+    std::vector<uint8_t> file_to_stream(std::span<const uint8_t> input_file_data);
+
     /**
      * @brief Validates that a JSON file and an Arrow file contain identical data.
      *

integration_tests/src/integration_tools.cpp

@@ -112,6 +112,31 @@ namespace integration_tools
         return output_stream_data;
     }
 
+    std::vector<uint8_t> file_to_stream(std::span<const uint8_t> input_file_data)
+    {
+        if (input_file_data.empty())
+        {
+            throw std::runtime_error("Input file data is empty");
+        }
+
+        std::vector<sparrow::record_batch> record_batches;
+        try
+        {
+            record_batches = sparrow_ipc::deserialize_file(input_file_data);
+        }
+        catch (const std::exception& e)
+        {
+            throw std::runtime_error("Failed to deserialize file: " + std::string(e.what()));
+        }
+
+        std::vector<uint8_t> output_stream_data;
+        sparrow_ipc::memory_output_stream stream(output_stream_data);
+        sparrow_ipc::serializer serializer(stream);
+        serializer << record_batches << sparrow_ipc::end_stream;
+
+        return output_stream_data;
+    }
+
     bool compare_record_batch(
         const sparrow::record_batch& rb1,
         const sparrow::record_batch& rb2,

integration_tests/test_integration_tools.cpp

@@ -184,7 +184,7 @@ TEST_SUITE("Integration Tools Tests")
 
         REQUIRE(json_data.contains("batches"));
         const size_t expected_batch_count = json_data["batches"].size();
-        REQUIRE_GT(expected_batch_count, 0);
+        REQUIRE_EQ(expected_batch_count, 2);
 
         // Step 1: JSON -> Arrow file
         const std::vector<uint8_t> arrow_file_data = integration_tools::json_file_to_arrow_file(json_file);

src/stream_file_serializer.cpp

@@ -75,11 +75,11 @@ namespace sparrow_ipc
         const auto fields_vec = create_children(footer_builder, record_batch);
         const auto schema_offset = org::apache::arrow::flatbuf::CreateSchema(
             footer_builder,
-            org::apache::arrow::flatbuf::Endianness::Little,
+            org::apache::arrow::flatbuf::Endianness::Little, // TODO: make configurable
             fields_vec
         );
 
-        // Create empty dictionaries vector
+        // Create empty dictionaries vector // TODO: Support dictionaries if needed
         auto dictionaries_fb = footer_builder.CreateVectorOfStructs(
             std::vector<org::apache::arrow::flatbuf::Block>{}
         );