wip

Author: Alex-PLACET

integration_tests/arrow_json_to_file.cpp

@@ -35,7 +35,7 @@ int main(int argc, char* argv[])
     try
     {
         // Convert JSON file to stream using the library
-        std::vector<uint8_t> stream_data = integration_tools::json_file_to_stream(json_path);
+        std::vector<uint8_t> stream_data = integration_tools::json_file_to_arrow_file(json_path);
 
         // Write the binary stream to the output file
         std::ofstream output_file(output_path, std::ios::out | std::ios::binary);

integration_tests/arrow_stream_to_file.cpp

@@ -26,7 +26,7 @@ int main(int argc, char* argv[])
     if (argc != 3)
     {
         std::cerr << "Usage: " << argv[0] << " <input_file_path> <output_file_path>\n";
-        std::cerr << "Reads an Arrow IPC stream from a file and writes it to another file.\n";
+        std::cerr << "Reads an Arrow IPC stream from a file and writes it to an Arrow file.\n";
         return EXIT_FAILURE;
     }
 

integration_tests/include/integration_tools.hpp

@@ -5,10 +5,26 @@
 #include <vector>
 
 #include <nlohmann/json.hpp>
+
 #include <sparrow/record_batch.hpp>
 
 namespace integration_tools
 {
+    /**
+     * @brief Converts a JSON file to Arrow IPC file format.
+     *
+     * Reads a JSON file from the specified path and converts its contents into
+     * Apache Arrow IPC (Inter-Process Communication) file format, returning the
+     * serialized Arrow data as a byte vector.
+     *
+     * @param json_path The filesystem path to the input JSON file to be converted.
+     * @return std::vector<uint8_t> A byte vector containing the Arrow IPC file data.
+     *
+     * @throws std::filesystem::filesystem_error If the file cannot be accessed or read.
+     * @throws std::runtime_error If the JSON parsing or Arrow conversion fails.
+     */
+    std::vector<uint8_t> json_file_to_arrow_file(const std::filesystem::path& json_path);
+
     /**
      * @brief Reads a JSON file and converts it to Arrow IPC stream format.
      *
@@ -44,10 +60,8 @@ namespace integration_tools
      * @return true if the data matches, false otherwise
      * @throws std::runtime_error on parsing or deserialization errors
      */
-    bool validate_json_against_stream(
-        const std::filesystem::path& json_path,
-        std::span<const uint8_t> stream_data
-    );
+    bool
+    validate_json_against_stream(const std::filesystem::path& json_path, std::span<const uint8_t> stream_data);
 
     /**
      * @brief Compares two record batches for equality.

integration_tests/src/integration_tools.cpp

@@ -4,6 +4,7 @@
 #include <iostream>
 #include <iterator>
 #include <sstream>
+#include "sparrow_ipc/stream_file_serializer.hpp"
 
 #if defined(__cpp_lib_format)
 #    include <format>
@@ -17,6 +18,15 @@
 
 namespace integration_tools
 {
+    std::vector<uint8_t> json_file_to_arrow_file(const std::filesystem::path& json_path)
+    {
+        // Convert JSON file to stream first
+        std::vector<uint8_t> stream_data = json_file_to_stream(json_path);
+
+        // Then convert stream to file format
+        return stream_to_file(std::span<const uint8_t>(stream_data));
+    }
+
     std::vector<uint8_t> json_file_to_stream(const std::filesystem::path& json_path)
     {
         // Check if the JSON file exists
@@ -106,8 +116,8 @@ namespace integration_tools
         // Re-serialize the record batches to ensure a valid output stream
         std::vector<uint8_t> output_stream_data;
         sparrow_ipc::memory_output_stream stream(output_stream_data);
-        sparrow_ipc::serializer serializer(stream);
-        serializer << record_batches << sparrow_ipc::end_stream;
+        sparrow_ipc::stream_file_serializer serializer(stream);
+        serializer << record_batches << sparrow_ipc::end_file;
 
         return output_stream_data;
     }