Skip to content

Commit dd6b9cd

Browse files
Alex-PLACETAlex-PLACET
committed
fix
1 parent 8cf8396 commit dd6b9cd

File tree

4 files changed

+57
-57
lines changed

4 files changed

+57
-57
lines changed

‎integration_tests/arrow_validate.cpp‎

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,13 @@
88
#include "integration_tools.hpp"
99

1010
/**
11-
* @brief Validates that a JSON file and an Arrow stream file contain identical data.
11+
* @brief Validates that a JSON file and an Arrow arrow file contain identical data.
1212
*
1313
* This program reads a JSON file containing Arrow record batches and an Arrow IPC
14-
* stream file, converts both to vectors of record batches, and compares them
14+
* arrow file, converts both to vectors of record batches, and compares them
1515
* element-by-element to ensure they are identical.
1616
*
17-
* Usage: validate <json_file_path> <stream_file_path>
17+
* Usage: validate <json_file_path> <arrow_file_path>
1818
*
1919
* @param argc Number of command-line arguments
2020
* @param argv Array of command-line arguments
@@ -26,59 +26,59 @@ int main(int argc, char* argv[])
2626
if (argc != 3)
2727
{
2828
std::cerr << "Usage: " << argv[0] << " <json_file_path> <stream_file_path>\n";
29-
std::cerr << "Validates that a JSON file and an Arrow stream file contain identical data.\n";
29+
std::cerr << "Validates that a JSON file and an Arrow file contain identical data.\n";
3030
return EXIT_FAILURE;
3131
}
3232

3333
const std::filesystem::path json_path(argv[1]);
34-
const std::filesystem::path stream_path(argv[2]);
34+
const std::filesystem::path arrow_file_path(argv[2]);
3535

3636
try
3737
{
3838
// Check if the stream file exists
39-
if (!std::filesystem::exists(stream_path))
39+
if (!std::filesystem::exists(arrow_file_path))
4040
{
41-
std::cerr << "Error: Stream file not found: " << stream_path << "\n";
41+
std::cerr << "Error: Arrow file not found: " << arrow_file_path << "\n";
4242
return EXIT_FAILURE;
4343
}
4444

4545
std::cout << "Loading JSON file: " << json_path << "\n";
46-
std::cout << "Loading stream file: " << stream_path << "\n";
46+
std::cout << "Loading Arrow file: " << arrow_file_path << "\n";
4747

4848
// Read the stream file
49-
std::ifstream stream_file(stream_path, std::ios::in | std::ios::binary);
50-
if (!stream_file.is_open())
49+
std::ifstream arrow_file(arrow_file_path, std::ios::in | std::ios::binary);
50+
if (!arrow_file.is_open())
5151
{
52-
std::cerr << "Error: Could not open stream file: " << stream_path << "\n";
52+
std::cerr << "Error: Could not open arrow file: " << arrow_file_path << "\n";
5353
return EXIT_FAILURE;
5454
}
5555

56-
std::vector<uint8_t> stream_data(
57-
(std::istreambuf_iterator<char>(stream_file)),
56+
std::vector<uint8_t> arrow_file_data(
57+
(std::istreambuf_iterator<char>(arrow_file)),
5858
std::istreambuf_iterator<char>()
5959
);
60-
stream_file.close();
60+
arrow_file.close();
6161

62-
if (stream_data.empty())
62+
if (arrow_file_data.empty())
6363
{
64-
std::cerr << "Error: Stream file is empty.\n";
64+
std::cerr << "Error: Arrow file is empty.\n";
6565
return EXIT_FAILURE;
6666
}
6767

6868
// Validate using the library
69-
bool matches = integration_tools::validate_json_against_stream(
69+
bool matches = integration_tools::validate_json_against_arrow_file(
7070
json_path,
71-
std::span<const uint8_t>(stream_data)
71+
std::span<const uint8_t>(arrow_file_data)
7272
);
7373

7474
if (matches)
7575
{
76-
std::cout << "\n✓ Validation successful: JSON and stream files contain identical data!\n";
76+
std::cout << "\n✓ Validation successful: JSON and Arrow files contain identical data!\n";
7777
return EXIT_SUCCESS;
7878
}
7979
else
8080
{
81-
std::cerr << "\n✗ Validation failed: JSON and stream files contain different data.\n";
81+
std::cerr << "\n✗ Validation failed: JSON and Arrow files contain different data.\n";
8282
return EXIT_FAILURE;
8383
}
8484
}

‎integration_tests/include/integration_tools.hpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,15 +53,15 @@ namespace integration_tools
5353
std::vector<uint8_t> stream_to_file(std::span<const uint8_t> input_stream_data);
5454

5555
/**
56-
* @brief Validates that a JSON file and an Arrow stream contain identical data.
56+
* @brief Validates that a JSON file and an Arrow file contain identical data.
5757
*
5858
* @param json_path Path to the JSON file
59-
* @param stream_data Binary Arrow IPC stream data
59+
* @param stream_data Binary Arrow IPC file data
6060
* @return true if the data matches, false otherwise
6161
* @throws std::runtime_error on parsing or deserialization errors
6262
*/
6363
bool
64-
validate_json_against_stream(const std::filesystem::path& json_path, std::span<const uint8_t> stream_data);
64+
validate_json_against_arrow_file(const std::filesystem::path& json_path, std::span<const uint8_t> arrow_file_data);
6565

6666
/**
6767
* @brief Compares two record batches for equality.

‎integration_tests/src/integration_tools.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -245,9 +245,9 @@ namespace integration_tools
245245
return all_match;
246246
}
247247

248-
bool validate_json_against_stream(
248+
bool validate_json_against_arrow_file(
249249
const std::filesystem::path& json_path,
250-
std::span<const uint8_t> stream_data
250+
std::span<const uint8_t> arrow_file_data
251251
)
252252
{
253253
// Check if the JSON file exists
@@ -303,15 +303,15 @@ namespace integration_tools
303303
}
304304

305305
// Deserialize the stream
306-
if (stream_data.empty())
306+
if (arrow_file_data.empty())
307307
{
308308
throw std::runtime_error("Stream data is empty");
309309
}
310310

311311
std::vector<sparrow::record_batch> stream_batches;
312312
try
313313
{
314-
stream_batches = sparrow_ipc::deserialize_file(stream_data);
314+
stream_batches = sparrow_ipc::deserialize_file(arrow_file_data);
315315
}
316316
catch (const std::exception& e)
317317
{

‎integration_tests/test_integration_tools.cpp‎

Lines changed: 30 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#include "doctest/doctest.h"
1212
#include "integration_tools.hpp"
1313
#include "sparrow_ipc/deserialize.hpp"
14+
#include "sparrow_ipc/stream_file_serializer.hpp"
1415

1516
TEST_SUITE("Integration Tools Tests")
1617
{
@@ -31,12 +32,12 @@ TEST_SUITE("Integration Tools Tests")
3132
CHECK_THROWS_AS(integration_tools::stream_to_file(std::span<const uint8_t>(empty_data)), std::runtime_error);
3233
}
3334

34-
TEST_CASE("validate_json_against_stream - Non-existent JSON file")
35+
TEST_CASE("validate_json_against_arrow_file - Non-existent JSON file")
3536
{
3637
const std::filesystem::path non_existent = "non_existent_file_12345.json";
3738
std::vector<uint8_t> dummy_stream = {1, 2, 3};
3839
CHECK_THROWS_AS(
39-
integration_tools::validate_json_against_stream(non_existent, std::span<const uint8_t>(dummy_stream)),
40+
integration_tools::validate_json_against_arrow_file(non_existent, std::span<const uint8_t>(dummy_stream)),
4041
std::runtime_error
4142
);
4243
}
@@ -76,7 +77,7 @@ TEST_SUITE("Integration Tools Tests")
7677
std::ifstream stream_file(input_stream, std::ios::binary);
7778
REQUIRE(stream_file.is_open());
7879

79-
std::vector<uint8_t> input_data(
80+
const std::vector<uint8_t> input_data(
8081
(std::istreambuf_iterator<char>(stream_file)),
8182
std::istreambuf_iterator<char>()
8283
);
@@ -88,7 +89,7 @@ TEST_SUITE("Integration Tools Tests")
8889
CHECK_GT(output_data.size(), 0);
8990

9091
// Verify the output is valid
91-
auto batches = sparrow_ipc::deserialize_stream(std::span<const uint8_t>(output_data));
92+
const auto batches = sparrow_ipc::deserialize_file(std::span<const uint8_t>(output_data));
9293
CHECK_GT(batches.size(), 0);
9394
}
9495

@@ -103,16 +104,16 @@ TEST_SUITE("Integration Tools Tests")
103104
}
104105

105106
// Step 1: JSON -> stream
106-
std::vector<uint8_t> stream_data = integration_tools::json_file_to_stream(json_file);
107+
const std::vector<uint8_t> stream_data = integration_tools::json_file_to_stream(json_file);
107108
REQUIRE_GT(stream_data.size(), 0);
108109

109110
// Step 2: stream -> file
110-
std::vector<uint8_t> file_data = integration_tools::stream_to_file(std::span<const uint8_t>(stream_data));
111+
const std::vector<uint8_t> file_data = integration_tools::stream_to_file(std::span<const uint8_t>(stream_data));
111112
REQUIRE_GT(file_data.size(), 0);
112113

113114
// Step 3: Compare the results - both should deserialize to same data
114-
auto stream_batches = sparrow_ipc::deserialize_stream(std::span<const uint8_t>(stream_data));
115-
auto file_batches = sparrow_ipc::deserialize_stream(std::span<const uint8_t>(file_data));
115+
const auto stream_batches = sparrow_ipc::deserialize_stream(std::span<const uint8_t>(stream_data));
116+
const auto file_batches = sparrow_ipc::deserialize_file(std::span<const uint8_t>(file_data));
116117

117118
REQUIRE_EQ(stream_batches.size(), file_batches.size());
118119
for (size_t i = 0; i < stream_batches.size(); ++i)
@@ -121,7 +122,7 @@ TEST_SUITE("Integration Tools Tests")
121122
}
122123
}
123124

124-
TEST_CASE("validate_json_against_stream - Successful validation")
125+
TEST_CASE("validate_json_against_arrow_file - Successful validation")
125126
{
126127
const std::filesystem::path json_file = tests_resources_files_path / "generated_primitive.json";
127128

@@ -131,42 +132,41 @@ TEST_SUITE("Integration Tools Tests")
131132
return;
132133
}
133134

134-
// Convert JSON to stream
135-
std::vector<uint8_t> stream_data = integration_tools::json_file_to_stream(json_file);
135+
const std::vector<uint8_t> arrow_file_data = integration_tools::json_file_to_arrow_file(json_file);
136136

137137
// Validate
138-
bool matches = integration_tools::validate_json_against_stream(
138+
const bool matches = integration_tools::validate_json_against_arrow_file(
139139
json_file,
140-
std::span<const uint8_t>(stream_data)
140+
std::span<const uint8_t>(arrow_file_data)
141141
);
142142
CHECK(matches);
143143
}
144144

145-
TEST_CASE("validate_json_against_stream - With reference stream file")
145+
TEST_CASE("validate_json_against_arrow_file - With reference stream file")
146146
{
147147
const std::filesystem::path json_file = tests_resources_files_path / "generated_primitive.json";
148-
const std::filesystem::path stream_file = tests_resources_files_path / "generated_primitive.stream";
148+
const std::filesystem::path arrow_file = tests_resources_files_path / "generated_primitive.arrow_file";
149149

150-
if (!std::filesystem::exists(json_file) || !std::filesystem::exists(stream_file))
150+
if (!std::filesystem::exists(json_file) || !std::filesystem::exists(arrow_file))
151151
{
152152
MESSAGE("Skipping test: test file(s) not found");
153153
return;
154154
}
155155

156156
// Read the stream file
157-
std::ifstream stream_input(stream_file, std::ios::binary);
157+
std::ifstream stream_input(arrow_file, std::ios::binary);
158158
REQUIRE(stream_input.is_open());
159159

160-
std::vector<uint8_t> stream_data(
160+
const std::vector<uint8_t> arrow_file_data(
161161
(std::istreambuf_iterator<char>(stream_input)),
162162
std::istreambuf_iterator<char>()
163163
);
164164
stream_input.close();
165165

166166
// Validate
167-
bool matches = integration_tools::validate_json_against_stream(
167+
bool matches = integration_tools::validate_json_against_arrow_file(
168168
json_file,
169-
std::span<const uint8_t>(stream_data)
169+
std::span<const uint8_t>(arrow_file_data)
170170
);
171171
CHECK(matches);
172172
}
@@ -182,18 +182,18 @@ TEST_SUITE("Integration Tools Tests")
182182
}
183183

184184
// Convert JSON to stream
185-
std::vector<uint8_t> stream_data = integration_tools::json_file_to_stream(json_file);
186-
REQUIRE_GT(stream_data.size(), 0);
185+
const std::vector<uint8_t> arrow_file_data = integration_tools::json_file_to_arrow_file(json_file);
186+
REQUIRE_GT(arrow_file_data.size(), 0);
187187

188188
// Validate that the stream matches the JSON
189-
bool matches = integration_tools::validate_json_against_stream(
189+
const bool matches = integration_tools::validate_json_against_arrow_file(
190190
json_file,
191-
std::span<const uint8_t>(stream_data)
191+
std::span<const uint8_t>(arrow_file_data)
192192
);
193193
CHECK(matches);
194194

195195
// Also verify by deserializing
196-
auto batches = sparrow_ipc::deserialize_stream(std::span<const uint8_t>(stream_data));
196+
auto batches = sparrow_ipc::deserialize_file(std::span<const uint8_t>(arrow_file_data));
197197
CHECK_GT(batches.size(), 0);
198198
}
199199

@@ -244,18 +244,18 @@ TEST_SUITE("Integration Tools Tests")
244244
SUBCASE(filename.c_str())
245245
{
246246
// Convert to stream
247-
std::vector<uint8_t> stream_data = integration_tools::json_file_to_stream(json_file);
248-
REQUIRE_GT(stream_data.size(), 0);
247+
const std::vector<uint8_t> arrow_file_data = integration_tools::json_file_to_arrow_file(json_file);
248+
REQUIRE_GT(arrow_file_data.size(), 0);
249249

250250
// Validate
251-
bool matches = integration_tools::validate_json_against_stream(
251+
const bool matches = integration_tools::validate_json_against_arrow_file(
252252
json_file,
253-
std::span<const uint8_t>(stream_data)
253+
std::span<const uint8_t>(arrow_file_data)
254254
);
255255
CHECK(matches);
256256

257257
// Verify we can deserialize
258-
auto batches = sparrow_ipc::deserialize_stream(std::span<const uint8_t>(stream_data));
258+
const auto batches = sparrow_ipc::deserialize_file(std::span<const uint8_t>(arrow_file_data));
259259
CHECK_GE(batches.size(), 0);
260260
}
261261
}

0 commit comments

Comments
 (0)