From e95ab9af3b10a348b9dbfa11c40f823bfb820e53 Mon Sep 17 00:00:00 2001 From: Hind Montassif Date: Fri, 5 Dec 2025 11:13:02 +0100 Subject: [PATCH 1/8] Add commented fix to check failure --- src/serialize_utils.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/serialize_utils.cpp b/src/serialize_utils.cpp index 45ab970..1fff3e9 100644 --- a/src/serialize_utils.cpp +++ b/src/serialize_utils.cpp @@ -93,6 +93,10 @@ namespace sparrow_ipc return col.data_type(); } ); +// for (const auto& col : rb.columns()) +// { +// dtypes.push_back(col.data_type()); +// } return dtypes; } } From c85ac857d7c9603c7530abc7736c33dff1a39d9f Mon Sep 17 00:00:00 2001 From: Hind Montassif Date: Fri, 5 Dec 2025 11:19:23 +0100 Subject: [PATCH 2/8] Uncomment fix --- src/serialize_utils.cpp | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/serialize_utils.cpp b/src/serialize_utils.cpp index 1fff3e9..6fa824e 100644 --- a/src/serialize_utils.cpp +++ b/src/serialize_utils.cpp @@ -85,18 +85,18 @@ namespace sparrow_ipc { std::vector dtypes; dtypes.reserve(rb.nb_columns()); - std::ranges::transform( - rb.columns(), - std::back_inserter(dtypes), - [](const auto& col) - { - return col.data_type(); - } - ); -// for (const auto& col : rb.columns()) -// { -// dtypes.push_back(col.data_type()); -// } +// std::ranges::transform( +// rb.columns(), +// std::back_inserter(dtypes), +// [](const auto& col) +// { +// return col.data_type(); +// } +// ); + for (const auto& col : rb.columns()) + { + dtypes.push_back(col.data_type()); + } return dtypes; } } From 6bae15f8a0d133e2af43948c96abc322d8537063 Mon Sep 17 00:00:00 2001 From: Hind Montassif Date: Fri, 5 Dec 2025 11:33:42 +0100 Subject: [PATCH 3/8] Attempt on metadata.cpp --- src/metadata.cpp | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/metadata.cpp b/src/metadata.cpp index 3638f76..1b2ceea 100644 --- a/src/metadata.cpp +++ b/src/metadata.cpp @@ -10,14 +10,10 @@ namespace sparrow_ipc { std::vector sparrow_metadata; sparrow_metadata.reserve(metadata.size()); - std::ranges::transform( - metadata, - std::back_inserter(sparrow_metadata), - [](const auto& kv) - { - return sparrow::metadata_pair{kv->key()->str(), kv->value()->str()}; - } - ); + for (const auto& kv : metadata) + { + sparrow_metadata.emplace_back(kv->key()->str(), kv->value()->str()); + } return sparrow_metadata; } } \ No newline at end of file From 2608881a6e02a85d5db2e74ca1226ee7b13fd1e6 Mon Sep 17 00:00:00 2001 From: Hind Montassif Date: Fri, 5 Dec 2025 11:48:03 +0100 Subject: [PATCH 4/8] Attempt on equal --- include/sparrow_ipc/magic_values.hpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/include/sparrow_ipc/magic_values.hpp b/include/sparrow_ipc/magic_values.hpp index e90187c..f97240e 100644 --- a/include/sparrow_ipc/magic_values.hpp +++ b/include/sparrow_ipc/magic_values.hpp @@ -36,13 +36,21 @@ namespace sparrow_ipc template [[nodiscard]] bool is_continuation(const R& buf) { - return std::ranges::equal(buf, continuation); + if (std::ranges::size(buf) != continuation.size()) + { + return false; + } + return std::equal(std::ranges::begin(buf), std::ranges::end(buf), continuation.begin()); } template [[nodiscard]] bool is_end_of_stream(const R& buf) { - return std::ranges::equal(buf, end_of_stream); + if (std::ranges::size(buf) != end_of_stream.size()) + { + return false; + } + return std::equal(std::ranges::begin(buf), std::ranges::end(buf), end_of_stream.begin()); } template From cdd45610f8479ec65a9546a80b8c931340887b21 Mon Sep 17 00:00:00 2001 From: Hind Montassif Date: Fri, 5 Dec 2025 11:57:42 +0100 Subject: [PATCH 5/8] Try debug without junit report --- .github/workflows/windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index f695e56..743f424 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -55,7 +55,7 @@ jobs: - name: Run tests working-directory: build run: | - cmake --build . --config ${{ matrix.build_type }} --target run_tests_with_junit_report + cmake --build . --config ${{ matrix.build_type }} --target run_tests - name: Build example working-directory: build From 74e0d210263ac71ac12ce6ecb1fff2ef519964dc Mon Sep 17 00:00:00 2001 From: Hind Montassif Date: Fri, 5 Dec 2025 13:08:34 +0100 Subject: [PATCH 6/8] Debug --- .github/workflows/windows.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 743f424..ea44e24 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -53,9 +53,18 @@ jobs: run: cmake --build . --config ${{ matrix.build_type }} --target test_sparrow_ipc_lib - name: Run tests + id: run_tests_step working-directory: build run: | cmake --build . --config ${{ matrix.build_type }} --target run_tests + continue-on-error: true + + - name: Display test log on failure + if: steps.run_tests_step.outcome == 'failure' + working-directory: build + run: | + echo "Test run failed. Displaying log file:" + cat ./tests/Testing/Temporary/LastTest.log - name: Build example working-directory: build From aa81bd7ec0864af1768f9da2318b969aac0a58ee Mon Sep 17 00:00:00 2001 From: Hind Montassif Date: Fri, 5 Dec 2025 14:18:00 +0100 Subject: [PATCH 7/8] Another try --- include/sparrow_ipc/flatbuffer_utils.hpp | 3 ++- src/flatbuffer_utils.cpp | 31 ++++++++++++------------ src/serialize_utils.cpp | 18 +++++--------- src/serializer.cpp | 4 +-- 4 files changed, 25 insertions(+), 31 deletions(-) diff --git a/include/sparrow_ipc/flatbuffer_utils.hpp b/include/sparrow_ipc/flatbuffer_utils.hpp index 592c006..4b3e42f 100644 --- a/include/sparrow_ipc/flatbuffer_utils.hpp +++ b/include/sparrow_ipc/flatbuffer_utils.hpp @@ -200,8 +200,9 @@ namespace sparrow_ipc { std::vector buffers; int64_t offset = 0; - for (const auto& column : record_batch.columns()) + for (size_t i = 0; i < record_batch.nb_columns(); ++i) { + const auto& column = record_batch.get_column(i); const auto& arrow_proxy = sparrow::detail::array_access::get_arrow_proxy(column); fill_buffers_func(arrow_proxy, buffers, offset); } diff --git a/src/flatbuffer_utils.cpp b/src/flatbuffer_utils.cpp index 4c4e79a..ec453c9 100644 --- a/src/flatbuffer_utils.cpp +++ b/src/flatbuffer_utils.cpp @@ -468,13 +468,13 @@ namespace sparrow_ipc ::flatbuffers::Offset<::flatbuffers::Vector<::flatbuffers::Offset>> create_children(flatbuffers::FlatBufferBuilder& builder, const sparrow::record_batch& record_batch) { - const auto& columns = record_batch.columns(); std::vector> children_vec; - children_vec.reserve(columns.size()); + children_vec.reserve(record_batch.nb_columns()); const auto names = record_batch.names(); - for (size_t i = 0; i < columns.size(); ++i) + for (size_t i = 0; i < record_batch.nb_columns(); ++i) { - const auto& arrow_schema = sparrow::detail::array_access::get_arrow_proxy(columns[i]).schema(); + const auto& column = record_batch.get_column(i); + const auto& arrow_schema = sparrow::detail::array_access::get_arrow_proxy(column).schema(); flatbuffers::Offset field = create_field( builder, arrow_schema, @@ -523,9 +523,10 @@ namespace sparrow_ipc create_fieldnodes(const sparrow::record_batch& record_batch) { std::vector nodes; - nodes.reserve(record_batch.columns().size()); - for (const auto& column : record_batch.columns()) + nodes.reserve(record_batch.nb_columns()); + for (size_t i = 0; i < record_batch.nb_columns(); ++i) { + const auto& column = record_batch.get_column(i); fill_fieldnodes(sparrow::detail::array_access::get_arrow_proxy(column), nodes); } return nodes; @@ -608,16 +609,14 @@ namespace sparrow_ipc std::optional compression, std::optional> cache) { - return std::accumulate( - record_batch.columns().begin(), - record_batch.columns().end(), - int64_t{0}, - [&](int64_t acc, const sparrow::array& arr) - { - const auto& arrow_proxy = sparrow::detail::array_access::get_arrow_proxy(arr); - return acc + calculate_body_size(arrow_proxy, compression, cache); - } - ); + int64_t acc = 0; + for (size_t i = 0; i < record_batch.nb_columns(); ++i) + { + const auto& arr = record_batch.get_column(i); + const auto& arrow_proxy = sparrow::detail::array_access::get_arrow_proxy(arr); + acc += calculate_body_size(arrow_proxy, compression, cache); + } + return acc; } flatbuffers::FlatBufferBuilder get_record_batch_message_builder(const sparrow::record_batch& record_batch, diff --git a/src/serialize_utils.cpp b/src/serialize_utils.cpp index 6fa824e..1efa864 100644 --- a/src/serialize_utils.cpp +++ b/src/serialize_utils.cpp @@ -38,10 +38,12 @@ namespace sparrow_ipc std::optional compression, std::optional> cache) { - std::for_each(record_batch.columns().begin(), record_batch.columns().end(), [&](const auto& column) { + for (size_t i = 0; i < record_batch.nb_columns(); ++i) + { + const auto& column = record_batch.get_column(i); const auto& arrow_proxy = sparrow::detail::array_access::get_arrow_proxy(column); fill_body(arrow_proxy, stream, compression, cache); - }); + } } std::size_t calculate_schema_message_size(const sparrow::record_batch& record_batch) @@ -85,17 +87,9 @@ namespace sparrow_ipc { std::vector dtypes; dtypes.reserve(rb.nb_columns()); -// std::ranges::transform( -// rb.columns(), -// std::back_inserter(dtypes), -// [](const auto& col) -// { -// return col.data_type(); -// } -// ); - for (const auto& col : rb.columns()) + for (size_t i = 0; i < rb.nb_columns(); ++i) { - dtypes.push_back(col.data_type()); + dtypes.push_back(rb.get_column(i).data_type()); } return dtypes; } diff --git a/src/serializer.cpp b/src/serializer.cpp index b12459c..0b222bb 100644 --- a/src/serializer.cpp +++ b/src/serializer.cpp @@ -23,9 +23,9 @@ namespace sparrow_ipc { std::vector dtypes; dtypes.reserve(rb.nb_columns()); - for (const auto& col : rb.columns()) + for (size_t i = 0; i < rb.nb_columns(); ++i) { - dtypes.push_back(col.data_type()); + dtypes.push_back(rb.get_column(i).data_type()); } return dtypes; } From 4904b0efd8df718d23c00adba6a7ffa699bcdf54 Mon Sep 17 00:00:00 2001 From: Hind Montassif Date: Fri, 5 Dec 2025 16:18:56 +0100 Subject: [PATCH 8/8] Remove unnecessary fixes --- .github/workflows/windows.yml | 11 +---------- include/sparrow_ipc/magic_values.hpp | 12 ++---------- src/metadata.cpp | 14 +++++++++----- 3 files changed, 12 insertions(+), 25 deletions(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index ea44e24..f695e56 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -53,18 +53,9 @@ jobs: run: cmake --build . --config ${{ matrix.build_type }} --target test_sparrow_ipc_lib - name: Run tests - id: run_tests_step working-directory: build run: | - cmake --build . --config ${{ matrix.build_type }} --target run_tests - continue-on-error: true - - - name: Display test log on failure - if: steps.run_tests_step.outcome == 'failure' - working-directory: build - run: | - echo "Test run failed. Displaying log file:" - cat ./tests/Testing/Temporary/LastTest.log + cmake --build . --config ${{ matrix.build_type }} --target run_tests_with_junit_report - name: Build example working-directory: build diff --git a/include/sparrow_ipc/magic_values.hpp b/include/sparrow_ipc/magic_values.hpp index f97240e..e90187c 100644 --- a/include/sparrow_ipc/magic_values.hpp +++ b/include/sparrow_ipc/magic_values.hpp @@ -36,21 +36,13 @@ namespace sparrow_ipc template [[nodiscard]] bool is_continuation(const R& buf) { - if (std::ranges::size(buf) != continuation.size()) - { - return false; - } - return std::equal(std::ranges::begin(buf), std::ranges::end(buf), continuation.begin()); + return std::ranges::equal(buf, continuation); } template [[nodiscard]] bool is_end_of_stream(const R& buf) { - if (std::ranges::size(buf) != end_of_stream.size()) - { - return false; - } - return std::equal(std::ranges::begin(buf), std::ranges::end(buf), end_of_stream.begin()); + return std::ranges::equal(buf, end_of_stream); } template diff --git a/src/metadata.cpp b/src/metadata.cpp index 1b2ceea..699229b 100644 --- a/src/metadata.cpp +++ b/src/metadata.cpp @@ -10,10 +10,14 @@ namespace sparrow_ipc { std::vector sparrow_metadata; sparrow_metadata.reserve(metadata.size()); - for (const auto& kv : metadata) - { - sparrow_metadata.emplace_back(kv->key()->str(), kv->value()->str()); - } + std::ranges::transform( + metadata, + std::back_inserter(sparrow_metadata), + [](const auto& kv) + { + return sparrow::metadata_pair{kv->key()->str(), kv->value()->str()}; + } + ); return sparrow_metadata; } -} \ No newline at end of file +}