-
Notifications
You must be signed in to change notification settings - Fork 4
Rework deserializing #15
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from 12 commits
Commits
Show all changes
22 commits
Select commit
Hold shift + click to select a range
df92795
Rework serialization
Alex-PLACET 190af27
wip
Alex-PLACET c6f0202
wip
Alex-PLACET 620ea81
wip
Alex-PLACET 9fd39b8
Fix osx build
Alex-PLACET 5b779ba
fix compilation
Alex-PLACET 9c32fc0
wip
Alex-PLACET 1c1cd18
compilation fix
Alex-PLACET 2db5795
wip
Alex-PLACET e618b0b
Use std::span
Alex-PLACET 4b8cf6c
wip
Alex-PLACET db55113
wip
Alex-PLACET 9294c41
fix
Alex-PLACET 6359136
wip
Alex-PLACET b6734ad
Remove serialization
Alex-PLACET c36b7de
wip
Alex-PLACET 0ae315d
Avoid recreating metadata
Alex-PLACET b94eea7
address comments
Alex-PLACET 115d3a9
Update conda env
Alex-PLACET e4bb2e9
TRY FIX
Alex-PLACET dd01882
Upgrade sparrow version
Alex-PLACET 02d7322
Fix windows run tests
Alex-PLACET File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| # sparrow-ipc cmake module | ||
| # This module sets the following variables in your project:: | ||
| # | ||
| # sparrow-ipc_FOUND - true if sparrow-ipc found on the system | ||
| # sparrow-ipc_INCLUDE_DIRS - the directory containing sparrow-ipc headers | ||
| # sparrow-ipc_LIBRARY - empty | ||
|
|
||
| @PACKAGE_INIT@ | ||
|
|
||
| include(CMakeFindDependencyMacro) | ||
|
|
||
| if("@USE_DATE_POLYFILL@") | ||
| find_dependency(date) | ||
| endif() | ||
|
|
||
| if("@CREATE_JSON_READER_TARGET@") | ||
| find_dependency(nlohmann_json) | ||
| endif() | ||
|
|
||
| find_dependency(sparrow) | ||
| find_dependency(FlatBuffers) | ||
|
|
||
| if(NOT TARGET sparrow-ipc::sparrow-ipc) | ||
| include("${CMAKE_CURRENT_LIST_DIR}/@[email protected]") | ||
| get_target_property(@PROJECT_NAME@_INCLUDE_DIRS sparrow-ipc::sparrow-ipc INTERFACE_INCLUDE_DIRECTORIES) | ||
| get_target_property(@PROJECT_NAME@_LIBRARY sparrow-ipc::sparrow-ipc LOCATION) | ||
| endif() |
This file was deleted.
Oops, something went wrong.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
|
|
||
| #pragma once | ||
|
|
||
| #include <vector> | ||
|
|
||
| #include <sparrow/c_interface.hpp> | ||
|
|
||
| #include "sparrow_ipc/config/config.hpp" | ||
|
|
||
| namespace sparrow_ipc | ||
| { | ||
| [[nodiscard]] SPARROW_IPC_API ArrowArray make_non_owning_arrow_array( | ||
| int64_t length, | ||
| int64_t null_count, | ||
| int64_t offset, | ||
| std::vector<std::uint8_t*>&& buffers, | ||
| size_t children_count, | ||
| ArrowArray** children, | ||
| ArrowArray* dictionary | ||
| ); | ||
|
|
||
| SPARROW_IPC_API void release_non_owning_arrow_array(ArrowArray* array); | ||
|
|
||
| SPARROW_IPC_API void fill_non_owning_arrow_array( | ||
| ArrowArray& array, | ||
| int64_t length, | ||
| int64_t null_count, | ||
| int64_t offset, | ||
| std::vector<std::uint8_t*>&& buffers, | ||
| size_t children_count, | ||
| ArrowArray** children, | ||
| ArrowArray* dictionary | ||
| ); | ||
| } |
25 changes: 25 additions & 0 deletions
25
include/sparrow_ipc/arrow_interface/arrow_array/private_data.hpp
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| #pragma once | ||
|
|
||
| #include <cstdint> | ||
| #include <vector> | ||
|
|
||
| #include "sparrow_ipc/config/config.hpp" | ||
|
|
||
| namespace sparrow_ipc | ||
| { | ||
| class non_owning_arrow_array_private_data | ||
| { | ||
| public: | ||
|
|
||
| explicit constexpr non_owning_arrow_array_private_data(std::vector<std::uint8_t*>&& buffers_pointers) | ||
| : m_buffers_pointers(std::move(buffers_pointers)) | ||
| { | ||
| } | ||
|
|
||
| [[nodiscard]] SPARROW_IPC_API const void** buffers_ptrs() noexcept; | ||
|
|
||
| private: | ||
|
|
||
| std::vector<std::uint8_t*> m_buffers_pointers; | ||
| }; | ||
| } |
62 changes: 62 additions & 0 deletions
62
include/sparrow_ipc/arrow_interface/arrow_array_schema_common_release.hpp
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,62 @@ | ||
|
|
||
| #pragma once | ||
|
|
||
| #include <sparrow/c_interface.hpp> | ||
|
|
||
| #include "arrow_array/private_data.hpp" | ||
| #include "arrow_schema/private_data.hpp" | ||
|
|
||
| namespace sparrow_ipc | ||
| { | ||
| /** | ||
| * Release the children and dictionnary of an `ArrowArray` or `ArrowSchema`. | ||
| * | ||
| * @tparam T `ArrowArray` or `ArrowSchema` | ||
| * @param t The `ArrowArray` or `ArrowSchema` to release. | ||
| */ | ||
| template <class T> | ||
| requires std::same_as<T, ArrowArray> || std::same_as<T, ArrowSchema> | ||
| void release_common_non_owning_arrow(T& t) | ||
| { | ||
| using private_data_type = std::conditional_t< | ||
| std::same_as<T, ArrowArray>, | ||
| non_owning_arrow_array_private_data, | ||
| non_owning_arrow_schema_private_data>; | ||
| if (t.release == nullptr) | ||
| { | ||
| return; | ||
| } | ||
| SPARROW_ASSERT_TRUE(t.private_data != nullptr); | ||
| const auto private_data = static_cast<const private_data_type*>(t.private_data); | ||
|
|
||
| if (t.dictionary) | ||
| { | ||
| if (t.dictionary->release) | ||
| { | ||
| t.dictionary->release(t.dictionary); | ||
| } | ||
| delete t.dictionary; | ||
| t.dictionary = nullptr; | ||
| } | ||
|
|
||
| if (t.children) | ||
| { | ||
| for (int64_t i = 0; i < t.n_children; ++i) | ||
| { | ||
| T* child = t.children[i]; | ||
| if (child) | ||
| { | ||
| if (child->release) | ||
| { | ||
| child->release(child); | ||
| } | ||
| delete child; | ||
| child = nullptr; | ||
| } | ||
| } | ||
| delete[] t.children; | ||
| t.children = nullptr; | ||
| } | ||
| t.release = nullptr; | ||
| } | ||
| } | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The private data is allocated in
fill_non_owning_arrow_schema, so I would expect this function to deallocate it?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It's deallocated in the release_non_owning_arrow_array and release_non_owning_arrow_schema function, but you are right, let's move it to the common releaser