diff --git a/encodings/alp/src/alp/array.rs b/encodings/alp/src/alp/array.rs index 4d9761180e8..011547720e4 100644 --- a/encodings/alp/src/alp/array.rs +++ b/encodings/alp/src/alp/array.rs @@ -17,6 +17,7 @@ use vortex_array::Precision; use vortex_array::ProstMetadata; use vortex_array::SerializeMetadata; use vortex_array::VectorExecutor; +use vortex_array::buffer::BufferHandle; use vortex_array::patches::Patches; use vortex_array::patches::PatchesMetadata; use vortex_array::serde::ArrayChildren; @@ -34,7 +35,6 @@ use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityChild; use vortex_array::vtable::ValidityVTableFromChild; use vortex_array::vtable::VisitorVTable; -use vortex_buffer::BufferHandle; use vortex_dtype::DType; use vortex_dtype::PType; use vortex_error::VortexError; diff --git a/encodings/alp/src/alp_rd/array.rs b/encodings/alp/src/alp_rd/array.rs index fd767f9e01c..408b2366ddd 100644 --- a/encodings/alp/src/alp_rd/array.rs +++ b/encodings/alp/src/alp_rd/array.rs @@ -18,6 +18,7 @@ use vortex_array::ProstMetadata; use vortex_array::SerializeMetadata; use vortex_array::ToCanonical; use vortex_array::arrays::PrimitiveArray; +use vortex_array::buffer::BufferHandle; use vortex_array::patches::Patches; use vortex_array::patches::PatchesMetadata; use vortex_array::serde::ArrayChildren; @@ -37,7 +38,6 @@ use vortex_array::vtable::ValidityChild; use vortex_array::vtable::ValidityVTableFromChild; use vortex_array::vtable::VisitorVTable; use vortex_buffer::Buffer; -use vortex_buffer::BufferHandle; use vortex_dtype::DType; use vortex_dtype::Nullability; use vortex_dtype::PType; diff --git a/encodings/bytebool/src/array.rs b/encodings/bytebool/src/array.rs index f40d7424783..0227c285da3 100644 --- a/encodings/bytebool/src/array.rs +++ b/encodings/bytebool/src/array.rs @@ -15,6 +15,7 @@ use vortex_array::EmptyMetadata; use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::arrays::BoolArray; +use vortex_array::buffer::BufferHandle; use vortex_array::serde::ArrayChildren; use vortex_array::stats::ArrayStats; use vortex_array::stats::StatsSetRef; @@ -32,7 +33,6 @@ use vortex_array::vtable::ValidityHelper; use vortex_array::vtable::ValidityVTableFromValidityHelper; use vortex_array::vtable::VisitorVTable; use vortex_buffer::BitBuffer; -use vortex_buffer::BufferHandle; use vortex_buffer::ByteBuffer; use vortex_dtype::DType; use vortex_error::VortexExpect; diff --git a/encodings/datetime-parts/src/array.rs b/encodings/datetime-parts/src/array.rs index 41bf19bb0ce..7cd732e1aa1 100644 --- a/encodings/datetime-parts/src/array.rs +++ b/encodings/datetime-parts/src/array.rs @@ -16,6 +16,7 @@ use vortex_array::Precision; use vortex_array::ProstMetadata; use vortex_array::SerializeMetadata; use vortex_array::arrays::TemporalArray; +use vortex_array::buffer::BufferHandle; use vortex_array::serde::ArrayChildren; use vortex_array::stats::ArrayStats; use vortex_array::stats::StatsSetRef; @@ -30,7 +31,6 @@ use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityChild; use vortex_array::vtable::ValidityVTableFromChild; use vortex_array::vtable::VisitorVTable; -use vortex_buffer::BufferHandle; use vortex_dtype::DType; use vortex_dtype::Nullability; use vortex_dtype::PType; diff --git a/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs b/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs index f7dd0fee93c..462fb2aefc1 100644 --- a/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs +++ b/encodings/decimal-byte-parts/src/decimal_byte_parts/mod.rs @@ -21,6 +21,7 @@ use vortex_array::ProstMetadata; use vortex_array::SerializeMetadata; use vortex_array::ToCanonical; use vortex_array::arrays::DecimalArray; +use vortex_array::buffer::BufferHandle; use vortex_array::serde::ArrayChildren; use vortex_array::stats::ArrayStats; use vortex_array::stats::StatsSetRef; @@ -37,7 +38,6 @@ use vortex_array::vtable::ValidityChild; use vortex_array::vtable::ValidityHelper; use vortex_array::vtable::ValidityVTableFromChild; use vortex_array::vtable::VisitorVTable; -use vortex_buffer::BufferHandle; use vortex_dtype::DType; use vortex_dtype::DecimalDType; use vortex_dtype::PType; diff --git a/encodings/fastlanes/src/bitpacking/vtable/mod.rs b/encodings/fastlanes/src/bitpacking/vtable/mod.rs index 855b298a7fa..b3da8310e28 100644 --- a/encodings/fastlanes/src/bitpacking/vtable/mod.rs +++ b/encodings/fastlanes/src/bitpacking/vtable/mod.rs @@ -6,6 +6,7 @@ use vortex_array::DeserializeMetadata; use vortex_array::ExecutionCtx; use vortex_array::ProstMetadata; use vortex_array::SerializeMetadata; +use vortex_array::buffer::BufferHandle; use vortex_array::patches::Patches; use vortex_array::patches::PatchesMetadata; use vortex_array::serde::ArrayChildren; @@ -17,7 +18,6 @@ use vortex_array::vtable::ArrayVTableExt; use vortex_array::vtable::NotSupported; use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityVTableFromValidityHelper; -use vortex_buffer::BufferHandle; use vortex_dtype::DType; use vortex_dtype::PType; use vortex_error::VortexError; diff --git a/encodings/fastlanes/src/delta/vtable/mod.rs b/encodings/fastlanes/src/delta/vtable/mod.rs index bde78a1220a..adfbc9057ca 100644 --- a/encodings/fastlanes/src/delta/vtable/mod.rs +++ b/encodings/fastlanes/src/delta/vtable/mod.rs @@ -10,6 +10,7 @@ use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::ProstMetadata; use vortex_array::VectorExecutor; +use vortex_array::buffer::BufferHandle; use vortex_array::serde::ArrayChildren; use vortex_array::vtable; use vortex_array::vtable::ArrayId; @@ -18,7 +19,6 @@ use vortex_array::vtable::ArrayVTableExt; use vortex_array::vtable::NotSupported; use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityVTableFromChildSliceHelper; -use vortex_buffer::BufferHandle; use vortex_dtype::DType; use vortex_dtype::NativePType; use vortex_dtype::PType; diff --git a/encodings/fastlanes/src/for/vtable/mod.rs b/encodings/fastlanes/src/for/vtable/mod.rs index e2bff6a6d36..07fc74d3d36 100644 --- a/encodings/fastlanes/src/for/vtable/mod.rs +++ b/encodings/fastlanes/src/for/vtable/mod.rs @@ -7,6 +7,7 @@ use std::fmt::Formatter; use vortex_array::ArrayRef; use vortex_array::DeserializeMetadata; use vortex_array::SerializeMetadata; +use vortex_array::buffer::BufferHandle; use vortex_array::serde::ArrayChildren; use vortex_array::vtable; use vortex_array::vtable::ArrayId; @@ -15,7 +16,6 @@ use vortex_array::vtable::ArrayVTableExt; use vortex_array::vtable::NotSupported; use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityVTableFromChild; -use vortex_buffer::BufferHandle; use vortex_dtype::DType; use vortex_error::VortexResult; use vortex_error::vortex_bail; diff --git a/encodings/fastlanes/src/rle/vtable/mod.rs b/encodings/fastlanes/src/rle/vtable/mod.rs index 8d08c3740cf..0aba9335fa9 100644 --- a/encodings/fastlanes/src/rle/vtable/mod.rs +++ b/encodings/fastlanes/src/rle/vtable/mod.rs @@ -4,6 +4,7 @@ use prost::Message; use vortex_array::ArrayRef; use vortex_array::ProstMetadata; +use vortex_array::buffer::BufferHandle; use vortex_array::serde::ArrayChildren; use vortex_array::vtable; use vortex_array::vtable::ArrayId; @@ -12,7 +13,6 @@ use vortex_array::vtable::ArrayVTableExt; use vortex_array::vtable::NotSupported; use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityVTableFromChildSliceHelper; -use vortex_buffer::BufferHandle; use vortex_dtype::DType; use vortex_dtype::Nullability; use vortex_dtype::PType; diff --git a/encodings/fsst/src/array.rs b/encodings/fsst/src/array.rs index 337e06c8240..54a10fde1ce 100644 --- a/encodings/fsst/src/array.rs +++ b/encodings/fsst/src/array.rs @@ -24,6 +24,7 @@ use vortex_array::ProstMetadata; use vortex_array::SerializeMetadata; use vortex_array::arrays::VarBinArray; use vortex_array::arrays::VarBinVTable; +use vortex_array::buffer::BufferHandle; use vortex_array::serde::ArrayChildren; use vortex_array::stats::ArrayStats; use vortex_array::stats::StatsSetRef; @@ -39,7 +40,6 @@ use vortex_array::vtable::ValidityChild; use vortex_array::vtable::ValidityVTableFromChild; use vortex_array::vtable::VisitorVTable; use vortex_buffer::Buffer; -use vortex_buffer::BufferHandle; use vortex_dtype::DType; use vortex_dtype::Nullability; use vortex_dtype::PType; diff --git a/encodings/pco/src/array.rs b/encodings/pco/src/array.rs index 71c2d4855e8..2b1220138a5 100644 --- a/encodings/pco/src/array.rs +++ b/encodings/pco/src/array.rs @@ -28,6 +28,7 @@ use vortex_array::ProstMetadata; use vortex_array::ToCanonical; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::PrimitiveVTable; +use vortex_array::buffer::BufferHandle; use vortex_array::compute::filter; use vortex_array::serde::ArrayChildren; use vortex_array::stats::ArrayStats; @@ -47,7 +48,6 @@ use vortex_array::vtable::ValidityHelper; use vortex_array::vtable::ValiditySliceHelper; use vortex_array::vtable::ValidityVTableFromValiditySliceHelper; use vortex_array::vtable::VisitorVTable; -use vortex_buffer::BufferHandle; use vortex_buffer::BufferMut; use vortex_buffer::ByteBuffer; use vortex_buffer::ByteBufferMut; diff --git a/encodings/runend/src/array.rs b/encodings/runend/src/array.rs index cc0fa9059e7..37546d04c46 100644 --- a/encodings/runend/src/array.rs +++ b/encodings/runend/src/array.rs @@ -16,6 +16,7 @@ use vortex_array::ProstMetadata; use vortex_array::SerializeMetadata; use vortex_array::ToCanonical; use vortex_array::arrays::PrimitiveVTable; +use vortex_array::buffer::BufferHandle; use vortex_array::search_sorted::SearchSorted; use vortex_array::search_sorted::SearchSortedSide; use vortex_array::serde::ArrayChildren; @@ -31,7 +32,6 @@ use vortex_array::vtable::CanonicalVTable; use vortex_array::vtable::NotSupported; use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityVTable; -use vortex_buffer::BufferHandle; use vortex_dtype::DType; use vortex_dtype::Nullability; use vortex_dtype::PType; diff --git a/encodings/sequence/src/array.rs b/encodings/sequence/src/array.rs index 7f86271b535..03bb2e6d165 100644 --- a/encodings/sequence/src/array.rs +++ b/encodings/sequence/src/array.rs @@ -16,6 +16,7 @@ use vortex_array::ProstMetadata; use vortex_array::SerializeMetadata; use vortex_array::arrays::FilterVTable; use vortex_array::arrays::PrimitiveArray; +use vortex_array::buffer::BufferHandle; use vortex_array::serde::ArrayChildren; use vortex_array::stats::ArrayStats; use vortex_array::stats::StatsSetRef; @@ -32,7 +33,6 @@ use vortex_array::vtable::OperationsVTable; use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityVTable; use vortex_array::vtable::VisitorVTable; -use vortex_buffer::BufferHandle; use vortex_buffer::BufferMut; use vortex_dtype::DType; use vortex_dtype::NativePType; diff --git a/encodings/sparse/src/lib.rs b/encodings/sparse/src/lib.rs index f35daa91cde..1cbaa5a992b 100644 --- a/encodings/sparse/src/lib.rs +++ b/encodings/sparse/src/lib.rs @@ -19,6 +19,7 @@ use vortex_array::Precision; use vortex_array::ProstMetadata; use vortex_array::ToCanonical; use vortex_array::arrays::ConstantArray; +use vortex_array::buffer::BufferHandle; use vortex_array::compute::Operator; use vortex_array::compute::compare; use vortex_array::compute::fill_null; @@ -42,7 +43,6 @@ use vortex_array::vtable::ValidityVTable; use vortex_array::vtable::VisitorVTable; use vortex_buffer::BitBufferMut; use vortex_buffer::Buffer; -use vortex_buffer::BufferHandle; use vortex_buffer::ByteBufferMut; use vortex_dtype::DType; use vortex_dtype::NativePType; diff --git a/encodings/zigzag/src/array.rs b/encodings/zigzag/src/array.rs index bd556370430..83d15c5d7c8 100644 --- a/encodings/zigzag/src/array.rs +++ b/encodings/zigzag/src/array.rs @@ -15,6 +15,7 @@ use vortex_array::EmptyMetadata; use vortex_array::IntoArray; use vortex_array::Precision; use vortex_array::ToCanonical; +use vortex_array::buffer::BufferHandle; use vortex_array::serde::ArrayChildren; use vortex_array::stats::ArrayStats; use vortex_array::stats::StatsSetRef; @@ -31,7 +32,6 @@ use vortex_array::vtable::VTable; use vortex_array::vtable::ValidityChild; use vortex_array::vtable::ValidityVTableFromChild; use vortex_array::vtable::VisitorVTable; -use vortex_buffer::BufferHandle; use vortex_dtype::DType; use vortex_dtype::PType; use vortex_dtype::match_each_unsigned_integer_ptype; diff --git a/encodings/zstd/src/array.rs b/encodings/zstd/src/array.rs index d36cda68b75..3b9a7c27623 100644 --- a/encodings/zstd/src/array.rs +++ b/encodings/zstd/src/array.rs @@ -22,6 +22,7 @@ use vortex_array::accessor::ArrayAccessor; use vortex_array::arrays::ConstantArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::VarBinViewArray; +use vortex_array::buffer::BufferHandle; use vortex_array::compute::filter; use vortex_array::serde::ArrayChildren; use vortex_array::stats::ArrayStats; @@ -43,7 +44,6 @@ use vortex_array::vtable::ValidityVTableFromValiditySliceHelper; use vortex_array::vtable::VisitorVTable; use vortex_buffer::Alignment; use vortex_buffer::Buffer; -use vortex_buffer::BufferHandle; use vortex_buffer::BufferMut; use vortex_buffer::ByteBuffer; use vortex_buffer::ByteBufferMut; diff --git a/vortex-array/Cargo.toml b/vortex-array/Cargo.toml index 5c690f768fb..f4a896caee1 100644 --- a/vortex-array/Cargo.toml +++ b/vortex-array/Cargo.toml @@ -67,7 +67,7 @@ vortex-mask = { workspace = true } vortex-proto = { workspace = true, features = ["expr"] } vortex-scalar = { workspace = true } vortex-session = { workspace = true } -vortex-utils = { workspace = true } +vortex-utils = { workspace = true, features = ["dyn-traits"] } vortex-vector = { workspace = true } [features] diff --git a/vortex-array/src/arrays/bool/vtable/mod.rs b/vortex-array/src/arrays/bool/vtable/mod.rs index 581d83d1649..7e5c49ff25e 100644 --- a/vortex-array/src/arrays/bool/vtable/mod.rs +++ b/vortex-array/src/arrays/bool/vtable/mod.rs @@ -2,7 +2,6 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use vortex_buffer::BitBuffer; -use vortex_buffer::BufferHandle; use vortex_dtype::DType; use vortex_error::VortexExpect; use vortex_error::VortexResult; @@ -15,6 +14,7 @@ use crate::DeserializeMetadata; use crate::ProstMetadata; use crate::SerializeMetadata; use crate::arrays::BoolArray; +use crate::buffer::BufferHandle; use crate::executor::ExecutionCtx; use crate::serde::ArrayChildren; use crate::validity::Validity; diff --git a/vortex-array/src/arrays/chunked/vtable/mod.rs b/vortex-array/src/arrays/chunked/vtable/mod.rs index a24ad0c1360..7331561c00a 100644 --- a/vortex-array/src/arrays/chunked/vtable/mod.rs +++ b/vortex-array/src/arrays/chunked/vtable/mod.rs @@ -2,7 +2,6 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use itertools::Itertools; -use vortex_buffer::BufferHandle; use vortex_dtype::DType; use vortex_dtype::Nullability; use vortex_dtype::PType; @@ -19,6 +18,7 @@ use crate::ToCanonical; use crate::arrays::ChunkedArray; use crate::arrays::PrimitiveArray; use crate::arrays::chunked::vtable::rules::PARENT_RULES; +use crate::buffer::BufferHandle; use crate::serde::ArrayChildren; use crate::validity::Validity; use crate::vtable; diff --git a/vortex-array/src/arrays/constant/vtable/mod.rs b/vortex-array/src/arrays/constant/vtable/mod.rs index c4c4fb24ef7..6e151e74e34 100644 --- a/vortex-array/src/arrays/constant/vtable/mod.rs +++ b/vortex-array/src/arrays/constant/vtable/mod.rs @@ -3,7 +3,6 @@ use std::fmt::Debug; -use vortex_buffer::BufferHandle; use vortex_dtype::DType; use vortex_error::VortexResult; use vortex_error::vortex_bail; @@ -18,6 +17,7 @@ use crate::ArrayRef; use crate::EmptyMetadata; use crate::arrays::ConstantArray; use crate::arrays::constant::vtable::rules::PARENT_RULES; +use crate::buffer::BufferHandle; use crate::executor::ExecutionCtx; use crate::serde::ArrayChildren; use crate::vtable; diff --git a/vortex-array/src/arrays/decimal/vtable/mod.rs b/vortex-array/src/arrays/decimal/vtable/mod.rs index 5688d1fa7fa..fc5986dd094 100644 --- a/vortex-array/src/arrays/decimal/vtable/mod.rs +++ b/vortex-array/src/arrays/decimal/vtable/mod.rs @@ -3,7 +3,6 @@ use vortex_buffer::Alignment; use vortex_buffer::Buffer; -use vortex_buffer::BufferHandle; use vortex_dtype::DType; use vortex_dtype::NativeDecimalType; use vortex_dtype::PrecisionScale; @@ -20,6 +19,7 @@ use crate::DeserializeMetadata; use crate::ProstMetadata; use crate::SerializeMetadata; use crate::arrays::DecimalArray; +use crate::buffer::BufferHandle; use crate::executor::ExecutionCtx; use crate::serde::ArrayChildren; use crate::validity::Validity; diff --git a/vortex-array/src/arrays/dict/vtable/mod.rs b/vortex-array/src/arrays/dict/vtable/mod.rs index 2727465c6bc..1bbaa9c4652 100644 --- a/vortex-array/src/arrays/dict/vtable/mod.rs +++ b/vortex-array/src/arrays/dict/vtable/mod.rs @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use vortex_buffer::BufferHandle; use vortex_compute::take::Take; use vortex_dtype::DType; use vortex_dtype::Nullability; @@ -20,6 +19,7 @@ use crate::ProstMetadata; use crate::SerializeMetadata; use crate::VectorExecutor; use crate::arrays::vtable::rules::PARENT_RULES; +use crate::buffer::BufferHandle; use crate::executor::ExecutionCtx; use crate::serde::ArrayChildren; use crate::vtable; diff --git a/vortex-array/src/arrays/extension/vtable/mod.rs b/vortex-array/src/arrays/extension/vtable/mod.rs index a8e457f5931..e0ad7be3597 100644 --- a/vortex-array/src/arrays/extension/vtable/mod.rs +++ b/vortex-array/src/arrays/extension/vtable/mod.rs @@ -8,7 +8,6 @@ mod rules; mod validity; mod visitor; -use vortex_buffer::BufferHandle; use vortex_dtype::DType; use vortex_error::VortexExpect; use vortex_error::VortexResult; @@ -21,6 +20,7 @@ use crate::EmptyMetadata; use crate::VectorExecutor; use crate::arrays::extension::ExtensionArray; use crate::arrays::extension::vtable::rules::PARENT_RULES; +use crate::buffer::BufferHandle; use crate::executor::ExecutionCtx; use crate::serde::ArrayChildren; use crate::vtable; diff --git a/vortex-array/src/arrays/filter/vtable.rs b/vortex-array/src/arrays/filter/vtable.rs index 11f5e4106dd..a21c461474b 100644 --- a/vortex-array/src/arrays/filter/vtable.rs +++ b/vortex-array/src/arrays/filter/vtable.rs @@ -6,7 +6,6 @@ use std::fmt::Formatter; use std::hash::Hasher; use std::ops::Range; -use vortex_buffer::BufferHandle; use vortex_compute::filter::Filter; use vortex_dtype::DType; use vortex_error::VortexExpect; @@ -30,6 +29,7 @@ use crate::Precision; use crate::VectorExecutor; use crate::arrays::filter::array::FilterArray; use crate::arrays::filter::rules::PARENT_RULES; +use crate::buffer::BufferHandle; use crate::executor::ExecutionCtx; use crate::serde::ArrayChildren; use crate::stats::StatsSetRef; diff --git a/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs b/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs index 7b2be007d8f..46ac077eb05 100644 --- a/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs +++ b/vortex-array/src/arrays/fixed_size_list/vtable/mod.rs @@ -3,7 +3,6 @@ use std::sync::Arc; -use vortex_buffer::BufferHandle; use vortex_dtype::DType; use vortex_error::VortexExpect; use vortex_error::VortexResult; @@ -16,6 +15,7 @@ use crate::ArrayRef; use crate::EmptyMetadata; use crate::VectorExecutor; use crate::arrays::FixedSizeListArray; +use crate::buffer::BufferHandle; use crate::executor::ExecutionCtx; use crate::serde::ArrayChildren; use crate::validity::Validity; diff --git a/vortex-array/src/arrays/list/vtable/mod.rs b/vortex-array/src/arrays/list/vtable/mod.rs index 1dec5725102..48fb78c7e43 100644 --- a/vortex-array/src/arrays/list/vtable/mod.rs +++ b/vortex-array/src/arrays/list/vtable/mod.rs @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use vortex_buffer::BufferHandle; use vortex_dtype::DType; use vortex_dtype::Nullability; use vortex_dtype::PType; @@ -16,6 +15,7 @@ use crate::ExecutionCtx; use crate::ProstMetadata; use crate::arrays::ListArray; use crate::arrays::list::vtable::kernel::PARENT_KERNELS; +use crate::buffer::BufferHandle; use crate::metadata::DeserializeMetadata; use crate::metadata::SerializeMetadata; use crate::serde::ArrayChildren; diff --git a/vortex-array/src/arrays/listview/vtable/mod.rs b/vortex-array/src/arrays/listview/vtable/mod.rs index 33917a8b62a..47f7545d266 100644 --- a/vortex-array/src/arrays/listview/vtable/mod.rs +++ b/vortex-array/src/arrays/listview/vtable/mod.rs @@ -3,7 +3,6 @@ use std::sync::Arc; -use vortex_buffer::BufferHandle; use vortex_dtype::DType; use vortex_dtype::Nullability; use vortex_dtype::PType; @@ -21,6 +20,7 @@ use crate::SerializeMetadata; use crate::VectorExecutor; use crate::arrays::ListViewArray; use crate::arrays::listview::vtable::rules::PARENT_RULES; +use crate::buffer::BufferHandle; use crate::executor::ExecutionCtx; use crate::serde::ArrayChildren; use crate::validity::Validity; diff --git a/vortex-array/src/arrays/masked/vtable/mod.rs b/vortex-array/src/arrays/masked/vtable/mod.rs index bb19e24d99d..213b3a02255 100644 --- a/vortex-array/src/arrays/masked/vtable/mod.rs +++ b/vortex-array/src/arrays/masked/vtable/mod.rs @@ -6,7 +6,6 @@ mod canonical; mod operations; mod validity; -use vortex_buffer::BufferHandle; use vortex_dtype::DType; use vortex_error::VortexExpect; use vortex_error::VortexResult; @@ -21,6 +20,7 @@ use crate::ArrayRef; use crate::EmptyMetadata; use crate::VectorExecutor; use crate::arrays::masked::MaskedArray; +use crate::buffer::BufferHandle; use crate::executor::ExecutionCtx; use crate::serde::ArrayChildren; use crate::validity::Validity; diff --git a/vortex-array/src/arrays/null/mod.rs b/vortex-array/src/arrays/null/mod.rs index 6954de51ca6..adb5b72b35c 100644 --- a/vortex-array/src/arrays/null/mod.rs +++ b/vortex-array/src/arrays/null/mod.rs @@ -4,7 +4,6 @@ use std::hash::Hash; use std::ops::Range; -use vortex_buffer::BufferHandle; use vortex_dtype::DType; use vortex_error::VortexResult; use vortex_error::vortex_ensure; @@ -20,6 +19,7 @@ use crate::Canonical; use crate::EmptyMetadata; use crate::IntoArray; use crate::Precision; +use crate::buffer::BufferHandle; use crate::executor::ExecutionCtx; use crate::serde::ArrayChildren; use crate::stats::ArrayStats; diff --git a/vortex-array/src/arrays/primitive/vtable/mod.rs b/vortex-array/src/arrays/primitive/vtable/mod.rs index 41f73821fb2..af993391090 100644 --- a/vortex-array/src/arrays/primitive/vtable/mod.rs +++ b/vortex-array/src/arrays/primitive/vtable/mod.rs @@ -3,7 +3,6 @@ use vortex_buffer::Alignment; use vortex_buffer::Buffer; -use vortex_buffer::BufferHandle; use vortex_dtype::DType; use vortex_dtype::PType; use vortex_dtype::match_each_native_ptype; @@ -16,6 +15,7 @@ use vortex_vector::primitive::PVector; use crate::ArrayRef; use crate::EmptyMetadata; use crate::arrays::PrimitiveArray; +use crate::buffer::BufferHandle; use crate::executor::ExecutionCtx; use crate::serde::ArrayChildren; use crate::validity::Validity; diff --git a/vortex-array/src/arrays/scalar_fn/vtable/mod.rs b/vortex-array/src/arrays/scalar_fn/vtable/mod.rs index 8b25dd32923..15a091f325e 100644 --- a/vortex-array/src/arrays/scalar_fn/vtable/mod.rs +++ b/vortex-array/src/arrays/scalar_fn/vtable/mod.rs @@ -11,7 +11,6 @@ use std::marker::PhantomData; use std::ops::Deref; use itertools::Itertools; -use vortex_buffer::BufferHandle; use vortex_dtype::DType; use vortex_error::VortexExpect; use vortex_error::VortexResult; @@ -29,6 +28,7 @@ use crate::arrays::scalar_fn::array::ScalarFnArray; use crate::arrays::scalar_fn::metadata::ScalarFnMetadata; use crate::arrays::scalar_fn::rules::PARENT_RULES; use crate::arrays::scalar_fn::rules::RULES; +use crate::buffer::BufferHandle; use crate::executor::ExecutionCtx; use crate::expr; use crate::expr::ExecutionArgs; diff --git a/vortex-array/src/arrays/struct_/vtable/mod.rs b/vortex-array/src/arrays/struct_/vtable/mod.rs index 1b946a38c26..63b0e80d425 100644 --- a/vortex-array/src/arrays/struct_/vtable/mod.rs +++ b/vortex-array/src/arrays/struct_/vtable/mod.rs @@ -4,7 +4,6 @@ use std::sync::Arc; use itertools::Itertools; -use vortex_buffer::BufferHandle; use vortex_dtype::DType; use vortex_error::VortexExpect; use vortex_error::VortexResult; @@ -18,6 +17,7 @@ use crate::EmptyMetadata; use crate::VectorExecutor; use crate::arrays::struct_::StructArray; use crate::arrays::struct_::vtable::rules::PARENT_RULES; +use crate::buffer::BufferHandle; use crate::executor::ExecutionCtx; use crate::serde::ArrayChildren; use crate::validity::Validity; diff --git a/vortex-array/src/arrays/varbin/vtable/mod.rs b/vortex-array/src/arrays/varbin/vtable/mod.rs index c3ec946f839..5e15bccabd4 100644 --- a/vortex-array/src/arrays/varbin/vtable/mod.rs +++ b/vortex-array/src/arrays/varbin/vtable/mod.rs @@ -1,7 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use vortex_buffer::BufferHandle; use vortex_dtype::DType; use vortex_dtype::Nullability; use vortex_dtype::PType; @@ -15,6 +14,7 @@ use crate::DeserializeMetadata; use crate::ProstMetadata; use crate::SerializeMetadata; use crate::arrays::varbin::VarBinArray; +use crate::buffer::BufferHandle; use crate::serde::ArrayChildren; use crate::validity::Validity; use crate::vtable; diff --git a/vortex-array/src/arrays/varbinview/vtable/mod.rs b/vortex-array/src/arrays/varbinview/vtable/mod.rs index 172ba0299fd..508bc48b4e1 100644 --- a/vortex-array/src/arrays/varbinview/vtable/mod.rs +++ b/vortex-array/src/arrays/varbinview/vtable/mod.rs @@ -4,7 +4,6 @@ use std::sync::Arc; use vortex_buffer::Buffer; -use vortex_buffer::BufferHandle; use vortex_buffer::ByteBuffer; use vortex_dtype::DType; use vortex_error::VortexExpect; @@ -19,6 +18,7 @@ use vortex_vector::binaryview::StringVector; use crate::ArrayRef; use crate::EmptyMetadata; use crate::arrays::varbinview::VarBinViewArray; +use crate::buffer::BufferHandle; use crate::executor::ExecutionCtx; use crate::serde::ArrayChildren; use crate::validity::Validity; diff --git a/vortex-array/src/arrow/array.rs b/vortex-array/src/arrow/array.rs index 4cb469fcb77..b724114fec3 100644 --- a/vortex-array/src/arrow/array.rs +++ b/vortex-array/src/arrow/array.rs @@ -7,7 +7,6 @@ use std::ops::Range; use arrow_array::ArrayRef as ArrowArrayRef; use vortex_buffer::BitBuffer; -use vortex_buffer::BufferHandle; use vortex_dtype::DType; use vortex_dtype::Nullability; use vortex_dtype::arrow::FromArrowType; @@ -28,6 +27,7 @@ use crate::IntoArray; use crate::Precision; use crate::arrays::BoolArray; use crate::arrow::FromArrowArray; +use crate::buffer::BufferHandle; use crate::serde::ArrayChildren; use crate::stats::ArrayStats; use crate::stats::StatsSetRef; diff --git a/vortex-array/src/buffer.rs b/vortex-array/src/buffer.rs new file mode 100644 index 00000000000..370a2501eb9 --- /dev/null +++ b/vortex-array/src/buffer.rs @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::fmt::Debug; +use std::hash::Hash; +use std::sync::Arc; + +use vortex_buffer::ByteBuffer; +use vortex_error::VortexResult; +use vortex_error::vortex_bail; +use vortex_utils::dyn_traits::DynEq; +use vortex_utils::dyn_traits::DynHash; + +/// A buffer can be either on the CPU or on an attached device (e.g. GPU). +/// The Device implementation will come later. +#[derive(Debug, Clone)] +pub enum BufferHandle { + /// On the host/cpu. + Host(ByteBuffer), + /// On the device. + Device(Arc), +} + +/// A buffer that is stored on a device (e.g. GPU). +pub trait DeviceBuffer: 'static + Send + Sync + Debug + DynEq + DynHash { + /// Returns the length of the buffer in bytes. + fn len(&self) -> usize; + + /// Returns true if the buffer is empty. + fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Attempts to copy the device buffer to a host ByteBuffer. + fn to_host(self: Arc) -> VortexResult; +} + +impl Hash for dyn DeviceBuffer { + fn hash(&self, state: &mut H) { + self.dyn_hash(state); + } +} + +impl PartialEq for dyn DeviceBuffer { + fn eq(&self, other: &Self) -> bool { + self.dyn_eq(other) + } +} +impl Eq for dyn DeviceBuffer {} + +impl BufferHandle { + /// Fetches the cpu buffer and fails otherwise. + pub fn bytes(&self) -> &ByteBuffer { + match self { + BufferHandle::Host(b) => b, + BufferHandle::Device(_) => todo!(), + } + } + + /// Fetches the cpu buffer and fails otherwise. + pub fn into_bytes(self) -> ByteBuffer { + match self { + BufferHandle::Host(b) => b, + BufferHandle::Device(_) => todo!(), + } + } + + /// Attempts to convert this handle into a CPU ByteBuffer. + /// Returns an error if the buffer is on a device. + pub fn try_to_bytes(self) -> VortexResult { + match self { + BufferHandle::Host(b) => Ok(b), + BufferHandle::Device(_) => vortex_bail!("cannot move device_buffer to buffer"), + } + } +} diff --git a/vortex-array/src/hash.rs b/vortex-array/src/hash.rs index 4ba5cf636ad..b88a92212d7 100644 --- a/vortex-array/src/hash.rs +++ b/vortex-array/src/hash.rs @@ -4,14 +4,15 @@ use std::any::Any; use std::hash::Hash; use std::hash::Hasher; +use std::sync::Arc; use vortex_buffer::BitBuffer; use vortex_buffer::Buffer; -use vortex_buffer::BufferHandle; use vortex_mask::Mask; use crate::Array; use crate::ArrayRef; +use crate::buffer::BufferHandle; use crate::patches::Patches; use crate::validity::Validity; @@ -258,8 +259,8 @@ impl ArrayEq for Patches { impl ArrayHash for BufferHandle { fn array_hash(&self, state: &mut H, precision: Precision) { match self { - BufferHandle::Buffer(b) => b.array_hash(state, precision), - BufferHandle::DeviceBuffer => (), + BufferHandle::Host(b) => b.array_hash(state, precision), + BufferHandle::Device(_) => (), } } } @@ -267,8 +268,11 @@ impl ArrayHash for BufferHandle { impl ArrayEq for BufferHandle { fn array_eq(&self, other: &Self, precision: Precision) -> bool { match (self, other) { - (BufferHandle::Buffer(b), BufferHandle::Buffer(b2)) => b.array_eq(b2, precision), - (BufferHandle::DeviceBuffer, BufferHandle::DeviceBuffer) => true, + (BufferHandle::Host(b), BufferHandle::Host(b2)) => b.array_eq(b2, precision), + (BufferHandle::Device(b), BufferHandle::Device(b2)) => match precision { + Precision::Ptr => Arc::ptr_eq(b, b2), + Precision::Value => b.eq(b2), + }, _ => false, } } diff --git a/vortex-array/src/lib.rs b/vortex-array/src/lib.rs index e16f9e1d8ba..bad7d44e9f8 100644 --- a/vortex-array/src/lib.rs +++ b/vortex-array/src/lib.rs @@ -31,6 +31,7 @@ pub mod aliases; mod array; pub mod arrays; pub mod arrow; +pub mod buffer; pub mod builders; pub mod builtins; mod canonical; diff --git a/vortex-array/src/serde.rs b/vortex-array/src/serde.rs index 3df50437211..854378385ea 100644 --- a/vortex-array/src/serde.rs +++ b/vortex-array/src/serde.rs @@ -12,7 +12,6 @@ use flatbuffers::WIPOffset; use flatbuffers::root; use itertools::Itertools; use vortex_buffer::Alignment; -use vortex_buffer::BufferHandle; use vortex_buffer::ByteBuffer; use vortex_dtype::DType; use vortex_dtype::TryFromBytes; @@ -34,6 +33,7 @@ use crate::ArrayContext; use crate::ArrayRef; use crate::ArrayVisitor; use crate::ArrayVisitorExt; +use crate::buffer::BufferHandle; use crate::stats::StatsSet; /// Options for serializing an array. @@ -494,7 +494,7 @@ impl ArrayParts { .aligned(Alignment::from_exponent(fb_buf.alignment_exponent())); offset += buffer_len; - BufferHandle::Buffer(buffer) + BufferHandle::Host(buffer) }) .collect(); @@ -565,7 +565,7 @@ impl TryFrom for ArrayParts { .aligned(Alignment::from_exponent(fb_buffer.alignment_exponent())); offset += buffer_len; - BufferHandle::Buffer(buffer) + BufferHandle::Host(buffer) }) .collect(); diff --git a/vortex-array/src/vtable/dyn_.rs b/vortex-array/src/vtable/dyn_.rs index 6fbb6005ddc..0b5406a6b05 100644 --- a/vortex-array/src/vtable/dyn_.rs +++ b/vortex-array/src/vtable/dyn_.rs @@ -11,7 +11,6 @@ use std::hash::Hasher; use std::sync::Arc; use arcref::ArcRef; -use vortex_buffer::BufferHandle; use vortex_dtype::DType; use vortex_error::VortexExpect; use vortex_error::VortexResult; @@ -27,6 +26,7 @@ use crate::ArrayAdapter; use crate::ArrayRef; use crate::Canonical; use crate::IntoArray; +use crate::buffer::BufferHandle; use crate::executor::ExecutionCtx; use crate::serde::ArrayChildren; use crate::vtable::EncodeVTable; diff --git a/vortex-array/src/vtable/mod.rs b/vortex-array/src/vtable/mod.rs index d0621af0c02..4ba5889f61a 100644 --- a/vortex-array/src/vtable/mod.rs +++ b/vortex-array/src/vtable/mod.rs @@ -23,7 +23,6 @@ pub use encode::*; pub use operations::*; pub use validity::*; pub use visitor::*; -use vortex_buffer::BufferHandle; use vortex_dtype::DType; use vortex_error::VortexResult; use vortex_vector::Vector; @@ -32,6 +31,7 @@ use crate::Array; use crate::ArrayRef; use crate::IntoArray; use crate::VectorExecutor; +use crate::buffer::BufferHandle; use crate::executor::ExecutionCtx; use crate::serde::ArrayChildren; diff --git a/vortex-buffer/src/lib.rs b/vortex-buffer/src/lib.rs index 5f5c1a19836..8646be4139a 100644 --- a/vortex-buffer/src/lib.rs +++ b/vortex-buffer/src/lib.rs @@ -55,9 +55,6 @@ pub use buffer_mut::*; pub use bytes::*; pub use r#const::*; pub use string::*; -use vortex_error::VortexResult; -use vortex_error::vortex_bail; - mod alignment; #[cfg(feature = "arrow")] mod arrow; @@ -85,41 +82,3 @@ pub type ByteBufferMut = BufferMut; /// A const-aligned buffer of u8. pub type ConstByteBuffer = ConstBuffer; - -#[derive(Debug, Clone)] -/// A buffer can be either on the CPU or on an attached device (e.g. GPU). -/// The Device implementation will come later. -pub enum BufferHandle { - /// On the host/cpu. - Buffer(ByteBuffer), - /// On the device. - // TODO: impl this. - DeviceBuffer, -} - -impl BufferHandle { - /// Fetches the cpu buffer and fails otherwise. - pub fn bytes(&self) -> &ByteBuffer { - match self { - BufferHandle::Buffer(b) => b, - BufferHandle::DeviceBuffer => todo!(), - } - } - - /// Fetches the cpu buffer and fails otherwise. - pub fn into_bytes(self) -> ByteBuffer { - match self { - BufferHandle::Buffer(b) => b, - BufferHandle::DeviceBuffer => todo!(), - } - } - - /// Attempts to convert this handle into a CPU ByteBuffer. - /// Returns an error if the buffer is on a device. - pub fn try_to_bytes(self) -> VortexResult { - match self { - BufferHandle::Buffer(b) => Ok(b), - BufferHandle::DeviceBuffer => vortex_bail!("cannot move device_buffer to buffer"), - } - } -} diff --git a/vortex-file/src/segments/source.rs b/vortex-file/src/segments/source.rs index 1f8faffe482..58517505ce1 100644 --- a/vortex-file/src/segments/source.rs +++ b/vortex-file/src/segments/source.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use futures::FutureExt; use futures::TryFutureExt; -use vortex_buffer::BufferHandle; +use vortex_array::buffer::BufferHandle; use vortex_error::VortexError; use vortex_error::vortex_err; use vortex_io::VortexReadAt; @@ -41,7 +41,7 @@ impl SegmentSource for FileSegmentSource { maybe_fut .ok_or_else(|| vortex_err!("Missing segment: {}", id))? .await - .map(BufferHandle::Buffer) + .map(BufferHandle::Host) } .boxed() } diff --git a/vortex-layout/src/segments/cache.rs b/vortex-layout/src/segments/cache.rs index 16ae8485705..155ed038ec7 100644 --- a/vortex-layout/src/segments/cache.rs +++ b/vortex-layout/src/segments/cache.rs @@ -9,7 +9,7 @@ use moka::future::Cache; use moka::future::CacheBuilder; use moka::policy::EvictionPolicy; use rustc_hash::FxBuildHasher; -use vortex_buffer::BufferHandle; +use vortex_array::buffer::BufferHandle; use vortex_buffer::ByteBuffer; use vortex_error::VortexExpect; use vortex_error::VortexResult; @@ -130,11 +130,11 @@ impl SegmentSource for SegmentCacheSourceAdapter { async move { if let Ok(Some(segment)) = cache.get(id).await { tracing::debug!("Resolved segment {} from cache", id); - return Ok(BufferHandle::Buffer(segment)); + return Ok(BufferHandle::Host(segment)); } let result = delegate.await?; // Cache only CPU buffers; device buffers are not cached. - if let BufferHandle::Buffer(ref buffer) = result + if let BufferHandle::Host(ref buffer) = result && let Err(e) = cache.put(id, buffer.clone()).await { tracing::warn!("Failed to store segment {} in cache: {}", id, e); diff --git a/vortex-layout/src/segments/shared.rs b/vortex-layout/src/segments/shared.rs index 0c442ecb527..f0a321e035f 100644 --- a/vortex-layout/src/segments/shared.rs +++ b/vortex-layout/src/segments/shared.rs @@ -7,7 +7,7 @@ use futures::FutureExt; use futures::TryFutureExt; use futures::future::BoxFuture; use futures::future::WeakShared; -use vortex_buffer::BufferHandle; +use vortex_array::buffer::BufferHandle; use vortex_error::SharedVortexResult; use vortex_error::VortexError; use vortex_error::VortexExpect; diff --git a/vortex-layout/src/segments/source.rs b/vortex-layout/src/segments/source.rs index 11256da0356..a48a79b2889 100644 --- a/vortex-layout/src/segments/source.rs +++ b/vortex-layout/src/segments/source.rs @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use futures::future::BoxFuture; -use vortex_buffer::BufferHandle; +use vortex_array::buffer::BufferHandle; use vortex_error::VortexResult; use crate::segments::SegmentId; diff --git a/vortex-layout/src/segments/test.rs b/vortex-layout/src/segments/test.rs index 10141150e95..e22deec994d 100644 --- a/vortex-layout/src/segments/test.rs +++ b/vortex-layout/src/segments/test.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use async_trait::async_trait; use futures::FutureExt; use parking_lot::Mutex; -use vortex_buffer::BufferHandle; +use vortex_array::buffer::BufferHandle; use vortex_buffer::ByteBuffer; use vortex_buffer::ByteBufferMut; use vortex_error::VortexExpect; @@ -30,7 +30,7 @@ impl SegmentSource for TestSegments { let buffer = self.segments.lock().get(*id as usize).cloned(); async move { buffer - .map(BufferHandle::Buffer) + .map(BufferHandle::Host) .ok_or_else(|| vortex_err!("Segment not found")) } .boxed() diff --git a/vortex-python/src/arrays/py/vtable.rs b/vortex-python/src/arrays/py/vtable.rs index b478cc39ac1..519533ed1a0 100644 --- a/vortex-python/src/arrays/py/vtable.rs +++ b/vortex-python/src/arrays/py/vtable.rs @@ -18,6 +18,7 @@ use vortex::array::Canonical; use vortex::array::Precision; use vortex::array::RawMetadata; use vortex::array::SerializeMetadata; +use vortex::array::buffer::BufferHandle; use vortex::array::serde::ArrayChildren; use vortex::array::stats::StatsSetRef; use vortex::array::validity::Validity; @@ -32,7 +33,6 @@ use vortex::array::vtable::OperationsVTable; use vortex::array::vtable::VTable; use vortex::array::vtable::ValidityVTable; use vortex::array::vtable::VisitorVTable; -use vortex::buffer::BufferHandle; use vortex::compute::ComputeFn; use vortex::compute::InvocationArgs; use vortex::compute::Output; diff --git a/vortex-utils/src/dyn_traits.rs b/vortex-utils/src/dyn_traits.rs index 16ce7086942..d25fc826d46 100644 --- a/vortex-utils/src/dyn_traits.rs +++ b/vortex-utils/src/dyn_traits.rs @@ -6,6 +6,9 @@ use std::any::Any; /// A dynamic equality trait for types implementing [`PartialEq`] or [`Eq`]. +/// +/// Developers should implement `PartialEq` or `Eq` for `dyn Trait` types and +/// delegate to this trait for dynamic equality checks. pub trait DynEq: Any + private::SealedEq { /// Compares `self` with another `Any` type for equality. fn dyn_eq(&self, other: &dyn Any) -> bool;