Initial Vortex Layouts (#1805)

Initial implementation of the new structure of vortex layouts per #1676 

* Only flat layout works.
* I'm not 100% sure on the trait APIs, these will evolve as we pad out
the implementation.
* StructLayout will be worked on as part of #1782 so will probably come
last.

Up next:
* Implementation of ChunkedLayout

Open Questions:
* What is the API that e.g. Python users have to precisely configure
layout strategies? Can I override a layout writer for a specific field?
* Similarly, how can we configure the layout scanners? Can I configure a
level 0 chunked layout differently from level 2 in a
chunk-of-struct-of-chunk world?

Author: gatesn

Cargo.lock

@@ -16,6 +16,7 @@ members = [
     "vortex-flatbuffers",
     "vortex-io",
     "vortex-ipc",
+    "vortex-layout",
     "vortex-proto",
     "vortex-sampling-compressor",
     "vortex-scalar",

Cargo.toml

@@ -18,6 +18,7 @@ dtype = []
 scalar = ["dtype"]
 array = ["dtype", "scalar"]
 ipc = ["array"]
+layout = ["array"]
 file = ["ipc"]
 
 [dependencies]

vortex-flatbuffers/Cargo.toml

@@ -0,0 +1,29 @@
+/// A `Layout` is a recursive data structure describing the physical layout of Vortex arrays in random access storage.
+/// As a starting, concrete example, the first three Layout encodings are defined as:
+///
+/// 1. encoding == 1, `Flat` -> one buffer, zero child Layouts
+/// 2. encoding == 2, `Chunked` -> zero buffers, one or more child Layouts (used for chunks of rows)
+/// 3. encoding == 3, `Columnar` -> zero buffers, one or more child Layouts (used for columns of structs)
+///
+/// The `row_count` represents the number of rows represented by this Layout. This is very useful for
+/// pruning the Layout tree based on row filters.
+///
+/// The `metadata` field is fully opaque at this layer, and allows the Layout implementation corresponding to
+/// `encoding` to embed additional information that may be useful for the reader. For example, the `ChunkedLayout`
+/// uses the first byte of the `metadata` array as a boolean to indicate whether the first child Layout represents
+/// the statistics table for the other chunks. 
+table Layout {
+    /// The ID of the encoding used for this Layout.
+    encoding: uint16;
+    /// The number of rows of data represented by this Layout.
+    row_count: uint64;
+    /// Any additional metadata this layout needs to interpret its children.
+    /// This does not include data-specific metadata, which the layout should store in a segment.
+    metadata: [ubyte];
+    /// The children of this Layout.
+    children: [Layout];
+    /// Identifiers for each `Segment` of data required by this layout.
+    segments: [uint32];
+}
+
+root_type Layout;

vortex-flatbuffers/flatbuffers/vortex-layout/layout.fbs

@@ -0,0 +1,267 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+
+// @generated
+
+use core::mem;
+use core::cmp::Ordering;
+
+extern crate flatbuffers;
+use self::flatbuffers::{EndianScalar, Follow};
+
+pub enum LayoutOffset {}
+#[derive(Copy, Clone, PartialEq)]
+
+/// A `Layout` is a recursive data structure describing the physical layout of Vortex arrays in random access storage.
+/// As a starting, concrete example, the first three Layout encodings are defined as:
+///
+/// 1. encoding == 1, `Flat` -> one buffer, zero child Layouts
+/// 2. encoding == 2, `Chunked` -> zero buffers, one or more child Layouts (used for chunks of rows)
+/// 3. encoding == 3, `Columnar` -> zero buffers, one or more child Layouts (used for columns of structs)
+///
+/// The `row_count` represents the number of rows represented by this Layout. This is very useful for
+/// pruning the Layout tree based on row filters.
+///
+/// The `metadata` field is fully opaque at this layer, and allows the Layout implementation corresponding to
+/// `encoding` to embed additional information that may be useful for the reader. For example, the `ChunkedLayout`
+/// uses the first byte of the `metadata` array as a boolean to indicate whether the first child Layout represents
+/// the statistics table for the other chunks. 
+pub struct Layout<'a> {
+  pub _tab: flatbuffers::Table<'a>,
+}
+
+impl<'a> flatbuffers::Follow<'a> for Layout<'a> {
+  type Inner = Layout<'a>;
+  #[inline]
+  unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+    Self { _tab: flatbuffers::Table::new(buf, loc) }
+  }
+}
+
+impl<'a> Layout<'a> {
+  pub const VT_ENCODING: flatbuffers::VOffsetT = 4;
+  pub const VT_ROW_COUNT: flatbuffers::VOffsetT = 6;
+  pub const VT_METADATA: flatbuffers::VOffsetT = 8;
+  pub const VT_CHILDREN: flatbuffers::VOffsetT = 10;
+  pub const VT_SEGMENTS: flatbuffers::VOffsetT = 12;
+
+  #[inline]
+  pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
+    Layout { _tab: table }
+  }
+  #[allow(unused_mut)]
+  pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>(
+    _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>,
+    args: &'args LayoutArgs<'args>
+  ) -> flatbuffers::WIPOffset<Layout<'bldr>> {
+    let mut builder = LayoutBuilder::new(_fbb);
+    builder.add_row_count(args.row_count);
+    if let Some(x) = args.segments { builder.add_segments(x); }
+    if let Some(x) = args.children { builder.add_children(x); }
+    if let Some(x) = args.metadata { builder.add_metadata(x); }
+    builder.add_encoding(args.encoding);
+    builder.finish()
+  }
+
+
+  /// The ID of the encoding used for this Layout.
+  #[inline]
+  pub fn encoding(&self) -> u16 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u16>(Layout::VT_ENCODING, Some(0)).unwrap()}
+  }
+  /// The number of rows of data represented by this Layout.
+  #[inline]
+  pub fn row_count(&self) -> u64 {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<u64>(Layout::VT_ROW_COUNT, Some(0)).unwrap()}
+  }
+  /// Any additional metadata this layout needs to interpret its children.
+  /// This does not include data-specific metadata, which the layout should store in a segment.
+  #[inline]
+  pub fn metadata(&self) -> Option<flatbuffers::Vector<'a, u8>> {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'a, u8>>>(Layout::VT_METADATA, None)}
+  }
+  /// The children of this Layout.
+  #[inline]
+  pub fn children(&self) -> Option<flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset<Layout<'a>>>> {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset<Layout>>>>(Layout::VT_CHILDREN, None)}
+  }
+  /// Identifiers for each `Segment` of data required by this layout.
+  #[inline]
+  pub fn segments(&self) -> Option<flatbuffers::Vector<'a, u32>> {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'a, u32>>>(Layout::VT_SEGMENTS, None)}
+  }
+}
+
+impl flatbuffers::Verifiable for Layout<'_> {
+  #[inline]
+  fn run_verifier(
+    v: &mut flatbuffers::Verifier, pos: usize
+  ) -> Result<(), flatbuffers::InvalidFlatbuffer> {
+    use self::flatbuffers::Verifiable;
+    v.visit_table(pos)?
+     .visit_field::<u16>("encoding", Self::VT_ENCODING, false)?
+     .visit_field::<u64>("row_count", Self::VT_ROW_COUNT, false)?
+     .visit_field::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'_, u8>>>("metadata", Self::VT_METADATA, false)?
+     .visit_field::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'_, flatbuffers::ForwardsUOffset<Layout>>>>("children", Self::VT_CHILDREN, false)?
+     .visit_field::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'_, u32>>>("segments", Self::VT_SEGMENTS, false)?
+     .finish();
+    Ok(())
+  }
+}
+pub struct LayoutArgs<'a> {
+    pub encoding: u16,
+    pub row_count: u64,
+    pub metadata: Option<flatbuffers::WIPOffset<flatbuffers::Vector<'a, u8>>>,
+    pub children: Option<flatbuffers::WIPOffset<flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset<Layout<'a>>>>>,
+    pub segments: Option<flatbuffers::WIPOffset<flatbuffers::Vector<'a, u32>>>,
+}
+impl<'a> Default for LayoutArgs<'a> {
+  #[inline]
+  fn default() -> Self {
+    LayoutArgs {
+      encoding: 0,
+      row_count: 0,
+      metadata: None,
+      children: None,
+      segments: None,
+    }
+  }
+}
+
+pub struct LayoutBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> {
+  fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>,
+  start_: flatbuffers::WIPOffset<flatbuffers::TableUnfinishedWIPOffset>,
+}
+impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> LayoutBuilder<'a, 'b, A> {
+  #[inline]
+  pub fn add_encoding(&mut self, encoding: u16) {
+    self.fbb_.push_slot::<u16>(Layout::VT_ENCODING, encoding, 0);
+  }
+  #[inline]
+  pub fn add_row_count(&mut self, row_count: u64) {
+    self.fbb_.push_slot::<u64>(Layout::VT_ROW_COUNT, row_count, 0);
+  }
+  #[inline]
+  pub fn add_metadata(&mut self, metadata: flatbuffers::WIPOffset<flatbuffers::Vector<'b , u8>>) {
+    self.fbb_.push_slot_always::<flatbuffers::WIPOffset<_>>(Layout::VT_METADATA, metadata);
+  }
+  #[inline]
+  pub fn add_children(&mut self, children: flatbuffers::WIPOffset<flatbuffers::Vector<'b , flatbuffers::ForwardsUOffset<Layout<'b >>>>) {
+    self.fbb_.push_slot_always::<flatbuffers::WIPOffset<_>>(Layout::VT_CHILDREN, children);
+  }
+  #[inline]
+  pub fn add_segments(&mut self, segments: flatbuffers::WIPOffset<flatbuffers::Vector<'b , u32>>) {
+    self.fbb_.push_slot_always::<flatbuffers::WIPOffset<_>>(Layout::VT_SEGMENTS, segments);
+  }
+  #[inline]
+  pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> LayoutBuilder<'a, 'b, A> {
+    let start = _fbb.start_table();
+    LayoutBuilder {
+      fbb_: _fbb,
+      start_: start,
+    }
+  }
+  #[inline]
+  pub fn finish(self) -> flatbuffers::WIPOffset<Layout<'a>> {
+    let o = self.fbb_.end_table(self.start_);
+    flatbuffers::WIPOffset::new(o.value())
+  }
+}
+
+impl core::fmt::Debug for Layout<'_> {
+  fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+    let mut ds = f.debug_struct("Layout");
+      ds.field("encoding", &self.encoding());
+      ds.field("row_count", &self.row_count());
+      ds.field("metadata", &self.metadata());
+      ds.field("children", &self.children());
+      ds.field("segments", &self.segments());
+      ds.finish()
+  }
+}
+#[inline]
+/// Verifies that a buffer of bytes contains a `Layout`
+/// and returns it.
+/// Note that verification is still experimental and may not
+/// catch every error, or be maximally performant. For the
+/// previous, unchecked, behavior use
+/// `root_as_layout_unchecked`.
+pub fn root_as_layout(buf: &[u8]) -> Result<Layout, flatbuffers::InvalidFlatbuffer> {
+  flatbuffers::root::<Layout>(buf)
+}
+#[inline]
+/// Verifies that a buffer of bytes contains a size prefixed
+/// `Layout` and returns it.
+/// Note that verification is still experimental and may not
+/// catch every error, or be maximally performant. For the
+/// previous, unchecked, behavior use
+/// `size_prefixed_root_as_layout_unchecked`.
+pub fn size_prefixed_root_as_layout(buf: &[u8]) -> Result<Layout, flatbuffers::InvalidFlatbuffer> {
+  flatbuffers::size_prefixed_root::<Layout>(buf)
+}
+#[inline]
+/// Verifies, with the given options, that a buffer of bytes
+/// contains a `Layout` and returns it.
+/// Note that verification is still experimental and may not
+/// catch every error, or be maximally performant. For the
+/// previous, unchecked, behavior use
+/// `root_as_layout_unchecked`.
+pub fn root_as_layout_with_opts<'b, 'o>(
+  opts: &'o flatbuffers::VerifierOptions,
+  buf: &'b [u8],
+) -> Result<Layout<'b>, flatbuffers::InvalidFlatbuffer> {
+  flatbuffers::root_with_opts::<Layout<'b>>(opts, buf)
+}
+#[inline]
+/// Verifies, with the given verifier options, that a buffer of
+/// bytes contains a size prefixed `Layout` and returns
+/// it. Note that verification is still experimental and may not
+/// catch every error, or be maximally performant. For the
+/// previous, unchecked, behavior use
+/// `root_as_layout_unchecked`.
+pub fn size_prefixed_root_as_layout_with_opts<'b, 'o>(
+  opts: &'o flatbuffers::VerifierOptions,
+  buf: &'b [u8],
+) -> Result<Layout<'b>, flatbuffers::InvalidFlatbuffer> {
+  flatbuffers::size_prefixed_root_with_opts::<Layout<'b>>(opts, buf)
+}
+#[inline]
+/// Assumes, without verification, that a buffer of bytes contains a Layout and returns it.
+/// # Safety
+/// Callers must trust the given bytes do indeed contain a valid `Layout`.
+pub unsafe fn root_as_layout_unchecked(buf: &[u8]) -> Layout {
+  flatbuffers::root_unchecked::<Layout>(buf)
+}
+#[inline]
+/// Assumes, without verification, that a buffer of bytes contains a size prefixed Layout and returns it.
+/// # Safety
+/// Callers must trust the given bytes do indeed contain a valid size prefixed `Layout`.
+pub unsafe fn size_prefixed_root_as_layout_unchecked(buf: &[u8]) -> Layout {
+  flatbuffers::size_prefixed_root_unchecked::<Layout>(buf)
+}
+#[inline]
+pub fn finish_layout_buffer<'a, 'b, A: flatbuffers::Allocator + 'a>(
+    fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>,
+    root: flatbuffers::WIPOffset<Layout<'a>>) {
+  fbb.finish(root, None);
+}
+
+#[inline]
+pub fn finish_size_prefixed_layout_buffer<'a, 'b, A: flatbuffers::Allocator + 'a>(fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, root: flatbuffers::WIPOffset<Layout<'a>>) {
+  fbb.finish_size_prefixed(root, None);
+}

vortex-flatbuffers/src/generated/layout.rs

@@ -92,6 +92,28 @@ pub mod scalar;
 /// ```
 pub mod footer;
 
+#[cfg(feature = "layout")]
+#[allow(clippy::all)]
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[allow(clippy::many_single_char_names)]
+#[allow(clippy::unwrap_used)]
+#[allow(dead_code)]
+#[allow(non_snake_case)]
+#[allow(non_camel_case_types)]
+#[allow(unsafe_op_in_unsafe_fn)]
+#[allow(unused_imports)]
+#[allow(unused_lifetimes)]
+#[allow(unused_qualifications)]
+#[rustfmt::skip]
+#[path = "./generated/layout.rs"]
+/// A serialized sequence of arrays, each with its buffers.
+///
+/// `layout.fbs`:
+/// ```flatbuffers
+#[doc = include_str!("../flatbuffers/vortex-layout/layout.fbs")]
+/// ```
+pub mod layout;
+
 #[cfg(feature = "ipc")]
 #[allow(clippy::all)]
 #[allow(clippy::derive_partial_eq_without_eq)]