[Variant] Add variant docs and examples (#7661)

# Which issue does this PR close?

We generally require a GitHub issue to be filed for all bug fixes and
enhancements and this helps us generate change logs for our releases.
You can link an issue to this PR using the GitHub syntax.

- Follow on to #7644
- Part of #6736

# Rationale for this change

Using the parquet APIs came up in
#7644 (comment) so I
wanted to help contribute some additional documentation / tests

# What changes are included in this PR?

Add documentation and tests about `Variant`, specifically some examples
of how to create `Variant` values

# Are there any user-facing changes?

More docs

---------

Co-authored-by: Ryan Johnson <[email protected]>

Author: alamb

parquet-variant/src/variant.rs

@@ -25,6 +25,7 @@ use arrow_schema::ArrowError;
 use chrono::{DateTime, NaiveDate, NaiveDateTime, Utc};
 use std::num::TryFromIntError;
 
+/// The number of bytes used to store offsets in the [`VariantMetadataHeader`]
 #[derive(Clone, Debug, Copy, PartialEq)]
 enum OffsetSizeBytes {
     One = 1,
@@ -91,7 +92,7 @@ impl OffsetSizeBytes {
     }
 }
 
-/// A parsed version of the variant metadata header byte.
+/// Header structure for [`VariantMetadata`]
 #[derive(Clone, Debug, Copy, PartialEq)]
 pub(crate) struct VariantMetadataHeader {
     version: u8,
@@ -140,8 +141,12 @@ impl VariantMetadataHeader {
     }
 }
 
+/// [`Variant`] Metadata
+///
+/// See the [Variant Spec] file for more information
+///
+/// [Variant Spec]: https://github.com/apache/parquet-format/blob/master/VariantEncoding.md#metadata-encoding
 #[derive(Clone, Copy, Debug, PartialEq)]
-/// Encodes the Variant Metadata, see the Variant spec file for more information
 pub struct VariantMetadata<'m> {
     bytes: &'m [u8],
     header: VariantMetadataHeader,
@@ -238,7 +243,7 @@ impl<'m> VariantMetadata<'m> {
     }
 }
 
-/// A parsed version of the variant object value header byte.
+/// Header structure for [`VariantObject`]
 #[derive(Clone, Debug, PartialEq)]
 pub(crate) struct VariantObjectHeader {
     field_offset_size: OffsetSizeBytes,
@@ -262,6 +267,7 @@ impl VariantObjectHeader {
     }
 }
 
+/// A [`Variant`] Object (struct with named fields).
 #[derive(Clone, Debug, PartialEq)]
 pub struct VariantObject<'m, 'v> {
     pub metadata: VariantMetadata<'m>,
@@ -282,6 +288,7 @@ impl<'m, 'v> VariantObject<'m, 'v> {
     /// particular, that all field ids exist in `metadata`, and all offsets are in-bounds and point
     /// to valid objects.
     // TODO: How to make the validation non-recursive while still making iterators safely infallible??
+    // See https://github.com/apache/arrow-rs/issues/7711
     pub fn try_new(metadata: VariantMetadata<'m>, value: &'v [u8]) -> Result<Self, ArrowError> {
         let header_byte = first_byte_from_slice(value)?;
         let header = VariantObjectHeader::try_new(header_byte)?;
@@ -420,10 +427,10 @@ impl VariantListHeader {
     }
 }
 
-/// Represents a variant array.
+/// [`Variant`] Array.
 ///
 /// NOTE: The "list" naming differs from the variant spec -- which calls it "array" -- in order to be
-/// consistent with parquet and arrow type naming. Otherwise, the name would conflict with the
+/// consistent with Parquet and Arrow type naming. Otherwise, the name would conflict with the
 /// `VariantArray : Array` we must eventually define for variant-typed arrow arrays.
 #[derive(Clone, Debug, PartialEq)]
 pub struct VariantList<'m, 'v> {
@@ -443,6 +450,7 @@ impl<'m, 'v> VariantList<'m, 'v> {
     /// This constructor verifies that `value` points to a valid variant array value. In particular,
     /// that all offsets are in-bounds and point to valid objects.
     // TODO: How to make the validation non-recursive while still making iterators safely infallible??
+    // See https://github.com/apache/arrow-rs/issues/7711
     pub fn try_new(metadata: VariantMetadata<'m>, value: &'v [u8]) -> Result<Self, ArrowError> {
         let header_byte = first_byte_from_slice(value)?;
         let header = VariantListHeader::try_new(header_byte)?;
@@ -536,33 +544,134 @@ impl<'m, 'v> VariantList<'m, 'v> {
     }
 }
 
-/// Variant value. May contain references to metadata and value
+/// Represents a [Parquet Variant]
+///
+/// The lifetimes `'m` and `'v` are for metadata and value buffers, respectively.
+///
+/// # Background
+///
+/// The [specification] says:
+///
+/// The Variant Binary Encoding allows representation of semi-structured data
+/// (e.g. JSON) in a form that can be efficiently queried by path. The design is
+/// intended to allow efficient access to nested data even in the presence of
+/// very wide or deep structures.
+///
+/// Another motivation for the representation is that (aside from metadata) each
+/// nested Variant value is contiguous and self-contained. For example, in a
+/// Variant containing an Array of Variant values, the representation of an
+/// inner Variant value, when paired with the metadata of the full variant, is
+/// itself a valid Variant.
+///
+/// When stored in Parquet files, Variant fields can also be *shredded*. Shredding
+/// refers to extracting some elements of the variant into separate columns for
+/// more efficient extraction/filter pushdown. The [Variant Shredding
+/// specification] describes the details of shredding Variant values as typed
+/// Parquet columns.
+///
+/// A Variant represents a type that contains one of:
+///
+/// * Primitive: A type and corresponding value (e.g. INT, STRING)
+///
+/// * Array: An ordered list of Variant values
+///
+/// * Object: An unordered collection of string/Variant pairs (i.e. key/value
+///   pairs). An object may not contain duplicate keys.
+///
+/// # Encoding
+///
+/// A Variant is encoded with 2 binary values, the value and the metadata. The
+/// metadata stores a header and an optional dictionary of field names which are
+/// referred to by offset in the value. The value is a binary representation of
+/// the actual data, and varies depending on the type.
+///
+/// # Design Goals
+///
+/// The design goals of the Rust API are as follows:
+/// 1. Speed / Zero copy access (no `clone`ing is required)
+/// 2. Safety
+/// 3. Follow standard Rust conventions
+///
+/// [Parquet Variant]: https://github.com/apache/parquet-format/blob/master/VariantEncoding.md
+/// [specification]: https://github.com/apache/parquet-format/blob/master/VariantEncoding.md
+/// [Variant Shredding specification]: https://github.com/apache/parquet-format/blob/master/VariantShredding.md
+///
+/// # Examples:
+///
+/// ## Creating `Variant` from Rust Types
+/// ```
+/// # use parquet_variant::Variant;
+/// // variants can be directly constructed
+/// let variant = Variant::Int32(123);
+/// // or constructed via `From` impls
+/// assert_eq!(variant, Variant::from(123i32));
+/// ```
+/// ## Creating `Variant` from metadata and value
+/// ```
+/// # use parquet_variant::{Variant, VariantMetadata};
+/// let metadata = [0x01, 0x00, 0x00];
+/// let value = [0x09, 0x48, 0x49];
+/// // parse the header metadata
+/// assert_eq!(
+///   Variant::ShortString("HI"),
+///   Variant::try_new(&metadata, &value).unwrap()
+/// );
+/// ```
+///
+/// ## Using `Variant` values
+/// ```
+/// # use parquet_variant::Variant;
+/// # let variant = Variant::Int32(123);
+/// // variants can be used in match statements like normal enums
+/// match variant {
+///   Variant::Int32(i) => println!("Integer: {}", i),
+///   Variant::String(s) => println!("String: {}", s),
+///   _ => println!("Other variant"),
+/// }
+/// ```
 #[derive(Clone, Debug, PartialEq)]
 pub enum Variant<'m, 'v> {
-    // TODO: Add types for the rest of the primitive types, once API is agreed upon
+    /// Primitive type: Null
     Null,
+    /// Primitive (type_id=1): INT(8, SIGNED)
     Int8(i8),
+    /// Primitive (type_id=1): INT(16, SIGNED)
     Int16(i16),
+    /// Primitive (type_id=1): INT(32, SIGNED)
     Int32(i32),
+    /// Primitive (type_id=1): INT(64, SIGNED)
     Int64(i64),
+    /// Primitive (type_id=1): DATE
     Date(NaiveDate),
+    /// Primitive (type_id=1): TIMESTAMP(isAdjustedToUTC=true, MICROS)
     TimestampMicros(DateTime<Utc>),
+    /// Primitive (type_id=1): TIMESTAMP(isAdjustedToUTC=false, MICROS)
     TimestampNtzMicros(NaiveDateTime),
+    /// Primitive (type_id=1): DECIMAL(precision, scale) 32-bits
     Decimal4 { integer: i32, scale: u8 },
+    /// Primitive (type_id=1): DECIMAL(precision, scale) 64-bits
     Decimal8 { integer: i64, scale: u8 },
+    /// Primitive (type_id=1): DECIMAL(precision, scale) 128-bits
     Decimal16 { integer: i128, scale: u8 },
+    /// Primitive (type_id=1): FLOAT
     Float(f32),
+    /// Primitive (type_id=1): DOUBLE
     Double(f64),
+    /// Primitive (type_id=1): BOOLEAN (true)
     BooleanTrue,
+    /// Primitive (type_id=1): BOOLEAN (false)
     BooleanFalse,
-
-    // Note: only need the *value* buffer
+    // Note: only need the *value* buffer for these types
+    /// Primitive (type_id=1): BINARY
     Binary(&'v [u8]),
+    /// Primitive (type_id=1): STRING
     String(&'v str),
+    /// Short String (type_id=2): STRING
     ShortString(&'v str),
-
     // need both metadata & value
+    /// Object (type_id=3): N/A
     Object(VariantObject<'m, 'v>),
+    /// Array (type_id=4): N/A
     List(VariantList<'m, 'v>),
 }
 
@@ -574,6 +683,7 @@ impl<'m, 'v> Variant<'m, 'v> {
     /// # use parquet_variant::{Variant, VariantMetadata};
     /// let metadata = [0x01, 0x00, 0x00];
     /// let value = [0x09, 0x48, 0x49];
+    /// // parse the header metadata
     /// assert_eq!(
     ///   Variant::ShortString("HI"),
     ///   Variant::try_new(&metadata, &value).unwrap()
@@ -629,7 +739,6 @@ impl<'m, 'v> Variant<'m, 'v> {
                 }
                 VariantPrimitiveType::BooleanTrue => Variant::BooleanTrue,
                 VariantPrimitiveType::BooleanFalse => Variant::BooleanFalse,
-                // TODO: Add types for the rest, once API is agreed upon
                 VariantPrimitiveType::Date => Variant::Date(decoder::decode_date(value_data)?),
                 VariantPrimitiveType::TimestampMicros => {
                     Variant::TimestampMicros(decoder::decode_timestamp_micros(value_data)?)