add docs explaining the u16 limitation

Signed-off-by: Connor Tsui <[email protected]>

Author: connortsui20

vortex-layout/src/layouts/dict/writer.rs

@@ -49,11 +49,21 @@ use crate::sequence::SequentialStreamAdapter;
 use crate::sequence::SequentialStreamExt;
 
 /// Constraints for dictionary layout encoding.
+///
+/// Note that [`max_len`](Self::max_len) is limited to `u16` (65,535 entries) by design. Since
+/// layout chunks are typically ~8k elements, having more than 64k unique values in a dictionary
+/// means dictionary encoding provides little compression benefit. If a column has very high
+/// cardinality, the fallback encoding strategy should be used instead.
 #[derive(Clone)]
 pub struct DictLayoutConstraints {
     /// Maximum size of the dictionary in bytes.
     pub max_bytes: usize,
-    /// Maximum dictionary length.
+    /// Maximum dictionary length. Limited to `u16` because dictionaries with more than 64k unique
+    /// values provide diminishing compression returns given typical chunk sizes (~8k elements).
+    ///
+    /// The codes dtype is chosen dynamically based on the actual dictionary size:
+    /// - [`PType::U8`] when the dictionary has at most 255 entries
+    /// - [`PType::U16`] when the dictionary has more than 255 entries
     pub max_len: u16,
 }