Implement String View (Utf8View/BinaryView) Optimizations

Introduces a two-stage filter for ByteView types. Stage 1 uses a fast DirectProbeFilter on masked views (len + prefix) for quick rejection; Stage 2 performs full verification only for potential long-string matches. Triggers for Utf8View and BinaryView.

Author: geoffreyclaude

datafusion/physical-expr/src/expressions/in_list/primitive_filter.rs

@@ -370,6 +370,16 @@ where
         ))
     }
 
+    /// Creates a DirectProbeFilter from an iterator of values.
+    ///
+    /// This is useful when building the filter from pre-processed values
+    /// (e.g., masked views for Utf8View).
+    pub(crate) fn from_values(values: impl Iterator<Item = T::Native>) -> Self {
+        // Collect into HashSet for deduplication
+        let unique_values: HashSet<_> = values.collect();
+        Self::from_values_inner(unique_values.into_iter(), 0)
+    }
+
     /// Internal constructor from deduplicated values
     fn from_values_inner(
         unique_values: impl Iterator<Item = T::Native>,

datafusion/physical-expr/src/expressions/in_list/result.rs

@@ -21,8 +21,6 @@
 //! from IN list membership tests, handling null propagation correctly
 //! according to SQL three-valued logic.
 
-#![expect(dead_code)]
-
 use arrow::array::BooleanArray;
 use arrow::buffer::{BooleanBuffer, NullBuffer};
 

datafusion/physical-expr/src/expressions/in_list/strategy.rs

@@ -28,7 +28,9 @@ use super::primitive_filter::*;
 use super::result::handle_dictionary;
 use super::static_filter::StaticFilter;
 use super::transform::{
-    make_bitmap_filter, make_branchless_filter, reinterpret_any_primitive_to,
+    make_bitmap_filter, make_branchless_filter, make_byte_view_masked_filter,
+    make_utf8view_branchless_filter, make_utf8view_hash_filter,
+    reinterpret_any_primitive_to, utf8view_all_short_strings,
 };
 
 // =============================================================================
@@ -104,6 +106,16 @@ pub(crate) fn instantiate_static_filter(
 
     let len = in_array.len();
     let dt = in_array.data_type();
+
+    // Special case: Utf8View with short strings can be reinterpreted as i128
+    if matches!(dt, DataType::Utf8View) && utf8view_all_short_strings(in_array.as_ref()) {
+        return if len <= BRANCHLESS_MAX_16B {
+            make_utf8view_branchless_filter(&in_array)
+        } else {
+            make_utf8view_hash_filter(&in_array)
+        };
+    }
+
     let strategy = select_strategy(dt, len);
 
     match (dt, strategy) {
@@ -124,6 +136,14 @@ pub(crate) fn instantiate_static_filter(
             exec_datafusion_err!("Hashed strategy selected but no filter for {:?}", dt)
         })?,
 
+        // Byte view filters (Utf8View, BinaryView)
+        (DataType::Utf8View, Generic) => {
+            make_byte_view_masked_filter::<StringViewType>(in_array)
+        }
+        (DataType::BinaryView, Generic) => {
+            make_byte_view_masked_filter::<BinaryViewType>(in_array)
+        }
+
         // Fallback for nested/complex types and strings (Phase 4: Strings use fallback)
         (_, Generic) => Ok(Arc::new(NestedTypeFilter::try_new(in_array)?)),
     }