Dictionary execution (#5791)

Signed-off-by: Nicholas Gates <[email protected]>

Author: gatesn

vortex-layout/src/layouts/dict/reader.rs

@@ -92,6 +92,7 @@ impl DictReader {
         // We capture the name, so it may be wrong if we re-use the same reader within multiple
         // different parent readers. But that's rare...
         let values_len = self.values_len;
+        let session = self.session.clone();
         self.values_array
             .get_or_init(move || {
                 self.values
@@ -102,7 +103,15 @@ impl DictReader {
                     )
                     .vortex_expect("must construct dict values array evaluation")
                     .map_err(Arc::new)
-                    .map_ok(|arr| arr.to_canonical().into_array())
+                    .map(move |array| {
+                        if *USE_VORTEX_OPERATORS {
+                            // We execute the array to avoid re-evaluating for every split.
+                            let array = array?;
+                            Ok(array.execute_vector(&session)?.into_array(array.dtype()))
+                        } else {
+                            Ok(array?.to_canonical().into_array())
+                        }
+                    })
                     .boxed()
                     .shared()
             })

vortex-layout/src/layouts/flat/reader.rs

@@ -37,16 +37,6 @@ use crate::segments::SegmentSource;
 //  actual expression? Perhaps all expressions are given a selection mask to decide for themselves?
 const EXPR_EVAL_THRESHOLD: f64 = 0.2;
 
-/// Below this mask density we will propagate filters one by one. In other words, we filter an
-/// array using a mask prior to running a filter expression, and then have to perform a more
-/// expensive rank intersection on the result. This threshold exists because filtering has a
-/// non-trivial cost, and often that cost outweighs evaluating the filter expression over a few
-/// more rows that are already known to be false.
-///
-/// TODO(ngates): this threshold should really be estimated based on the cost of the filter + the
-///  the cost of the expression itself.
-const FILTER_OF_FILTER_THRESHOLD: f64 = 0.8;
-
 pub struct FlatReader {
     layout: FlatLayout,
     name: Arc<str>,
@@ -157,7 +147,7 @@ impl LayoutReader for FlatReader {
             }
 
             let array_mask = if *USE_VORTEX_OPERATORS {
-                if mask.density() < FILTER_OF_FILTER_THRESHOLD {
+                if mask.density() < EXPR_EVAL_THRESHOLD {
                     // We have the choice to apply the filter or the expression first, we apply the
                     // expression first so that it can try pushing down itself and then the filter
                     // after this.