fix: filter

Signed-off-by: Alexander Droste <[email protected]>

Author: 0ax1

vortex-layout/src/layouts/flat/reader.rs

@@ -37,16 +37,6 @@ use crate::segments::SegmentSource;
 //  actual expression? Perhaps all expressions are given a selection mask to decide for themselves?
 const EXPR_EVAL_THRESHOLD: f64 = 0.2;
 
-/// Below this mask density we will propagate filters one by one. In other words, we filter an
-/// array using a mask prior to running a filter expression, and then have to perform a more
-/// expensive rank intersection on the result. This threshold exists because filtering has a
-/// non-trivial cost, and often that cost outweighs evaluating the filter expression over a few
-/// more rows that are already known to be false.
-///
-/// TODO(ngates): this threshold should really be estimated based on the cost of the filter + the
-///  the cost of the expression itself.
-const FILTER_OF_FILTER_THRESHOLD: f64 = 0.8;
-
 pub struct FlatReader {
     layout: FlatLayout,
     name: Arc<str>,
@@ -157,20 +147,13 @@ impl LayoutReader for FlatReader {
             }
 
             let array_mask = if *USE_VORTEX_OPERATORS {
-                if mask.density() < FILTER_OF_FILTER_THRESHOLD {
-                    // Run only over the pre-filtered rows.
-                    let array = array.filter(mask.clone())?;
-                    let array = array.apply(&expr)?;
-                    let array_mask = array.execute_mask(&session)?;
-
-                    mask.intersect_by_rank(&array_mask)
-                } else {
-                    // Run over the full array, with a simpler bitand at the end.
-                    let array = array.apply(&expr)?;
-                    let array_mask = array.execute_mask(&session)?;
+                // Always apply the expression to the full array first, avoiding the overhead of
+                // premature filter materialization. The operators can optimize the full expression
+                // tree more effectively than if we fragment the work into filter + apply.
+                let array = array.apply(&expr)?;
+                let array_mask = array.execute_mask(&session)?;
 
-                    mask.bitand(&array_mask)
-                }
+                mask.bitand(&array_mask)
             } else {
                 // TODO(ngates): the mask may actually be dense within a range, as is often the case when
                 //  we have approximate mask results from a zone map. In which case we could look at