Add ProjectionAboveLimit optimization

Author: srh

Cargo.toml

@@ -30,3 +30,9 @@ members = [
 ]
 
 exclude = ["python"]
+
+
+[patch.'https://github.com/cube-js/arrow-rs']
+# parquet = { path = "/Users/sam/arrow-rs", features = ["arrow"] }
+parquet = { path = "/Users/sam/arrow-rs/parquet" }
+arrow = { path = "/Users/sam/arrow-rs/arrow" }

datafusion/src/execution/context.rs

@@ -61,6 +61,7 @@ use crate::optimizer::constant_folding::ConstantFolding;
 use crate::optimizer::filter_push_down::FilterPushDown;
 use crate::optimizer::limit_push_down::LimitPushDown;
 use crate::optimizer::optimizer::OptimizerRule;
+use crate::optimizer::projection_above_limit::ProjectionAboveLimit;
 use crate::optimizer::projection_push_down::ProjectionPushDown;
 use crate::optimizer::simplify_expressions::SimplifyExpressions;
 use crate::physical_optimizer::merge_exec::AddCoalescePartitionsExec;
@@ -700,6 +701,7 @@ impl Default for ExecutionConfig {
                 Arc::new(HashBuildProbeOrder::new()),
                 Arc::new(LimitPushDown::new()),
                 Arc::new(FoldCrossJoinAggregate {}), // CubeStore extension.
+                Arc::new(ProjectionAboveLimit::new()),     // CubeStore extension.
             ],
             physical_optimizers: vec![
                 // NOTE: disabled in the CubeStore fork.

datafusion/src/optimizer/mod.rs

@@ -23,6 +23,7 @@ pub mod constant_folding;
 pub mod eliminate_limit;
 pub mod filter_push_down;
 pub mod hash_build_probe_order;
+pub mod projection_above_limit;
 pub mod limit_push_down;
 pub mod optimizer;
 pub mod projection_push_down;