feat[duckdb]: optimize array (#5738)

Signed-off-by: Joe Isaacs <[email protected]>

Author: joseph-isaacs

vortex-duckdb/src/scan.rs

@@ -26,6 +26,7 @@ use url::Url;
 use vortex::VortexSessionDefault;
 use vortex::array::ArrayRef;
 use vortex::array::ToCanonical;
+use vortex::array::optimizer::ArrayOptimizer;
 use vortex::dtype::FieldNames;
 use vortex::error::VortexExpect;
 use vortex::error::VortexResult;
@@ -43,6 +44,7 @@ use vortex::file::VortexFile;
 use vortex::file::VortexOpenOptions;
 use vortex::io::runtime::BlockingRuntime;
 use vortex::io::runtime::current::ThreadSafeIterator;
+use vortex::layout::layouts::USE_VORTEX_OPERATORS;
 use vortex::session::VortexSession;
 
 use crate::RUNTIME;
@@ -327,6 +329,12 @@ impl TableFunction for VortexTableFunction {
 
                 let (array_result, conversion_cache) = result?;
 
+                let array_result = if *USE_VORTEX_OPERATORS {
+                    array_result.optimize_recursive()?
+                } else {
+                    array_result
+                };
+
                 local_state.exporter = Some(ArrayExporter::try_new(
                     &array_result.to_struct(),
                     &conversion_cache,