Implement regexp optimization for .*foo.* patterns

Author: petern48

Cargo.lock

@@ -49,6 +49,7 @@ chrono = { workspace = true }
 datafusion-common = { workspace = true, default-features = true }
 datafusion-expr = { workspace = true }
 datafusion-expr-common = { workspace = true }
+datafusion-functions = { workspace = true }
 datafusion-physical-expr = { workspace = true }
 indexmap = { workspace = true }
 itertools = { workspace = true }

datafusion/optimizer/Cargo.toml

@@ -2366,6 +2366,7 @@ mod tests {
         interval_arithmetic::Interval,
         *,
     };
+    use datafusion_functions::expr_fn::contains as contains_fn;
     use datafusion_functions_window_common::field::WindowUDFFieldArgs;
     use datafusion_functions_window_common::partition::PartitionEvaluatorArgs;
     use datafusion_physical_expr::PhysicalExpr;
@@ -3381,6 +3382,19 @@ mod tests {
             col("c1").like(lit("%foo%")),
         );
 
+        // regular expression that matches a substring
+        assert_change(
+            regex_match(col("c1"), lit(".*foo.*")),
+            contains_fn(col("c1"), lit("foo")),
+        );
+
+        assert_change(
+            regex_not_match(col("c1"), lit(".*foo.*")),
+            Expr::Not(Box::new(contains_fn(col("c1"), lit("foo")))),
+        );
+
+        assert_change(regex_match(col("c1"), lit(".*.*")), col("c1").is_not_null());
+
         // regular expressions that match an exact literal
         assert_change(regex_match(col("c1"), lit("^$")), col("c1").eq(lit("")));
         assert_change(

datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs

@@ -18,6 +18,7 @@
 use datafusion_common::tree_node::Transformed;
 use datafusion_common::{DataFusionError, Result};
 use datafusion_expr::{BinaryExpr, Expr, Like, Operator, lit};
+use datafusion_functions::expr_fn::contains;
 use regex_syntax::hir::{Capture, Hir, HirKind, Literal, Look};
 
 use crate::simplify_expressions::expr_simplifier::StringScalar;
@@ -34,6 +35,8 @@ const ANY_CHAR_REGEX_PATTERN: &str = ".*";
 ///
 /// Typical cases this function can simplify:
 /// - empty regex pattern to `LIKE '%'`
+/// - `EQ .*foo.*` to `contains(left, "foo")`
+/// - `NE .*foo.*` to `NOT contains(left, "foo")`
 /// - literal regex patterns to `LIKE '%foo%'`
 /// - full anchored regex patterns (e.g. `^foo$`) to `= 'foo'`
 /// - partial anchored regex patterns (e.g. `^foo`) to `LIKE 'foo%'`
@@ -82,6 +85,32 @@ pub fn simplify_regex_expr(
         return Ok(Transformed::yes(new_expr));
     }
 
+    // Convert patterns of the form ".*foo.*" to `contains(left, "foo")`
+    if !mode.i
+        && let Some(inner) = pattern
+            // If pattern starts and ends with ".*"
+            .strip_prefix(ANY_CHAR_REGEX_PATTERN)
+            .and_then(|rest| rest.strip_suffix(ANY_CHAR_REGEX_PATTERN))
+            // If inner is all non-special characters
+            && inner.chars().all(|x| !is_special_character(x))
+    {
+        let new_expr = match (mode.not, inner.is_empty()) {
+            // contains(left, inner)
+            (false, false) => contains(*left, lit(inner)),
+            (false, true) => left.is_not_null(),
+            // not (contains(left, inner))
+            (true, false) => Expr::Not(Box::new(contains(*left, lit(inner)))),
+            (true, true) => {
+                return Ok(Transformed::no(Expr::BinaryExpr(BinaryExpr {
+                    left,
+                    op,
+                    right,
+                })));
+            }
+        };
+        return Ok(Transformed::yes(new_expr));
+    }
+
     match regex_syntax::Parser::new().parse(pattern) {
         Ok(hir) => {
             let kind = hir.kind();
@@ -202,6 +231,25 @@ fn is_safe_for_like(c: char) -> bool {
     (c != '%') && (c != '_')
 }
 
+fn is_special_character(c: char) -> bool {
+    matches!(
+        c,
+        '.' | '*'
+            | '+'
+            | '?'
+            | '|'
+            | '('
+            | ')'
+            | '['
+            | ']'
+            | '{'
+            | '}'
+            | '^'
+            | '$'
+            | '\\'
+    )
+}
+
 /// Returns true if the elements in a `Concat` pattern are:
 /// - `[Look::Start, Look::End]`
 /// - `[Look::Start, Literal(_), Look::End]`

datafusion/optimizer/src/simplify_expressions/regex.rs

@@ -58,7 +58,56 @@ c
 
 query T
 select b from t where b !~ '.*'
+
+# test regex .*literal.* simplifies to contains()
+query TT
+explain select b from t where b ~ '.*a.*'
+----
+logical_plan
+01)Filter: contains(t.b, Utf8("a"))
+02)--TableScan: t projection=[b]
+physical_plan
+01)FilterExec: contains(b@0, a)
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+query T
+select b from t where b ~ '.*a.*'
 ----
+a
+
+query TT
+explain select b from t where b !~ '.*a.*'
+----
+logical_plan
+01)Filter: NOT contains(t.b, Utf8("a"))
+02)--TableScan: t projection=[b]
+physical_plan
+01)FilterExec: NOT contains(b@0, a)
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+query T
+select b from t where b !~ '.*a.*'
+----
+c
+
+query TT
+explain select b from t where b ~ '.*.*'
+----
+logical_plan
+01)Filter: t.b IS NOT NULL
+02)--TableScan: t projection=[b]
+physical_plan
+01)FilterExec: b@0 IS NOT NULL
+02)--DataSourceExec: partitions=1, partition_sizes=[1]
+
+query T
+select b from t where b ~ '.*.*'
+----
+a
+c
+
+query T
+select b from t where b !~ '.*.*'
 
 query TT
 explain select * from t where a = a;