Add optional flags argument to regex tests (#247)

* feat: add optional flags argument to regex tests

* fix: disable flags tests on BigQuery

* fix: downgrade regex config errors to warnings

Author: tconbeer

README.md

@@ -600,7 +600,10 @@ tests:
 ### [expect_column_values_to_match_regex](macros/schema_tests/string_matching/expect_column_values_to_match_regex.sql)
 
 Expect column entries to be strings that match a given regular expression. Valid matches can be found anywhere in the string, for example "[at]+" will identify the following strings as expected: "cat", "hat", "aa", "a", and "t", and the following strings as unexpected: "fish", "dog".
-Optionally, `is_raw` indicates the `regex` pattern is a "raw" string and should be escaped. The default is `False`.
+
+Optional (keyword) arguments:
+- `is_raw` indicates the `regex` pattern is a "raw" string and should be escaped. The default is `False`.
+- `flags` is a string of one or more characters that are passed to the regex engine as flags (or parameters). Allowed flags are adapter-specific. A common flag is `i`, for case-insensitive matching. The default is no flags.
 
 *Applies to:* Column
 
@@ -610,12 +613,16 @@ tests:
       regex: "[at]+"
       row_condition: "id is not null" # (Optional)
       is_raw: True # (Optional)
+      flags: i # (Optional)
 ```
 
 ### [expect_column_values_to_not_match_regex](macros/schema_tests/string_matching/expect_column_values_to_not_match_regex.sql)
 
 Expect column entries to be strings that do NOT match a given regular expression. The regex must not match any portion of the provided string. For example, "[at]+" would identify the following strings as expected: "fish”, "dog”, and the following as unexpected: "cat”, "hat”.
-Optionally, `is_raw` indicates the `regex` pattern is a "raw" string and should be escaped. The default is `False`.
+
+Optional (keyword) arguments:
+- `is_raw` indicates the `regex` pattern is a "raw" string and should be escaped. The default is `False`.
+- `flags` is a string of one or more characters that are passed to the regex engine as flags (or parameters). Allowed flags are adapter-specific. A common flag is `i`, for case-insensitive matching. The default is no flags.
 
 *Applies to:* Column
 
@@ -625,12 +632,16 @@ tests:
       regex: "[at]+"
       row_condition: "id is not null" # (Optional)
       is_raw: True # (Optional)
+      flags: i # (Optional)
 ```
 
 ### [expect_column_values_to_match_regex_list](macros/schema_tests/string_matching/expect_column_values_to_match_regex_list.sql)
 
 Expect the column entries to be strings that can be matched to either any of or all of a list of regular expressions. Matches can be anywhere in the string.
-Optionally, `is_raw` indicates the `regex` patterns are "raw" strings and should be escaped. The default is `False`.
+
+Optional (keyword) arguments:
+- `is_raw` indicates the `regex` pattern is a "raw" string and should be escaped. The default is `False`.
+- `flags` is a string of one or more characters that are passed to the regex engine as flags (or parameters). Allowed flags are adapter-specific. A common flag is `i`, for case-insensitive matching. The default is no flags.
 
 *Applies to:* Column
 
@@ -641,12 +652,16 @@ tests:
       match_on: any # (Optional. Default is 'any', which applies an 'OR' for each regex. If 'all', it applies an 'AND' for each regex.)
       row_condition: "id is not null" # (Optional)
       is_raw: True # (Optional)
+      flags: i # (Optional)
 ```
 
 ### [expect_column_values_to_not_match_regex_list](macros/schema_tests/string_matching/expect_column_values_to_not_match_regex_list.sql)
 
 Expect the column entries to be strings that do not match any of a list of regular expressions. Matches can be anywhere in the string.
-Optionally, `is_raw` indicates the `regex` patterns are "raw" strings and should be escaped. The default is `False`.
+
+Optional (keyword) arguments:
+- `is_raw` indicates the `regex` pattern is a "raw" string and should be escaped. The default is `False`.
+- `flags` is a string of one or more characters that are passed to the regex engine as flags (or parameters). Allowed flags are adapter-specific. A common flag is `i`, for case-insensitive matching. The default is no flags.
 
 *Applies to:* Column
 
@@ -657,6 +672,7 @@ tests:
       match_on: any # (Optional. Default is 'any', which applies an 'OR' for each regex. If 'all', it applies an 'AND' for each regex.)
       row_condition: "id is not null" # (Optional)
       is_raw: True # (Optional)
+      flags: i # (Optional)
 ```
 
 ### [expect_column_values_to_match_like_pattern](macros/schema_tests/string_matching/expect_column_values_to_match_like_pattern.sql)

integration_tests/models/schema_tests/schema.yml

@@ -6,12 +6,26 @@ models:
         tests:
           - dbt_expectations.expect_column_values_to_match_regex:
               regex: "@[^.]*"
+          - dbt_expectations.expect_column_values_to_match_regex:
+              regex: "[A-Z]"
+              flags: i
+              config:
+                enabled: "{{ target.type in ['postgres', 'snowflake', 'redshift' ] }}"
           - dbt_expectations.expect_column_values_to_not_match_regex:
               regex: "&[^.]*"
+          - dbt_expectations.expect_column_values_to_not_match_regex:
+              regex: "[A-Z]"
           - dbt_expectations.expect_column_values_to_match_regex_list:
               regex_list: ["@[^.]*", "&[^.]*"]
+          - dbt_expectations.expect_column_values_to_match_regex_list:
+              regex_list: ["[A-G]", "[H-Z]"]
+              flags: i
+              config:
+                enabled: "{{ target.type in ['postgres', 'snowflake', 'redshift' ] }}"
           - dbt_expectations.expect_column_values_to_not_match_regex_list:
               regex_list: ["@[^.]*", "&[^.]*"]
+          - dbt_expectations.expect_column_values_to_not_match_regex_list:
+              regex_list: ["[A-G]", "[H-Z]"]
           - dbt_expectations.expect_column_values_to_match_like_pattern:
               like_pattern: "%@%"
           - dbt_expectations.expect_column_values_to_not_match_like_pattern:

macros/regex/regexp_instr.sql

@@ -1,33 +1,58 @@
-{% macro regexp_instr(source_value, regexp, position=1, occurrence=1, is_raw=False) %}
+{% macro regexp_instr(source_value, regexp, position=1, occurrence=1, is_raw=False, flags="") %}
 
     {{ adapter.dispatch('regexp_instr', 'dbt_expectations')(
-        source_value, regexp, position, occurrence, is_raw
+        source_value, regexp, position, occurrence, is_raw, flags
     ) }}
 
 {% endmacro %}
 
-{% macro default__regexp_instr(source_value, regexp, position, occurrence, is_raw) %}
+{% macro default__regexp_instr(source_value, regexp, position, occurrence, is_raw, flags) %}
+{# unclear if other databases support raw strings or flags #}
+{% if is_raw or flags %}
+    {{ exceptions.warn(
+            "is_raw and flags options are not supported for this adapter "
+            ~ "and are being ignored."
+    ) }}
+{% endif %}
 regexp_instr({{ source_value }}, '{{ regexp }}', {{ position }}, {{ occurrence }})
 {% endmacro %}
 
 {# Snowflake uses $$...$$ to escape raw strings #}
-{% macro snowflake__regexp_instr(source_value, regexp, position, occurrence, is_raw) %}
+{% macro snowflake__regexp_instr(source_value, regexp, position, occurrence, is_raw, flags) %}
 {%- set regexp = "$$" ~ regexp ~ "$$" if is_raw else "'" ~ regexp ~ "'" -%}
-regexp_instr({{ source_value }}, {{ regexp }}, {{ position }}, {{ occurrence }})
+{% if flags %}{{ dbt_expectations._validate_flags(flags, 'cimes') }}{% endif %}
+regexp_instr({{ source_value }}, {{ regexp }}, {{ position }}, {{ occurrence }}, 0, '{{ flags }}')
 {% endmacro %}
 
 {# BigQuery uses "r" to escape raw strings #}
-{% macro bigquery__regexp_instr(source_value, regexp, position, occurrence, is_raw) %}
+{% macro bigquery__regexp_instr(source_value, regexp, position, occurrence, is_raw, flags) %}
+{% if flags %}
+    {{ exceptions.warn(
+            "The flags option is not supported for BigQuery and is being ignored."
+    ) }}
+{% endif %}
 {%- set regexp = "r'" ~ regexp ~ "'" if is_raw else "'" ~ regexp ~ "'" -%}
 regexp_instr({{ source_value }}, {{ regexp }}, {{ position }}, {{ occurrence }})
 {% endmacro %}
 
 {# Postgres does not need to escape raw strings #}
-{% macro postgres__regexp_instr(source_value, regexp, position, occurrence, is_raw) %}
-array_length((select regexp_matches({{ source_value }}, '{{ regexp }}')), 1)
+{% macro postgres__regexp_instr(source_value, regexp, position, occurrence, is_raw, flags) %}
+{% if flags %}{{ dbt_expectations._validate_flags(flags, 'bcegimnpqstwx') }}{% endif %}
+array_length((select regexp_matches({{ source_value }}, '{{ regexp }}', '{{ flags }}')), 1)
 {% endmacro %}
 
 {# Unclear what Redshift does to escape raw strings #}
-{% macro redshift__regexp_instr(source_value, regexp, position, occurrence, is_raw) %}
-regexp_instr({{ source_value }}, '{{ regexp }}', {{ position }}, {{ occurrence }})
+{% macro redshift__regexp_instr(source_value, regexp, position, occurrence, is_raw, flags) %}
+{% if flags %}{{ dbt_expectations._validate_flags(flags, 'ciep') }}{% endif %}
+regexp_instr({{ source_value }}, '{{ regexp }}', {{ position }}, {{ occurrence }}, 0, '{{ flags }}')
 {% endmacro %}
+
+{% macro _validate_flags(flags, alphabet) %}
+{% for flag in flags %}
+    {% if flag not in alphabet %}
+    {{ exceptions.raise_compiler_error(
+        "flag " ~ flag ~ " not in list of allowed flags for this adapter: " ~ alphabet | join(", ")
+    ) }}
+    {% endif %}
+{% endfor %}
+{% endmacro %}

macros/schema_tests/string_matching/expect_column_values_to_match_regex.sql

@@ -1,11 +1,12 @@
 {% test expect_column_values_to_match_regex(model, column_name,
                                                     regex,
                                                     row_condition=None,
-                                                    is_raw=False
+                                                    is_raw=False,
+                                                    flags=""
                                                     ) %}
 
 {% set expression %}
-{{ dbt_expectations.regexp_instr(column_name, regex, is_raw=is_raw) }} > 0
+{{ dbt_expectations.regexp_instr(column_name, regex, is_raw=is_raw, flags=flags) }} > 0
 {% endset %}
 
 {{ dbt_expectations.expression_is_true(model,

macros/schema_tests/string_matching/expect_column_values_to_match_regex_list.sql

@@ -2,12 +2,13 @@
                                                     regex_list,
                                                     match_on="any",
                                                     row_condition=None,
-                                                    is_raw=False
+                                                    is_raw=False,
+                                                    flags=""
                                                     ) %}
 
 {% set expression %}
     {% for regex in regex_list %}
-    {{ dbt_expectations.regexp_instr(column_name, regex, is_raw=is_raw) }} > 0
+    {{ dbt_expectations.regexp_instr(column_name, regex, is_raw=is_raw, flags=flags) }} > 0
     {%- if not loop.last %}
     {{ " and " if match_on == "all" else " or "}}
     {% endif -%}

macros/schema_tests/string_matching/expect_column_values_to_not_match_regex.sql

@@ -1,11 +1,12 @@
 {% test expect_column_values_to_not_match_regex(model, column_name,
                                                     regex,
                                                     row_condition=None,
-                                                    is_raw=False
+                                                    is_raw=False,
+                                                    flags=""
                                                     ) %}
 
 {% set expression %}
-{{ dbt_expectations.regexp_instr(column_name, regex, is_raw=is_raw) }} = 0
+{{ dbt_expectations.regexp_instr(column_name, regex, is_raw=is_raw, flags=flags) }} = 0
 {% endset %}
 
 {{ dbt_expectations.expression_is_true(model,

macros/schema_tests/string_matching/expect_column_values_to_not_match_regex_list.sql

@@ -2,12 +2,13 @@
                                                     regex_list,
                                                     match_on="any",
                                                     row_condition=None,
-                                                    is_raw=False
+                                                    is_raw=False,
+                                                    flags=""
                                                     ) %}
 
 {% set expression %}
 {% for regex in regex_list %}
-{{ dbt_expectations.regexp_instr(column_name, regex, is_raw=is_raw) }} = 0
+{{ dbt_expectations.regexp_instr(column_name, regex, is_raw=is_raw, flags=flags) }} = 0
 {%- if not loop.last %}
 {{ " and " if match_on == "all" else " or "}}
 {% endif -%}