Get rid of google-re2 as dependency (apache#47493)

Following discusison in https://lists.apache.org/thread/nnttnblj3tcv2wqj79gwovtfhgq8lvmm
and PR merged apache#47360 and
opened issue apache#47364 ideal that could
prevent new additions of regexp to UI. I think it's time to remove
google-re2.

Author: potiuk

.pre-commit-config.yaml

@@ -421,15 +421,6 @@ repos:
         exclude: ^providers/.*/src/airflow/providers/
         entry: ./scripts/ci/pre_commit/check_cncf_k8s_used_for_k8s_executor_only.py
         additional_dependencies: ['rich>=12.4.4']
-      - id: check-google-re2-as-dependency
-        name: Check google-re2 declared as dep
-        description: Check google-re2 is declared as dependency when needed
-        entry: ./scripts/ci/pre_commit/check_google_re2_imports.py
-        language: python
-        pass_filenames: true
-        require_serial: true
-        files: ^providers/.*/src/airflow/providers/.*\.py$
-        additional_dependencies: ['rich>=12.4.4']
       - id: update-local-yml-file
         name: Update mounts in the local yml file
         entry: ./scripts/ci/pre_commit/local_yml_mounts.py
@@ -518,7 +509,7 @@ repos:
           ^providers/fab/src/airflow/providers/fab/migrations/versions/.*$|^providers/fab/src/airflow/providers/fab/migrations/versions|
           ^airflow/utils/db.py$|
           ^providers/fab/src/airflow/providers/fab/auth_manager/models/db.py$
-        additional_dependencies: ['packaging','google-re2']
+        additional_dependencies: ['packaging']
       - id: update-version
         name: Update versions in docs
         entry: ./scripts/ci/pre_commit/update_versions.py
@@ -1156,35 +1147,6 @@ repos:
         language: python
         pass_filenames: true
         files: ^tests/.*\.py$
-      - id: check-usage-of-re2-over-re
-        language: pygrep
-        name: Use re2 module instead of re
-        description: Use re2 module instead of re
-        entry: "^\\s*from re\\s|^\\s*import re\\s"
-        pass_filenames: true
-        files: \.py$
-        exclude: |
-          (?x)
-          ^airflow/configuration.py$ |
-          ^airflow/metrics/validators.py$ |
-          ^airflow/models/dag.py$ |
-          ^airflow/serialization/serde.py$ |
-          ^airflow/utils/file.py$ |
-          ^airflow/utils/helpers.py$ |
-          ^providers/ |
-          ^tests/ |
-          ^providers/tests/ |
-          ^providers/.*/tests/ |
-          ^task-sdk/src/airflow/sdk/definitions/dag.py$ |
-          ^task-sdk/src/airflow/sdk/execution_time/secrets_masker.py$ |
-          ^task-sdk/src/airflow/sdk/definitions/_internal/node.py$ |
-          ^dev/.*\.py$ |
-          ^scripts/.*\.py$ |
-          ^docker_tests/.*$ |
-          ^helm_tests/.*$ |
-          ^devel-common/.*$ |
-          ^docs/.*\.py$ |
-          ^hatch_build.py$
       - id: check-provider-docs-valid
         name: Validate provider doc files
         entry: ./scripts/ci/pre_commit/check_provider_docs.py

airflow/api_fastapi/core_api/routes/public/providers.py

@@ -17,7 +17,7 @@
 
 from __future__ import annotations
 
-import re2
+import re
 
 from airflow.api_fastapi.common.parameters import QueryLimit, QueryOffset
 from airflow.api_fastapi.common.router import AirflowRouter
@@ -28,7 +28,7 @@
 
 
 def _remove_rst_syntax(value: str) -> str:
-    return re2.sub("[`_<>]", "", value.strip(" \n."))
+    return re.sub("[`_<>]", "", value.strip(" \n."))
 
 
 def _provider_mapper(provider: ProviderInfo) -> ProviderResponse:

airflow/cli/commands/remote_commands/dag_command.py

@@ -24,11 +24,11 @@
 import json
 import logging
 import operator
+import re
 import subprocess
 import sys
 from typing import TYPE_CHECKING
 
-import re2
 from sqlalchemy import func, select
 
 from airflow.api.client import get_current_api_client
@@ -529,7 +529,7 @@ def dag_test(args, dag: DAG | None = None, session: Session = NEW_SESSION) -> No
     use_executor = args.use_executor
 
     mark_success_pattern = (
-        re2.compile(args.mark_success_pattern) if args.mark_success_pattern is not None else None
+        re.compile(args.mark_success_pattern) if args.mark_success_pattern is not None else None
     )
 
     with _airflow_parsing_context_manager(dag_id=args.dag_id):

airflow/cli/commands/remote_commands/provider_command.py

@@ -18,10 +18,9 @@
 
 from __future__ import annotations
 
+import re
 import sys
 
-import re2
-
 from airflow.cli.simple_table import AirflowConsole
 from airflow.providers_manager import ProvidersManager
 from airflow.utils.cli import suppress_logs_and_warning
@@ -31,7 +30,7 @@
 
 
 def _remove_rst_syntax(value: str) -> str:
-    return re2.sub("[`_<>]", "", value.strip(" \n."))
+    return re.sub("[`_<>]", "", value.strip(" \n."))
 
 
 @suppress_logs_and_warning

airflow/configuration.py

@@ -25,6 +25,7 @@
 import multiprocessing
 import os
 import pathlib
+import re
 import shlex
 import stat
 import subprocess
@@ -41,7 +42,6 @@
 from typing import IO, TYPE_CHECKING, Any, Union
 from urllib.parse import urlsplit
 
-import re2
 from packaging.version import parse as parse_version
 from typing_extensions import overload
 
@@ -63,7 +63,7 @@
     warnings.filterwarnings(action="default", category=DeprecationWarning, module="airflow")
     warnings.filterwarnings(action="default", category=PendingDeprecationWarning, module="airflow")
 
-_SQLITE3_VERSION_PATTERN = re2.compile(r"(?P<version>^\d+(?:\.\d+)*)\D?.*$")
+_SQLITE3_VERSION_PATTERN = re.compile(r"(?P<version>^\d+(?:\.\d+)*)\D?.*$")
 
 ConfigType = Union[str, int, float, bool]
 ConfigOptionsDictType = dict[str, ConfigType]
@@ -354,37 +354,37 @@ def inversed_deprecated_sections(self):
     # about. Mapping of section -> setting -> { old, replace, by_version }
     deprecated_values: dict[str, dict[str, tuple[Pattern, str, str]]] = {
         "core": {
-            "hostname_callable": (re2.compile(r":"), r".", "2.1"),
+            "hostname_callable": (re.compile(r":"), r".", "2.1"),
         },
         "webserver": {
-            "navbar_color": (re2.compile(r"(?i)\A#007A87\z"), "#fff", "2.1"),
-            "dag_default_view": (re2.compile(r"^tree$"), "grid", "3.0"),
+            "navbar_color": (re.compile(r"(?i)^#007A87$"), "#fff", "2.1"),
+            "dag_default_view": (re.compile(r"^tree$"), "grid", "3.0"),
         },
         "email": {
             "email_backend": (
-                re2.compile(r"^airflow\.contrib\.utils\.sendgrid\.send_email$"),
+                re.compile(r"^airflow\.contrib\.utils\.sendgrid\.send_email$"),
                 r"airflow.providers.sendgrid.utils.emailer.send_email",
                 "2.1",
             ),
         },
         "logging": {
             "log_filename_template": (
-                re2.compile(re2.escape("{{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{{ try_number }}.log")),
+                re.compile(re.escape("{{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{{ try_number }}.log")),
                 # The actual replacement value will be updated after defaults are loaded from config.yml
                 "XX-set-after-default-config-loaded-XX",
                 "3.0",
             ),
         },
         "api": {
             "auth_backends": (
-                re2.compile(r"^airflow\.api\.auth\.backend\.deny_all$|^$"),
+                re.compile(r"^airflow\.api\.auth\.backend\.deny_all$|^$"),
                 "airflow.providers.fab.auth_manager.api.auth.backend.session",
                 "3.0",
             ),
         },
         "elasticsearch": {
             "log_id_template": (
-                re2.compile("^" + re2.escape("{dag_id}-{task_id}-{logical_date}-{try_number}") + "$"),
+                re.compile("^" + re.escape("{dag_id}-{task_id}-{logical_date}-{try_number}") + "$"),
                 "{dag_id}-{task_id}-{run_id}-{map_index}-{try_number}",
                 "3.0",
             )
@@ -725,7 +725,7 @@ def _upgrade_postgres_metastore_conn(self):
                 stacklevel=1,
             )
             self.upgraded_values[(section, key)] = old_value
-            new_value = re2.sub("^" + re2.escape(f"{parsed.scheme}://"), f"{good_scheme}://", old_value)
+            new_value = re.sub("^" + re.escape(f"{parsed.scheme}://"), f"{good_scheme}://", old_value)
             self._update_env_var(section=section, name=key, new_value=new_value)
 
             # if the old value is set via env var, we need to wipe it

airflow/decorators/base.py

@@ -18,14 +18,14 @@
 
 import inspect
 import itertools
+import re
 import textwrap
 import warnings
 from collections.abc import Collection, Iterator, Mapping, Sequence
 from functools import cached_property, update_wrapper
 from typing import TYPE_CHECKING, Any, Callable, ClassVar, Generic, Protocol, TypeVar, cast, overload
 
 import attr
-import re2
 import typing_extensions
 
 from airflow.models.baseoperator import (
@@ -138,14 +138,14 @@ def get_unique_task_id(
         return task_id
 
     def _find_id_suffixes(dag: DAG) -> Iterator[int]:
-        prefix = re2.split(r"__\d+$", tg_task_id)[0]
+        prefix = re.split(r"__\d+$", tg_task_id)[0]
         for task_id in dag.task_ids:
-            match = re2.match(rf"^{prefix}__(\d+)$", task_id)
+            match = re.match(rf"^{prefix}__(\d+)$", task_id)
             if match:
                 yield int(match.group(1))
         yield 0  # Default if there's no matching task ID.
 
-    core = re2.split(r"__\d+$", task_id)[0]
+    core = re.split(r"__\d+$", task_id)[0]
     return f"{core}__{max(_find_id_suffixes(dag)) + 1}"
 
 

airflow/metrics/validators.py

@@ -21,15 +21,14 @@
 
 import abc
 import logging
+import re
 import string
 import warnings
 from collections.abc import Iterable
 from functools import partial, wraps
 from re import Pattern
 from typing import Callable, cast
 
-import re2
-
 from airflow.configuration import conf
 from airflow.exceptions import InvalidStatsNameException
 
@@ -82,7 +81,7 @@ class MetricNameLengthExemptionWarning(Warning):
     r"^dagrun\.schedule_delay\.(?P<dag_id>.*)$",
     r"^dagrun\.(?P<dag_id>.*)\.first_task_scheduling_delay$",
 }
-BACK_COMPAT_METRIC_NAMES: set[Pattern[str]] = {re2.compile(name) for name in BACK_COMPAT_METRIC_NAME_PATTERNS}
+BACK_COMPAT_METRIC_NAMES: set[Pattern[str]] = {re.compile(name) for name in BACK_COMPAT_METRIC_NAME_PATTERNS}
 
 OTEL_NAME_MAX_LENGTH = 63
 DEFAULT_VALIDATOR_TYPE = "allow"
@@ -161,7 +160,7 @@ def stat_name_otel_handler(
         # If the name is in the exceptions list, do not fail it for being too long.
         # It may still be deemed invalid for other reasons below.
         for exemption in BACK_COMPAT_METRIC_NAMES:
-            if re2.match(exemption, stat_name):
+            if re.match(exemption, stat_name):
                 # There is a back-compat exception for this name; proceed
                 name_length_exemption = True
                 matched_exemption = exemption.pattern
@@ -249,7 +248,7 @@ def test(self, name: str) -> bool:
 
     def _has_pattern_match(self, name: str) -> bool:
         for entry in self.validate_list or ():
-            if re2.findall(entry, name.strip().lower()):
+            if re.findall(entry, name.strip().lower()):
                 return True
         return False
 

airflow/models/connection.py

@@ -19,12 +19,12 @@
 
 import json
 import logging
+import re
 from contextlib import suppress
 from json import JSONDecodeError
 from typing import Any
 from urllib.parse import parse_qsl, quote, unquote, urlencode, urlsplit
 
-import re2
 from sqlalchemy import Boolean, Column, Integer, String, Text
 from sqlalchemy.orm import declared_attr, reconstructor, synonym
 
@@ -43,7 +43,7 @@
 # the symbols #,!,-,_,.,:,\,/ and () requiring at least one match.
 #
 # You can try the regex here: https://regex101.com/r/69033B/1
-RE_SANITIZE_CONN_ID = re2.compile(r"^[\w\#\!\(\)\-\.\:\/\\]{1,}$")
+RE_SANITIZE_CONN_ID = re.compile(r"^[\w\#\!\(\)\-\.\:\/\\]{1,}$")
 # the conn ID max len should be 250
 CONN_ID_MAX_LEN: int = 250
 
@@ -66,7 +66,7 @@ def sanitize_conn_id(conn_id: str | None, max_length=CONN_ID_MAX_LEN) -> str | N
     """
     # check if `conn_id` or our match group is `None` and the `conn_id` is within the specified length.
     if (not isinstance(conn_id, str) or len(conn_id) > max_length) or (
-        res := re2.match(RE_SANITIZE_CONN_ID, conn_id)
+        res := re.match(RE_SANITIZE_CONN_ID, conn_id)
     ) is None:
         return None
 

airflow/models/dag.py

@@ -21,6 +21,7 @@
 import copy
 import functools
 import logging
+import re
 import sys
 import time
 from collections import defaultdict
@@ -42,7 +43,6 @@
 import attrs
 import methodtools
 import pendulum
-import re2
 import sqlalchemy_jsonfield
 from dateutil.relativedelta import relativedelta
 from packaging import version as packaging_version
@@ -1722,7 +1722,7 @@ def add_logger_if_needed(ti: TaskInstance):
                     ti.task = tasks[ti.task_id]
 
                     mark_success = (
-                        re2.compile(mark_success_pattern).fullmatch(ti.task_id) is not None
+                        re.compile(mark_success_pattern).fullmatch(ti.task_id) is not None
                         if mark_success_pattern is not None
                         else False
                     )
@@ -1804,9 +1804,9 @@ def create_dagrun(
 
         # This is also done on the DagRun model class, but SQLAlchemy column
         # validator does not work well for some reason.
-        if not re2.match(RUN_ID_REGEX, run_id):
+        if not re.match(RUN_ID_REGEX, run_id):
             regex = airflow_conf.get("scheduler", "allowed_run_id_pattern").strip()
-            if not regex or not re2.match(regex, run_id):
+            if not regex or not re.match(regex, run_id):
                 raise ValueError(
                     f"The run_id provided '{run_id}' does not match regex pattern "
                     f"'{regex}' or '{RUN_ID_REGEX}'"

airflow/models/dagrun.py

@@ -19,11 +19,11 @@
 
 import itertools
 import os
+import re
 from collections import defaultdict
 from collections.abc import Iterable, Iterator, Sequence
 from typing import TYPE_CHECKING, Any, Callable, NamedTuple, TypeVar, overload
 
-import re2
 from sqlalchemy import (
     JSON,
     Column,
@@ -298,10 +298,10 @@ def __repr__(self):
     def validate_run_id(self, key: str, run_id: str) -> str | None:
         if not run_id:
             return None
-        if re2.match(RUN_ID_REGEX, run_id):
+        if re.match(RUN_ID_REGEX, run_id):
             return run_id
         regex = airflow_conf.get("scheduler", "allowed_run_id_pattern").strip()
-        if regex and re2.match(regex, run_id):
+        if regex and re.match(regex, run_id):
             return run_id
         raise ValueError(
             f"The run_id provided '{run_id}' does not match regex pattern '{regex}' or '{RUN_ID_REGEX}'"