Merge branch '2.3.x' into backport-61909

Author: rhshadrach

doc/source/user_guide/migration-3-strings.rst

@@ -118,12 +118,17 @@ through the ``str`` accessor will work the same:
 Overview of behavior differences and how to address them
 ---------------------------------------------------------
 
-The dtype is no longer object dtype
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The dtype is no longer a numpy "object" dtype
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 When inferring or reading string data, the data type of the resulting DataFrame
 column or Series will silently start being the new ``"str"`` dtype instead of
-``"object"`` dtype, and this can have some impact on your code.
+the numpy ``"object"`` dtype, and this can have some impact on your code.
+
+The new string dtype is a pandas data type ("extension dtype"), and no longer a
+numpy ``np.dtype`` instance. Therefore, passing the dtype of a string column to
+numpy functions will no longer work (e.g. passing it to a ``dtype=`` argument
+of a numpy function, or using ``np.issubdtype`` to check the dtype).
 
 Checking the dtype
 ^^^^^^^^^^^^^^^^^^

doc/source/whatsnew/v2.3.2.rst

@@ -22,7 +22,11 @@ become the default string dtype in pandas 3.0. See
 
 Bug fixes
 ^^^^^^^^^
--
+- Fix :meth:`~DataFrame.to_json` with ``orient="table"`` to correctly use the
+  "string" type in the JSON Table Schema for :class:`StringDtype` columns
+  (:issue:`61889`)
+- Fixed ``~Series.str.match``, ``~Series.str.fullmatch`` and ``~Series.str.contains``
+  with compiled regex for the Arrow-backed string dtype (:issue:`61964`, :issue:`61942`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_232.contributors:

pandas/core/arrays/_arrow_string_mixins.py

@@ -301,23 +301,29 @@ def _str_contains(
 
     def _str_match(
         self,
-        pat: str,
+        pat: str | re.Pattern,
         case: bool = True,
         flags: int = 0,
         na: Scalar | lib.NoDefault = lib.no_default,
     ):
-        if not pat.startswith("^"):
+        if isinstance(pat, re.Pattern):
+            # GH#61952
+            pat = pat.pattern
+        if isinstance(pat, str) and not pat.startswith("^"):
             pat = f"^{pat}"
         return self._str_contains(pat, case, flags, na, regex=True)
 
     def _str_fullmatch(
         self,
-        pat,
+        pat: str | re.Pattern,
         case: bool = True,
         flags: int = 0,
         na: Scalar | lib.NoDefault = lib.no_default,
     ):
-        if not pat.endswith("$") or pat.endswith("\\$"):
+        if isinstance(pat, re.Pattern):
+            # GH#61952
+            pat = pat.pattern
+        if isinstance(pat, str) and (not pat.endswith("$") or pat.endswith("\\$")):
             pat = f"{pat}$"
         return self._str_match(pat, case, flags, na)
 

pandas/core/arrays/categorical.py

@@ -2215,8 +2215,16 @@ def _repr_categories(self) -> list[str]:
         )
         from pandas.io.formats import format as fmt
 
+        formatter = None
+        if self.categories.dtype == "str":
+            # the extension array formatter defaults to boxed=True in format_array
+            # override here to boxed=False to be consistent with QUOTE_NONNUMERIC
+            formatter = cast(ExtensionArray, self.categories._values)._formatter(
+                boxed=False
+            )
+
         format_array = partial(
-            fmt.format_array, formatter=None, quoting=QUOTE_NONNUMERIC
+            fmt.format_array, formatter=formatter, quoting=QUOTE_NONNUMERIC
         )
         if len(self.categories) > max_categories:
             num = max_categories // 2

pandas/core/arrays/string_arrow.py

@@ -355,6 +355,8 @@ def _str_contains(
     ):
         if flags:
             return super()._str_contains(pat, case, flags, na, regex)
+        if isinstance(pat, re.Pattern):
+            pat = pat.pattern
 
         return ArrowStringArrayMixin._str_contains(self, pat, case, flags, na, regex)
 

pandas/core/indexing.py

@@ -2575,6 +2575,22 @@ def __getitem__(self, key):
         return super().__getitem__(key)
 
     def __setitem__(self, key, value) -> None:
+        if not PYPY and using_copy_on_write():
+            if sys.getrefcount(self.obj) <= 2:
+                warnings.warn(
+                    _chained_assignment_msg, ChainedAssignmentError, stacklevel=2
+                )
+        elif not PYPY and not using_copy_on_write():
+            ctr = sys.getrefcount(self.obj)
+            ref_count = 2
+            if not warn_copy_on_write() and _check_cacher(self.obj):
+                # see https://github.com/pandas-dev/pandas/pull/56060#discussion_r1399245221
+                ref_count += 1
+            if ctr <= ref_count:
+                warnings.warn(
+                    _chained_assignment_warning_msg, FutureWarning, stacklevel=2
+                )
+
         if self.ndim == 2 and not self._axes_are_unique:
             # GH#33041 fall back to .loc
             if not isinstance(key, tuple) or not all(is_scalar(x) for x in key):
@@ -2599,6 +2615,25 @@ def _convert_key(self, key):
                 raise ValueError("iAt based indexing can only have integer indexers")
         return key
 
+    def __setitem__(self, key, value) -> None:
+        if not PYPY and using_copy_on_write():
+            if sys.getrefcount(self.obj) <= 2:
+                warnings.warn(
+                    _chained_assignment_msg, ChainedAssignmentError, stacklevel=2
+                )
+        elif not PYPY and not using_copy_on_write():
+            ctr = sys.getrefcount(self.obj)
+            ref_count = 2
+            if not warn_copy_on_write() and _check_cacher(self.obj):
+                # see https://github.com/pandas-dev/pandas/pull/56060#discussion_r1399245221
+                ref_count += 1
+            if ctr <= ref_count:
+                warnings.warn(
+                    _chained_assignment_warning_msg, FutureWarning, stacklevel=2
+                )
+
+        return super().__setitem__(key, value)
+
 
 def _tuplify(ndim: int, loc: Hashable) -> tuple[Hashable | slice, ...]:
     """

pandas/core/strings/accessor.py

@@ -1353,8 +1353,8 @@ def match(self, pat: str, case: bool = True, flags: int = 0, na=lib.no_default):
 
         Parameters
         ----------
-        pat : str
-            Character sequence.
+        pat : str or compiled regex
+            Character sequence or regular expression.
         case : bool, default True
             If True, case sensitive.
         flags : int, default 0 (no flags)

pandas/core/strings/object_array.py

@@ -245,14 +245,15 @@ def rep(x, r):
 
     def _str_match(
         self,
-        pat: str,
+        pat: str | re.Pattern,
         case: bool = True,
         flags: int = 0,
         na: Scalar | lib.NoDefault = lib.no_default,
     ):
         if not case:
             flags |= re.IGNORECASE
-
+        if isinstance(pat, re.Pattern):
+            pat = pat.pattern
         regex = re.compile(pat, flags=flags)
 
         f = lambda x: regex.match(x) is not None
@@ -267,7 +268,8 @@ def _str_fullmatch(
     ):
         if not case:
             flags |= re.IGNORECASE
-
+        if isinstance(pat, re.Pattern):
+            pat = pat.pattern
         regex = re.compile(pat, flags=flags)
 
         f = lambda x: regex.fullmatch(x) is not None

pandas/io/json/_table_schema.py

@@ -90,8 +90,6 @@ def as_json_table_type(x: DtypeObj) -> str:
         return "datetime"
     elif lib.is_np_dtype(x, "m"):
         return "duration"
-    elif isinstance(x, ExtensionDtype):
-        return "any"
     elif is_string_dtype(x):
         return "string"
     else:
@@ -197,7 +195,7 @@ def convert_json_field_to_pandas_type(field) -> str | CategoricalDtype:
     """
     typ = field["type"]
     if typ == "string":
-        return "object"
+        return field.get("extDtype", None)
     elif typ == "integer":
         return field.get("extDtype", "int64")
     elif typ == "number":

pandas/tests/arrays/categorical/test_repr.py

@@ -19,16 +19,11 @@
 class TestCategoricalReprWithFactor:
     def test_print(self, using_infer_string):
         factor = Categorical(["a", "b", "b", "a", "a", "c", "c", "c"], ordered=True)
-        if using_infer_string:
-            expected = [
-                "['a', 'b', 'b', 'a', 'a', 'c', 'c', 'c']",
-                "Categories (3, str): [a < b < c]",
-            ]
-        else:
-            expected = [
-                "['a', 'b', 'b', 'a', 'a', 'c', 'c', 'c']",
-                "Categories (3, object): ['a' < 'b' < 'c']",
-            ]
+        dtype = "str" if using_infer_string else "object"
+        expected = [
+            "['a', 'b', 'b', 'a', 'a', 'c', 'c', 'c']",
+            f"Categories (3, {dtype}): ['a' < 'b' < 'c']",
+        ]
         expected = "\n".join(expected)
         actual = repr(factor)
         assert actual == expected