update

Author: lithomas1

pandas/core/arrays/_mixins.py

@@ -411,17 +411,10 @@ def _where(self: Self, mask: npt.NDArray[np.bool_], value) -> Self:
         """
         value = self._validate_setitem_value(value)
 
-        # Note: For backwards compatibility purposes
-        # StringArray returns an object array in __array__
-        # when it is backed by a numpy StringDType
-        # We need to work around that here.
-        if hasattr(value, "_ndarray") and value._ndarray.dtype.kind == "T":
-            value = value._ndarray
-
-        # np.where will not preserve the StringDType
-        # TODO: ask Nathan about this
-        # also TODO: this is a mess
         if self._ndarray.dtype.kind == "T":
+            # Handling non-string values and numpy StringDtype
+            # explicitly since we don't want to end up with object
+            # and lose the string dtype
             if value is np.nan:
                 value = libmissing.NA
                 res_values = self._ndarray.copy()

pandas/core/arrays/numpy_.py

@@ -166,7 +166,7 @@ def __array__(
         self, dtype: NpDtype | None = None, copy: bool | None = None
     ) -> np.ndarray:
         array = self._ndarray
-        # to_numpy on StringArray backed by StringDType should still return object dtype
+        # np.array on StringArray backed by StringDType should still return object dtype
         # for backwards compat
         if self._ndarray.dtype.kind == "T":
             array = array.astype(object)
@@ -516,8 +516,8 @@ def to_numpy(
         # to_numpy on StringArray backed by StringDType should still return object dtype
         # for backwards compat
         array = self._ndarray
-        if self._ndarray.dtype.kind == "T":
-            array = array.astype(object)
+        if dtype is None and self._ndarray.dtype.kind == "T":
+            dtype = object
         result = np.asarray(array, dtype=dtype)
         if na_value is not lib.no_default and mask.any():
             result = result.copy()

pandas/core/strings/object_array.py

@@ -623,7 +623,9 @@ def _str_isupper(self) -> BooleanArray:
     def _str_len(self) -> IntegerArray:
         if self._ndarray.dtype == object:
             return super()._str_len()
-        result = np.strings.str_len(self._ndarray)
+        na_mask = isna(self._ndarray)
+        result = np.empty_like(self._ndarray, dtype="int64")
+        result[~na_mask] = np.strings.str_len(self._ndarray[~na_mask])
         return IntegerArray(result, isna(self._ndarray))
 
     def _str_lstrip(self, to_strip=None):

pandas/tests/base/test_misc.py

@@ -103,19 +103,17 @@ def test_memory_usage(index_or_series_memory_obj, request):
     is_categorical = isinstance(obj.dtype, pd.CategoricalDtype) or (
         is_ser and isinstance(obj.index.dtype, pd.CategoricalDtype)
     )
-    is_object_string = is_dtype_equal(obj, "string[python]") or (
+    is_string_array = is_dtype_equal(obj, "string[python]") or (
         is_ser and is_dtype_equal(obj.index.dtype, "string[python]")
     )
-    if is_object_string and np_version_gt2:
-        mark = pytest.mark.xfail(
-            True, reason="NumPy does not expose an API to get StringDType memory usage"
-        )
+    if is_string_array and np_version_gt2:
+        mark = pytest.mark.xfail(reason="NumPy does not expose an API to get StringDType memory usage")
         request.applymarker(mark)
 
     if len(obj) == 0:
         expected = 0
         assert res_deep == res == expected
-    elif is_object or is_categorical:
+    elif is_object or is_categorical or is_string_array:
         # only deep will pick them up
         assert res_deep > res
     else:

pandas/tests/frame/test_constructors.py

@@ -3068,7 +3068,7 @@ def test_np_string_array(self, data):
 
         data["a"] = np.array(data["a"], dtype=StringDType())
         res = DataFrame(data)
-        assert res["a"].dtype == np.dtypes.StringDType()
+        assert res["a"].dtype == np.object_
         assert (res["a"] == data["a"]).all()