Code refactor, bug fixes and test fixes

Author: maushumee

.github/workflows/unit-tests.yml

@@ -59,7 +59,6 @@ jobs:
             extra_loc: "zh_CN"
           - name: "Future infer strings"
             env_file: actions-311.yaml
-            pattern: "not slow and not network and not single_cpu"
             pandas_future_infer_string: "1"
           - name: "Pypy"
             env_file: actions-pypy-39.yaml

ci/code_checks.sh

@@ -70,7 +70,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         --format=actions \
         -i ES01 `# For now it is ok if docstrings are missing the extended summary` \
         -i "pandas.Series.dt PR01" `# Accessors are implemented as classes, but we do not document the Parameters section` \
-        -i "pandas.MultiIndex.append PR07,SA01" \
         -i "pandas.MultiIndex.copy PR07,RT03,SA01" \
         -i "pandas.MultiIndex.get_level_values SA01" \
         -i "pandas.MultiIndex.get_loc PR07" \
@@ -223,7 +222,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.Timestamp.fromordinal SA01" \
         -i "pandas.Timestamp.fromtimestamp PR01,SA01" \
         -i "pandas.Timestamp.hour GL08" \
-        -i "pandas.Timestamp.isoweekday SA01" \
         -i "pandas.Timestamp.max PR02" \
         -i "pandas.Timestamp.microsecond GL08" \
         -i "pandas.Timestamp.min PR02" \
@@ -328,7 +326,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.core.groupby.DataFrameGroupBy.hist RT03" \
         -i "pandas.core.groupby.DataFrameGroupBy.indices SA01" \
         -i "pandas.core.groupby.DataFrameGroupBy.max SA01" \
-        -i "pandas.core.groupby.DataFrameGroupBy.median SA01" \
         -i "pandas.core.groupby.DataFrameGroupBy.min SA01" \
         -i "pandas.core.groupby.DataFrameGroupBy.nth PR02" \
         -i "pandas.core.groupby.DataFrameGroupBy.nunique SA01" \
@@ -347,7 +344,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.core.groupby.SeriesGroupBy.is_monotonic_decreasing SA01" \
         -i "pandas.core.groupby.SeriesGroupBy.is_monotonic_increasing SA01" \
         -i "pandas.core.groupby.SeriesGroupBy.max SA01" \
-        -i "pandas.core.groupby.SeriesGroupBy.median SA01" \
         -i "pandas.core.groupby.SeriesGroupBy.min SA01" \
         -i "pandas.core.groupby.SeriesGroupBy.nth PR02" \
         -i "pandas.core.groupby.SeriesGroupBy.ohlc SA01" \
@@ -362,7 +358,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.core.resample.Resampler.indices SA01" \
         -i "pandas.core.resample.Resampler.max PR01,RT03,SA01" \
         -i "pandas.core.resample.Resampler.mean SA01" \
-        -i "pandas.core.resample.Resampler.median SA01" \
         -i "pandas.core.resample.Resampler.min PR01,RT03,SA01" \
         -i "pandas.core.resample.Resampler.ohlc SA01" \
         -i "pandas.core.resample.Resampler.prod SA01" \

ci/run_tests.sh

@@ -16,11 +16,5 @@ if [[ "$PATTERN" ]]; then
   PYTEST_CMD="$PYTEST_CMD -m \"$PATTERN\""
 fi
 
-# temporarily let pytest always succeed (many tests are not yet passing in the
-# build enabling the future string dtype)
-if [[ "$PANDAS_FUTURE_INFER_STRING" == "1" ]]; then
-  PYTEST_CMD="$PYTEST_CMD || true"
-fi
-
 echo $PYTEST_CMD
 sh -c "$PYTEST_CMD"

pandas/_libs/lib.pyx

@@ -2702,7 +2702,7 @@ def maybe_convert_objects(ndarray[object] objects,
         if using_string_dtype() and is_string_array(objects, skipna=True):
             from pandas.core.arrays.string_ import StringDtype
 
-            dtype = StringDtype(storage="pyarrow_numpy")
+            dtype = StringDtype(storage="pyarrow", na_value=np.nan)
             return dtype.construct_array_type()._from_sequence(objects, dtype=dtype)
 
         elif convert_to_nullable_dtype and is_string_array(objects, skipna=True):

pandas/_libs/tslibs/nattype.pyx

@@ -441,6 +441,13 @@ class NaTType(_NaT):
 
         Monday == 1 ... Sunday == 7.
 
+        See Also
+        --------
+        Timestamp.weekday : Return the day of the week with Monday=0, Sunday=6.
+        Timestamp.isocalendar : Return a tuple containing ISO year, week number
+            and weekday.
+        datetime.date.isoweekday : Equivalent method in datetime module.
+
         Examples
         --------
         >>> ts = pd.Timestamp('2023-01-01 10:00:00')

pandas/_libs/tslibs/timestamps.pyx

@@ -2775,6 +2775,13 @@ default 'raise'
 
         Monday == 1 ... Sunday == 7.
 
+        See Also
+        --------
+        Timestamp.weekday : Return the day of the week with Monday=0, Sunday=6.
+        Timestamp.isocalendar : Return a tuple containing ISO year, week number
+            and weekday.
+        datetime.date.isoweekday : Equivalent method in datetime module.
+
         Examples
         --------
         >>> ts = pd.Timestamp('2023-01-01 10:00:00')

pandas/_testing/__init__.py

@@ -509,14 +509,14 @@ def shares_memory(left, right) -> bool:
     if (
         isinstance(left, ExtensionArray)
         and is_string_dtype(left.dtype)
-        and left.dtype.storage in ("pyarrow", "pyarrow_numpy")  # type: ignore[attr-defined]
+        and left.dtype.storage == "pyarrow"  # type: ignore[attr-defined]
     ):
         # https://github.com/pandas-dev/pandas/pull/43930#discussion_r736862669
         left = cast("ArrowExtensionArray", left)
         if (
             isinstance(right, ExtensionArray)
             and is_string_dtype(right.dtype)
-            and right.dtype.storage in ("pyarrow", "pyarrow_numpy")  # type: ignore[attr-defined]
+            and right.dtype.storage == "pyarrow"  # type: ignore[attr-defined]
         ):
             right = cast("ArrowExtensionArray", right)
             left_pa_data = left._pa_array

pandas/_testing/asserters.py

@@ -578,13 +578,19 @@ def raise_assert_detail(
 
     if isinstance(left, np.ndarray):
         left = pprint_thing(left)
-    elif isinstance(left, (CategoricalDtype, NumpyEADtype, StringDtype)):
+    elif isinstance(left, (CategoricalDtype, NumpyEADtype)):
         left = repr(left)
+    elif isinstance(left, StringDtype):
+        # TODO(infer_string) this special case could be avoided if we have
+        # a more informative repr https://github.com/pandas-dev/pandas/issues/59342
+        left = f"StringDtype(storage={left.storage}, na_value={left.na_value})"
 
     if isinstance(right, np.ndarray):
         right = pprint_thing(right)
-    elif isinstance(right, (CategoricalDtype, NumpyEADtype, StringDtype)):
+    elif isinstance(right, (CategoricalDtype, NumpyEADtype)):
         right = repr(right)
+    elif isinstance(right, StringDtype):
+        right = f"StringDtype(storage={right.storage}, na_value={right.na_value})"
 
     msg += f"""
 [left]:  {left}

pandas/core/arrays/arrow/array.py

@@ -575,10 +575,8 @@ def __getitem__(self, item: PositionalIndexer):
         if isinstance(item, np.ndarray):
             if not len(item):
                 # Removable once we migrate StringDtype[pyarrow] to ArrowDtype[string]
-                if self._dtype.name == "string" and self._dtype.storage in (
-                    "pyarrow",
-                    "pyarrow_numpy",
-                ):
+                if self._dtype.name == "string" and self._dtype.storage == "pyarrow":
+                    # TODO(infer_string) should this be large_string?
                     pa_dtype = pa.string()
                 else:
                     pa_dtype = self._dtype.pyarrow_dtype

pandas/core/arrays/numpy_.py

@@ -17,6 +17,7 @@
 from pandas.core.dtypes.dtypes import NumpyEADtype
 from pandas.core.dtypes.missing import isna
 
+import pandas as pd
 from pandas.core import (
     arraylike,
     missing,
@@ -248,74 +249,41 @@ def _validate_setitem_value(self, value):
         TypeError
         """
 
-        kind = self.dtype.kind
-
-        if kind == "b":
-            if lib.is_bool(value) or np.can_cast(type(value), self.dtype.type):
-                return value
-            if isinstance(value, NumpyExtensionArray) and (
-                lib.is_bool_array(value.to_numpy())
-            ):
-                return value
-
-        elif kind == "i":
-            if lib.is_integer(value) or np.can_cast(type(value), self.dtype.type):
-                return value
-            if isinstance(value, NumpyExtensionArray) and lib.is_integer_array(
-                value.to_numpy()
-            ):
-                return value
-
-        elif kind == "u":
-            if (lib.is_integer(value) and value > -1) or np.can_cast(
-                type(value), self.dtype.type
-            ):
-                return value
-
-        elif kind == "c":
-            if lib.is_complex(value) or np.can_cast(type(value), self.dtype.type):
-                return value
-
-        elif kind == "S":
-            if isinstance(value, str) or np.can_cast(type(value), self.dtype.type):
-                return value
-            if isinstance(value, NumpyExtensionArray) and lib.is_string_array(
-                value.to_numpy()
-            ):
-                return value
+        if type(value) == self.dtype.type:
+            return value
 
-        elif kind == "M":
-            if isinstance(value, np.datetime64):
-                return value
-            if isinstance(value, NumpyExtensionArray) and (
-                lib.is_date_array(value.to_numpy())
-                or lib.is_datetime_array(value.to_numpy())
-                or lib.is_datetime64_array(value.to_numpy())
-                or lib.is_datetime_with_singletz_array(value.to_numpy())
-            ):
-                return value
+        if isinstance(value, NumpyExtensionArray) and value.dtype == self.dtype:
+            return value
 
-        elif kind == "m":
-            if isinstance(value, np.timedelta64):
-                return value
-            if isinstance(value, NumpyExtensionArray) and (
-                lib.is_timedelta_or_timedelta64_array(value.to_numpy())
-                or lib.is_time_array(value.to_numpy())
-            ):
+        if (
+            isinstance(value, list)
+            or isinstance(value, NumpyExtensionArray)
+            or isinstance(value, np.ndarray)
+            or isinstance(value, pd.Series)
+        ):
+            try:
+                _ = pd.array(value, dtype=self.dtype)
                 return value
+            except ValueError:
+                print("Caught the error")
 
-        elif kind == "f":
-            if lib.is_float(value) or np.can_cast(type(value), self.dtype.type):
-                return value
-            if isinstance(value, NumpyExtensionArray) and lib.is_float_array(
-                value.to_numpy()
-            ):
+        if (
+            (
+                (lib.is_integer(value) or lib.is_float(value))
+                and self.dtype.kind in "iuf"
+            )
+            or (isinstance(value, str) and self.dtype.kind in "US")
+            or (self.dtype.kind == "O")
+        ) and not isna(value):
+            if self.dtype.type(value) == value:  # -> Problem
                 return value
 
-        elif np.can_cast(type(value), self.dtype.type):
+        if isna(value):
             return value
 
-        raise TypeError(f"Invalid value '{value!s}' for dtype {self.dtype}")
+        raise TypeError(
+            f"Invalid value '{value!s}' with type {type(value)} for dtype {self.dtype}"
+        )
 
     # Base EA class (and all other EA classes) don't have limit_area keyword
     # This can be removed here as well when the interpolate ffill/bfill method