Merge branch 'main' into api-nan-vs-na

Author: jbrockmendel

.pre-commit-config.yaml

@@ -121,7 +121,7 @@ repos:
         types: [python]
         stages: [manual]
         additional_dependencies: &pyright_dependencies
-        - [email protected].383
+        - [email protected].404
     -   id: pyright
         # note: assumes python env is setup and activated
         name: pyright reportGeneralTypeIssues

asv_bench/benchmarks/strings.py

@@ -8,6 +8,7 @@
     DataFrame,
     Index,
     Series,
+    StringDtype,
 )
 from pandas.arrays import StringArray
 
@@ -290,10 +291,10 @@ def setup(self):
         self.series_arr_nan = np.concatenate([self.series_arr, np.array([NA] * 1000)])
 
     def time_string_array_construction(self):
-        StringArray(self.series_arr)
+        StringArray(self.series_arr, dtype=StringDtype())
 
     def time_string_array_with_nan_construction(self):
-        StringArray(self.series_arr_nan)
+        StringArray(self.series_arr_nan, dtype=StringDtype())
 
     def peakmem_stringarray_construction(self):
-        StringArray(self.series_arr)
+        StringArray(self.series_arr, dtype=StringDtype())

doc/source/user_guide/migration-3-strings.rst

@@ -315,6 +315,37 @@ the :meth:`~pandas.Series.str.decode` method now has a ``dtype`` parameter to be
 able to specify object dtype instead of the default of string dtype for this use
 case.
 
+:meth:`Series.values` now returns an :class:`~pandas.api.extensions.ExtensionArray`
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+With object dtype, using ``.values`` on a Series will return the underlying NumPy array.
+
+.. code-block:: python
+
+   >>> ser = pd.Series(["a", "b", np.nan], dtype="object")
+   >>> type(ser.values)
+   <class 'numpy.ndarray'>
+
+However with the new string dtype, the underlying ExtensionArray is returned instead.
+
+.. code-block:: python
+
+   >>> ser = pd.Series(["a", "b", pd.NA], dtype="str")
+   >>> ser.values
+   <ArrowStringArray>
+   ['a', 'b', nan]
+   Length: 3, dtype: str
+
+If your code requires a NumPy array, you should use :meth:`Series.to_numpy`.
+
+.. code-block:: python
+
+   >>> ser = pd.Series(["a", "b", pd.NA], dtype="str")
+   >>> ser.to_numpy()
+   ['a' 'b' nan]
+
+In general, you should always prefer :meth:`Series.to_numpy` to get a NumPy array or :meth:`Series.array` to get an ExtensionArray over using :meth:`Series.values`.
+
 Notable bug fixes
 ~~~~~~~~~~~~~~~~~
 

doc/source/whatsnew/v3.0.0.rst

@@ -994,6 +994,7 @@ Indexing
 - Bug in :meth:`Series.__setitem__` when assigning boolean series with boolean indexer will raise ``LossySetitemError`` (:issue:`57338`)
 - Bug in printing :attr:`Index.names` and :attr:`MultiIndex.levels` would not escape single quotes (:issue:`60190`)
 - Bug in reindexing of :class:`DataFrame` with :class:`PeriodDtype` columns in case of consolidated block (:issue:`60980`, :issue:`60273`)
+- Bug in :meth:`Index.__getitem__` incorrectly raising with a 0-dim ``np.ndarray`` key (:issue:`55601`)
 
 Missing
 ^^^^^^^

environment.yml

@@ -77,7 +77,7 @@ dependencies:
 
   # code checks
   - flake8=7.1.0  # run in subprocess over docstring examples
-  - mypy=1.13.0  # pre-commit uses locally installed mypy
+  - mypy=1.17.1  # pre-commit uses locally installed mypy
   - tokenize-rt  # scripts/check_for_inconsistent_pandas_namespace.py
   - pre-commit>=4.2.0
 

pandas/_libs/src/datetime/pd_datetime.c

@@ -55,9 +55,23 @@ static int convert_pydatetime_to_datetimestruct(PyObject *dtobj,
   out->month = 1;
   out->day = 1;
 
-  out->year = PyLong_AsLong(PyObject_GetAttrString(obj, "year"));
-  out->month = PyLong_AsLong(PyObject_GetAttrString(obj, "month"));
-  out->day = PyLong_AsLong(PyObject_GetAttrString(obj, "day"));
+  tmp = PyObject_GetAttrString(obj, "year");
+  if (tmp == NULL)
+    return -1;
+  out->year = PyLong_AsLong(tmp);
+  Py_DECREF(tmp);
+
+  tmp = PyObject_GetAttrString(obj, "month");
+  if (tmp == NULL)
+    return -1;
+  out->month = PyLong_AsLong(tmp);
+  Py_DECREF(tmp);
+
+  tmp = PyObject_GetAttrString(obj, "day");
+  if (tmp == NULL)
+    return -1;
+  out->day = PyLong_AsLong(tmp);
+  Py_DECREF(tmp);
 
   // TODO(anyone): If we can get PyDateTime_IMPORT to work, we could use
   // PyDateTime_Check here, and less verbose attribute lookups.
@@ -70,10 +84,29 @@ static int convert_pydatetime_to_datetimestruct(PyObject *dtobj,
     return 0;
   }
 
-  out->hour = PyLong_AsLong(PyObject_GetAttrString(obj, "hour"));
-  out->min = PyLong_AsLong(PyObject_GetAttrString(obj, "minute"));
-  out->sec = PyLong_AsLong(PyObject_GetAttrString(obj, "second"));
-  out->us = PyLong_AsLong(PyObject_GetAttrString(obj, "microsecond"));
+  tmp = PyObject_GetAttrString(obj, "hour");
+  if (tmp == NULL)
+    return -1;
+  out->hour = PyLong_AsLong(tmp);
+  Py_DECREF(tmp);
+
+  tmp = PyObject_GetAttrString(obj, "minute");
+  if (tmp == NULL)
+    return -1;
+  out->min = PyLong_AsLong(tmp);
+  Py_DECREF(tmp);
+
+  tmp = PyObject_GetAttrString(obj, "second");
+  if (tmp == NULL)
+    return -1;
+  out->sec = PyLong_AsLong(tmp);
+  Py_DECREF(tmp);
+
+  tmp = PyObject_GetAttrString(obj, "microsecond");
+  if (tmp == NULL)
+    return -1;
+  out->us = PyLong_AsLong(tmp);
+  Py_DECREF(tmp);
 
   if (PyObject_HasAttrString(obj, "tzinfo")) {
     PyObject *offset = extract_utc_offset(obj);

pandas/_libs/tslibs/dtypes.pyi

@@ -28,33 +28,35 @@ class PeriodDtypeBase:
     def _td64_unit(self) -> str: ...
 
 class FreqGroup(Enum):
-    FR_ANN: int
-    FR_QTR: int
-    FR_MTH: int
-    FR_WK: int
-    FR_BUS: int
-    FR_DAY: int
-    FR_HR: int
-    FR_MIN: int
-    FR_SEC: int
-    FR_MS: int
-    FR_US: int
-    FR_NS: int
-    FR_UND: int
+    _value_: int
+    FR_ANN = ...
+    FR_QTR = ...
+    FR_MTH = ...
+    FR_WK = ...
+    FR_BUS = ...
+    FR_DAY = ...
+    FR_HR = ...
+    FR_MIN = ...
+    FR_SEC = ...
+    FR_MS = ...
+    FR_US = ...
+    FR_NS = ...
+    FR_UND = ...
     @staticmethod
     def from_period_dtype_code(code: int) -> FreqGroup: ...
 
 class Resolution(Enum):
-    RESO_NS: int
-    RESO_US: int
-    RESO_MS: int
-    RESO_SEC: int
-    RESO_MIN: int
-    RESO_HR: int
-    RESO_DAY: int
-    RESO_MTH: int
-    RESO_QTR: int
-    RESO_YR: int
+    _value_: int
+    RESO_NS = ...
+    RESO_US = ...
+    RESO_MS = ...
+    RESO_SEC = ...
+    RESO_MIN = ...
+    RESO_HR = ...
+    RESO_DAY = ...
+    RESO_MTH = ...
+    RESO_QTR = ...
+    RESO_YR = ...
     def __lt__(self, other: Resolution) -> bool: ...
     def __ge__(self, other: Resolution) -> bool: ...
     @property
@@ -67,17 +69,18 @@ class Resolution(Enum):
     def attr_abbrev(self) -> str: ...
 
 class NpyDatetimeUnit(Enum):
-    NPY_FR_Y: int
-    NPY_FR_M: int
-    NPY_FR_W: int
-    NPY_FR_D: int
-    NPY_FR_h: int
-    NPY_FR_m: int
-    NPY_FR_s: int
-    NPY_FR_ms: int
-    NPY_FR_us: int
-    NPY_FR_ns: int
-    NPY_FR_ps: int
-    NPY_FR_fs: int
-    NPY_FR_as: int
-    NPY_FR_GENERIC: int
+    _value_: int
+    NPY_FR_Y = ...
+    NPY_FR_M = ...
+    NPY_FR_W = ...
+    NPY_FR_D = ...
+    NPY_FR_h = ...
+    NPY_FR_m = ...
+    NPY_FR_s = ...
+    NPY_FR_ms = ...
+    NPY_FR_us = ...
+    NPY_FR_ns = ...
+    NPY_FR_ps = ...
+    NPY_FR_fs = ...
+    NPY_FR_as = ...
+    NPY_FR_GENERIC = ...

pandas/_testing/__init__.py

@@ -348,8 +348,9 @@ class SubclassedDataFrame(DataFrame):
     def _constructor(self):
         return lambda *args, **kwargs: SubclassedDataFrame(*args, **kwargs)
 
+    # error: Cannot override writeable attribute with read-only property
     @property
-    def _constructor_sliced(self):
+    def _constructor_sliced(self):  # type: ignore[override]
         return lambda *args, **kwargs: SubclassedSeries(*args, **kwargs)
 
 

pandas/_testing/_io.py

@@ -81,12 +81,12 @@ def round_trip_pathlib(writer, reader, path: str | None = None):
     if path is None:
         path = "___pathlib___"
     with ensure_clean(path) as path:
-        writer(Path(path))  # type: ignore[arg-type]
-        obj = reader(Path(path))  # type: ignore[arg-type]
+        writer(Path(path))
+        obj = reader(Path(path))
     return obj
 
 
-def write_to_compressed(compression, path, data, dest: str = "test") -> None:
+def write_to_compressed(compression, path: str, data, dest: str = "test") -> None:
     """
     Write data to a compressed file.
 
@@ -138,5 +138,9 @@ def write_to_compressed(compression, path, data, dest: str = "test") -> None:
     else:
         raise ValueError(f"Unrecognized compression type: {compression}")
 
-    with compress_method(path, mode=mode) as f:
+    # error: No overload variant of "ZipFile" matches argument types "str", "str"
+    # error: No overload variant of "BZ2File" matches argument types "str", "str"
+    # error: Argument "mode" to "TarFile" has incompatible type "str";
+    #  expected "Literal['r', 'a', 'w', 'x']
+    with compress_method(path, mode=mode) as f:  # type: ignore[call-overload, arg-type]
         getattr(f, method)(*args)

pandas/core/_numba/executor.py

@@ -87,7 +87,11 @@ def column_looper(
     else:
 
         @numba.jit(nopython=nopython, nogil=nogil, parallel=parallel)
-        def column_looper(
+        # error: Incompatible redefinition (redefinition with type
+        # "Callable[[ndarray[Any, Any], ndarray[Any, Any], ndarray[Any, Any],
+        # int, VarArg(Any)], Any]", original type "Callable[[ndarray[Any, Any],
+        # ndarray[Any, Any], int, int, VarArg(Any)], Any]")
+        def column_looper(  # type: ignore[misc]
             values: np.ndarray,
             start: np.ndarray,
             end: np.ndarray,