Merge branch 'main' into tempfile

Author: jsngn

asv_bench/benchmarks/algorithms.py

@@ -199,8 +199,8 @@ class SortIntegerArray:
     params = [10**3, 10**5]
 
     def setup(self, N):
-        data = np.arange(N, dtype=float)
-        data[40] = np.nan
+        data = np.arange(N, dtype=float).astype(object)
+        data[40] = pd.NA
         self.array = pd.array(data, dtype="Int64")
 
     def time_argsort(self, N):

asv_bench/benchmarks/frame_methods.py

@@ -4,6 +4,7 @@
 import numpy as np
 
 from pandas import (
+    NA,
     DataFrame,
     Index,
     MultiIndex,
@@ -445,6 +446,8 @@ def setup(self, inplace, dtype):
             values[::2] = np.nan
             if dtype == "Int64":
                 values = values.round()
+                values = values.astype(object)
+                values[::2] = NA
             self.df = DataFrame(values, dtype=dtype)
         self.fill_values = self.df.iloc[self.df.first_valid_index()].to_dict()
 

asv_bench/benchmarks/groupby.py

@@ -689,6 +689,10 @@ def setup(self, dtype, method, with_nans):
             null_vals = vals.astype(float, copy=True)
             null_vals[::2, :] = np.nan
             null_vals[::3, :] = np.nan
+            if dtype in ["Int64", "Float64"]:
+                null_vals = null_vals.astype(object)
+                null_vals[::2, :] = NA
+                null_vals[::3, :] = NA
             df = DataFrame(null_vals, columns=list("abcde"), dtype=dtype)
             df["key"] = keys
             self.df = df

doc/source/user_guide/text.rst

@@ -75,7 +75,7 @@ or convert from existing pandas data:
 
 .. ipython:: python
 
-   s1 = pd.Series([1, 2, np.nan], dtype="Int64")
+   s1 = pd.Series([1, 2, pd.NA], dtype="Int64")
    s1
    s2 = s1.astype("string")
    s2

doc/source/whatsnew/v0.21.0.rst

@@ -635,22 +635,17 @@ Previous behavior:
 
 New behavior:
 
-.. code-block:: ipython
+.. ipython:: python
 
-   In [1]: pi = pd.period_range('2017-01', periods=12, freq='M')
+   pi = pd.period_range('2017-01', periods=12, freq='M')
 
-   In [2]: s = pd.Series(np.arange(12), index=pi)
+   s = pd.Series(np.arange(12), index=pi)
 
-   In [3]: resampled = s.resample('2Q').mean()
+   resampled = s.resample('2Q').mean()
 
-   In [4]: resampled
-   Out[4]:
-   2017Q1    2.5
-   2017Q3    8.5
-   Freq: 2Q-DEC, dtype: float64
+   resampled
 
-   In [5]: resampled.index
-   Out[5]: PeriodIndex(['2017Q1', '2017Q3'], dtype='period[2Q-DEC]')
+   resampled.index
 
 Upsampling and calling ``.ohlc()`` previously returned a ``Series``, basically identical to calling ``.asfreq()``. OHLC upsampling now returns a DataFrame with columns ``open``, ``high``, ``low`` and ``close`` (:issue:`13083`). This is consistent with downsampling and ``DatetimeIndex`` behavior.
 

doc/source/whatsnew/v0.24.0.rst

@@ -50,7 +50,7 @@ marker of ``np.nan`` will infer to integer dtype. The display of the ``Series``
 
 .. ipython:: python
 
-   s = pd.Series([1, 2, np.nan], dtype='Int64')
+   s = pd.Series([1, 2, pd.NA], dtype='Int64')
    s
 
 
@@ -166,7 +166,7 @@ See the :ref:`dtypes docs <basics.dtypes>` for more on extension arrays.
 
 .. ipython:: python
 
-   pd.array([1, 2, np.nan], dtype='Int64')
+   pd.array([1, 2, pd.NA], dtype='Int64')
    pd.array(['a', 'b', 'c'], dtype='category')
 
 Passing data for which there isn't dedicated extension type (e.g. float, integer, etc.)

doc/source/whatsnew/v2.2.0.rst

@@ -664,7 +664,7 @@ Other Deprecations
 - Deprecated :meth:`DatetimeArray.__init__` and :meth:`TimedeltaArray.__init__`, use :func:`array` instead (:issue:`55623`)
 - Deprecated :meth:`Index.format`, use ``index.astype(str)`` or ``index.map(formatter)`` instead (:issue:`55413`)
 - Deprecated :meth:`Series.ravel`, the underlying array is already 1D, so ravel is not necessary (:issue:`52511`)
-- Deprecated :meth:`Series.resample` and :meth:`DataFrame.resample` with a :class:`PeriodIndex` (and the 'convention' keyword), convert to :class:`DatetimeIndex` (with ``.to_timestamp()``) before resampling instead (:issue:`53481`)
+- Deprecated :meth:`Series.resample` and :meth:`DataFrame.resample` with a :class:`PeriodIndex` (and the 'convention' keyword), convert to :class:`DatetimeIndex` (with ``.to_timestamp()``) before resampling instead (:issue:`53481`). Note: this deprecation was later undone in pandas 2.3.3 (:issue:`57033`)
 - Deprecated :meth:`Series.view`, use :meth:`Series.astype` instead to change the dtype (:issue:`20251`)
 - Deprecated :meth:`offsets.Tick.is_anchored`, use ``False`` instead (:issue:`55388`)
 - Deprecated ``core.internals`` members ``Block``, ``ExtensionBlock``, and ``DatetimeTZBlock``, use public APIs instead (:issue:`55139`)

doc/source/whatsnew/v2.3.3.rst

@@ -57,6 +57,14 @@ Bug fixes
 
 - The :meth:`DataFrame.iloc` now works correctly with ``copy_on_write`` option when assigning values after subsetting the columns of a homogeneous DataFrame (:issue:`60309`)
 
+Other changes
+~~~~~~~~~~~~~
+
+- The deprecation of using :meth:`Series.resample` and :meth:`DataFrame.resample`
+  with a :class:`PeriodIndex` (and the 'convention' keyword) has been undone.
+  Resampling with a :class:`PeriodIndex` is supported again, but a subset of
+  methods that return incorrect results will raise an error in pandas 3.0 (:issue:`57033`)
+
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_233.contributors:

doc/source/whatsnew/v3.0.0.rst

@@ -465,6 +465,55 @@ small behavior differences as collateral:
 - Adding or subtracting a :class:`Day` with a :class:`Timedelta` is no longer supported.
 - Adding or subtracting a :class:`Day` offset to a timezone-aware :class:`Timestamp` or datetime-like may lead to an ambiguous or non-existent time, which will raise.
 
+.. _whatsnew_300.api_breaking.nan_vs_na:
+
+Changed treatment of NaN values in pyarrow and numpy-nullable floating dtypes
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Previously, when dealing with a nullable dtype (e.g. ``Float64Dtype`` or ``int64[pyarrow]``), ``NaN`` was treated as interchangeable with :class:`NA` in some circumstances but not others. This was done to make adoption easier, but caused some confusion (:issue:`32265`). In 3.0, an option ``"mode.nan_is_na"`` (default ``True``) controls whether to treat ``NaN`` as equivalent to :class:`NA`.
+
+With ``pd.set_option("mode.nan_is_na", True)`` (again, this is the default), ``NaN`` can be passed to constructors, ``__setitem__``, ``__contains__`` and be treated the same as :class:`NA`. The only change users will see is that arithmetic and ``np.ufunc`` operations that previously introduced ``NaN`` entries produce :class:`NA` entries instead:
+
+*Old behavior:*
+
+.. code-block:: ipython
+
+    In [2]: ser = pd.Series([0, None], dtype=pd.Float64Dtype())
+    In [3]: ser / 0
+    Out[3]:
+    0     NaN
+    1    <NA>
+    dtype: Float64
+
+*New behavior:*
+
+.. ipython:: python
+
+    ser = pd.Series([0, None], dtype=pd.Float64Dtype())
+    ser / 0
+
+By contrast, with ``pd.set_option("mode.nan_is_na", False)``, ``NaN`` is always considered distinct and specifically as a floating-point value, so cannot be used with integer dtypes:
+
+*Old behavior:*
+
+.. code-block:: ipython
+
+    In [2]: ser = pd.Series([1, np.nan], dtype=pd.Float64Dtype())
+    In [3]: ser[1]
+    Out[3]: <NA>
+
+*New behavior:*
+
+.. ipython:: python
+
+    pd.set_option("mode.nan_is_na", False)
+    ser = pd.Series([1, np.nan], dtype=pd.Float64Dtype())
+    ser[1]
+
+If we had passed ``pd.Int64Dtype()`` or ``"int64[pyarrow]"`` for the dtype in the latter example, this would raise, as a float ``NaN`` cannot be held by an integer dtype.
+
+With ``"mode.nan_is_na"`` set to ``False``, ``ser.to_numpy()`` (and ``frame.values`` and ``np.asarray(obj)``) will convert to ``object`` dtype if :class:`NA` entries are present, where before they would coerce to ``NaN``.  To retain a float numpy dtype, explicitly pass ``na_value=np.nan`` to :meth:`Series.to_numpy`.
+
 .. _whatsnew_300.api_breaking.deps:
 
 Increased minimum version for Python

pandas/_config/__init__.py

@@ -33,3 +33,8 @@
 def using_string_dtype() -> bool:
     _mode_options = _global_config["future"]
     return _mode_options["infer_string"]
+
+
+def is_nan_na() -> bool:
+    _mode_options = _global_config["mode"]
+    return _mode_options["nan_is_na"]