Merge branch 'main' into bug-60228

Author: jbrockmendel

doc/source/user_guide/scale.rst

@@ -164,35 +164,35 @@ files. Each file in the directory represents a different year of the entire data
 .. ipython:: python
    :okwarning:
 
-   import pathlib
+   import glob
+   import tempfile
 
    N = 12
    starts = [f"20{i:>02d}-01-01" for i in range(N)]
    ends = [f"20{i:>02d}-12-13" for i in range(N)]
 
-   pathlib.Path("data/timeseries").mkdir(exist_ok=True)
+   tmpdir = tempfile.TemporaryDirectory(ignore_cleanup_errors=True)
 
    for i, (start, end) in enumerate(zip(starts, ends)):
        ts = make_timeseries(start=start, end=end, freq="1min", seed=i)
-       ts.to_parquet(f"data/timeseries/ts-{i:0>2d}.parquet")
+       ts.to_parquet(f"{tmpdir.name}/ts-{i:0>2d}.parquet")
 
 
 ::
 
-   data
-   └── timeseries
-       ├── ts-00.parquet
-       ├── ts-01.parquet
-       ├── ts-02.parquet
-       ├── ts-03.parquet
-       ├── ts-04.parquet
-       ├── ts-05.parquet
-       ├── ts-06.parquet
-       ├── ts-07.parquet
-       ├── ts-08.parquet
-       ├── ts-09.parquet
-       ├── ts-10.parquet
-       └── ts-11.parquet
+   tmpdir
+   ├── ts-00.parquet
+   ├── ts-01.parquet
+   ├── ts-02.parquet
+   ├── ts-03.parquet
+   ├── ts-04.parquet
+   ├── ts-05.parquet
+   ├── ts-06.parquet
+   ├── ts-07.parquet
+   ├── ts-08.parquet
+   ├── ts-09.parquet
+   ├── ts-10.parquet
+   └── ts-11.parquet
 
 Now we'll implement an out-of-core :meth:`pandas.Series.value_counts`. The peak memory usage of this
 workflow is the single largest chunk, plus a small series storing the unique value
@@ -202,13 +202,18 @@ work for arbitrary-sized datasets.
 .. ipython:: python
 
    %%time
-   files = pathlib.Path("data/timeseries/").glob("ts*.parquet")
+   files = glob.iglob(f"{tmpdir.name}/ts*.parquet")
    counts = pd.Series(dtype=int)
    for path in files:
        df = pd.read_parquet(path)
        counts = counts.add(df["name"].value_counts(), fill_value=0)
    counts.astype(int)
 
+.. ipython:: python
+   :suppress:
+
+   tmpdir.cleanup()
+
 Some readers, like :meth:`pandas.read_csv`, offer parameters to control the
 ``chunksize`` when reading a single file.
 

doc/source/whatsnew/v2.3.2.rst

@@ -22,6 +22,8 @@ become the default string dtype in pandas 3.0. See
 
 Bug fixes
 ^^^^^^^^^
+- Fix :meth:`~Series.str.isdigit` to correctly recognize unicode superscript
+  characters as digits for :class:`StringDtype` backed by PyArrow (:issue:`61466`)
 - Fix :meth:`~DataFrame.to_json` with ``orient="table"`` to correctly use the
   "string" type in the JSON Table Schema for :class:`StringDtype` columns
   (:issue:`61889`)

doc/source/whatsnew/v3.0.0.rst

@@ -666,6 +666,7 @@ Other Deprecations
 - Deprecated using ``epoch`` date format in :meth:`DataFrame.to_json` and :meth:`Series.to_json`, use ``iso`` instead. (:issue:`57063`)
 - Deprecated allowing ``fill_value`` that cannot be held in the original dtype (excepting NA values for integer and bool dtypes) in :meth:`Series.unstack` and :meth:`DataFrame.unstack` (:issue:`12189`, :issue:`53868`)
 - Deprecated allowing ``fill_value`` that cannot be held in the original dtype (excepting NA values for integer and bool dtypes) in :meth:`Series.shift` and :meth:`DataFrame.shift` (:issue:`53802`)
+- Deprecated slicing on a :class:`Series` or :class:`DataFrame` with a :class:`DatetimeIndex` using a ``datetime.date`` object, explicitly cast to :class:`Timestamp` instead (:issue:`35830`)
 
 .. ---------------------------------------------------------------------------
 .. _whatsnew_300.prior_deprecations:
@@ -998,6 +999,7 @@ MultiIndex
 - Bug in :class:`DataFrame` arithmetic operations in case of unaligned MultiIndex columns (:issue:`60498`)
 - Bug in :class:`DataFrame` arithmetic operations with :class:`Series` in case of unaligned MultiIndex (:issue:`61009`)
 - Bug in :meth:`MultiIndex.from_tuples` causing wrong output with input of type tuples having NaN values (:issue:`60695`, :issue:`60988`)
+- Bug in :meth:`DataFrame.__setitem__` where column alignment logic would reindex the assigned value with an empty index, incorrectly setting all values to ``NaN``.(:issue:`61841`)
 - Bug in :meth:`DataFrame.reindex` and :meth:`Series.reindex` where reindexing :class:`Index` to a :class:`MultiIndex` would incorrectly set all values to ``NaN``.(:issue:`60923`)
 
 I/O

pandas/core/arrays/_arrow_string_mixins.py

@@ -15,6 +15,7 @@
 from pandas.compat import (
     HAS_PYARROW,
     pa_version_under17p0,
+    pa_version_under21p0,
 )
 
 if HAS_PYARROW:
@@ -267,6 +268,12 @@ def _str_isdecimal(self):
         return self._convert_bool_result(result)
 
     def _str_isdigit(self):
+        if pa_version_under21p0:
+            # https://github.com/pandas-dev/pandas/issues/61466
+            res_list = self._apply_elementwise(str.isdigit)
+            return self._convert_bool_result(
+                pa.chunked_array(res_list, type=pa.bool_())
+            )
         result = pc.utf8_is_digit(self._pa_array)
         return self._convert_bool_result(result)
 

pandas/core/frame.py

@@ -2132,12 +2132,12 @@ def from_records(
         """
         Convert structured or record ndarray to DataFrame.
 
-        Creates a DataFrame object from a structured ndarray, or sequence of
+        Creates a DataFrame object from a structured ndarray, or iterable of
         tuples or dicts.
 
         Parameters
         ----------
-        data : structured ndarray, sequence of tuples or dicts
+        data : structured ndarray, iterable of tuples or dicts
             Structured input data.
         index : str, list of fields, array-like
             Field of array to use as the index, alternately a specific set of
@@ -4452,6 +4452,11 @@ def _set_item_frame_value(self, key, value: DataFrame) -> None:
                 loc, (slice, Series, np.ndarray, Index)
             ):
                 cols_droplevel = maybe_droplevels(cols, key)
+                if (
+                    not isinstance(cols_droplevel, MultiIndex)
+                    and not cols_droplevel.any()
+                ):
+                    return
                 if len(cols_droplevel) and not cols_droplevel.equals(value.columns):
                     value = value.reindex(cols_droplevel, axis=1)
 

pandas/core/indexes/datetimes.py

@@ -26,11 +26,13 @@
     to_offset,
 )
 from pandas._libs.tslibs.offsets import prefix_mapping
+from pandas.errors import Pandas4Warning
 from pandas.util._decorators import (
     cache_readonly,
     doc,
     set_module,
 )
+from pandas.util._exceptions import find_stack_level
 
 from pandas.core.dtypes.common import is_scalar
 from pandas.core.dtypes.dtypes import (
@@ -645,6 +647,13 @@ def _maybe_cast_slice_bound(self, label, side: str):
             # Pandas supports slicing with dates, treated as datetimes at midnight.
             # https://github.com/pandas-dev/pandas/issues/31501
             label = Timestamp(label).to_pydatetime()
+            warnings.warn(
+                # GH#35830 deprecate last remaining inconsistent date treatment
+                "Slicing with a datetime.date object is deprecated. "
+                "Explicitly cast to Timestamp instead.",
+                Pandas4Warning,
+                stacklevel=find_stack_level(),
+            )
 
         label = super()._maybe_cast_slice_bound(label, side)
         self._data._assert_tzawareness_compat(label)

pandas/core/reshape/melt.py

@@ -5,6 +5,8 @@
 
 import numpy as np
 
+from pandas.util._decorators import set_module
+
 from pandas.core.dtypes.common import (
     is_iterator,
     is_list_like,
@@ -39,6 +41,7 @@ def ensure_list_vars(arg_vars, variable: str, columns) -> list:
         return []
 
 
+@set_module("pandas")
 def melt(
     frame: DataFrame,
     id_vars=None,
@@ -275,6 +278,7 @@ def melt(
     return result
 
 
+@set_module("pandas")
 def lreshape(data: DataFrame, groups: dict, dropna: bool = True) -> DataFrame:
     """
     Reshape wide-format data to long. Generalized inverse of DataFrame.pivot.
@@ -361,6 +365,7 @@ def lreshape(data: DataFrame, groups: dict, dropna: bool = True) -> DataFrame:
     return data._constructor(mdata, columns=id_cols + pivot_cols)
 
 
+@set_module("pandas")
 def wide_to_long(
     df: DataFrame, stubnames, i, j, sep: str = "", suffix: str = r"\d+"
 ) -> DataFrame:

pandas/core/reshape/pivot.py

@@ -10,6 +10,7 @@
 import numpy as np
 
 from pandas._libs import lib
+from pandas.util._decorators import set_module
 
 from pandas.core.dtypes.cast import maybe_downcast_to_dtype
 from pandas.core.dtypes.common import (
@@ -50,6 +51,7 @@
     from pandas import DataFrame
 
 
+@set_module("pandas")
 def pivot_table(
     data: DataFrame,
     values=None,
@@ -699,6 +701,7 @@ def _convert_by(by):
     return by
 
 
+@set_module("pandas")
 def pivot(
     data: DataFrame,
     *,
@@ -917,6 +920,7 @@ def pivot(
     return result
 
 
+@set_module("pandas")
 def crosstab(
     index,
     columns,

pandas/core/reshape/tile.py

@@ -17,6 +17,7 @@
     Timestamp,
     lib,
 )
+from pandas.util._decorators import set_module
 
 from pandas.core.dtypes.common import (
     ensure_platform_int,
@@ -51,6 +52,7 @@
     )
 
 
+@set_module("pandas")
 def cut(
     x,
     bins,
@@ -287,6 +289,7 @@ def cut(
     return _postprocess_for_cut(fac, bins, retbins, original)
 
 
+@set_module("pandas")
 def qcut(
     x,
     q,

pandas/core/strings/accessor.py

@@ -3602,16 +3602,26 @@ def casefold(self):
     Series.str.isupper : Check whether all characters are uppercase.
     Series.str.istitle : Check whether all characters are titlecase.
 
-    Examples
-    --------
+    Notes
+    -----
     Similar to ``str.isdecimal`` but also includes special digits, like
     superscripted and subscripted digits in unicode.
 
+    The exact behavior of this method, i.e. which unicode characters are
+    considered as digits, depends on the backend used for string operations,
+    and there can be small differences.
+    For example, Python considers the ³ superscript character as a digit, but
+    not the ⅕ fraction character, while PyArrow considers both as digits. For
+    simple (ascii) decimal numbers, the behaviour is consistent.
+
+    Examples
+    --------
+
     >>> s3 = pd.Series(['23', '³', '⅕', ''])
     >>> s3.str.isdigit()
     0     True
-    1    False
-    2    False
+    1     True
+    2     True
     3    False
     dtype: bool
     """