Merge remote-tracking branch 'upstream/main' into perf/read-csv

Author: Alvaro-Kothe

.github/workflows/codeql.yml

@@ -28,8 +28,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v5
-      - uses: github/codeql-action/init@v3
+      - uses: github/codeql-action/init@v4
         with:
           languages: ${{ matrix.language }}
-      - uses: github/codeql-action/autobuild@v3
-      - uses: github/codeql-action/analyze@v3
+      - uses: github/codeql-action/autobuild@v4
+      - uses: github/codeql-action/analyze@v4

.github/workflows/wheels.yml

@@ -162,7 +162,7 @@ jobs:
         run: echo "sdist_name=$(cd ./dist && ls -d */)" >> "$GITHUB_ENV"
 
       - name: Build wheels
-        uses: pypa/[email protected].0
+        uses: pypa/[email protected].1
         with:
          package-dir: ./dist/${{ startsWith(matrix.buildplat[1], 'macosx') && env.sdist_name || needs.build_sdist.outputs.sdist_file }}
         env:

doc/source/whatsnew/v3.0.0.rst

@@ -655,6 +655,7 @@ Other API changes
   an empty ``RangeIndex`` or empty ``Index`` with object dtype when determining
   the dtype of the resulting Index (:issue:`60797`)
 - :class:`IncompatibleFrequency` now subclasses ``TypeError`` instead of ``ValueError``. As a result, joins with mismatched frequencies now cast to object like other non-comparable joins, and arithmetic with indexes with mismatched frequencies align (:issue:`55782`)
+- :class:`Series` "flex" methods like :meth:`Series.add` no longer allow passing a :class:`DataFrame` for ``other``; use the DataFrame reversed method instead (:issue:`46179`)
 - :meth:`CategoricalIndex.append` no longer attempts to cast different-dtype indexes to the caller's dtype (:issue:`41626`)
 - :meth:`ExtensionDtype.construct_array_type` is now a regular method instead of a ``classmethod`` (:issue:`58663`)
 - Comparison operations between :class:`Index` and :class:`Series` now consistently return :class:`Series` regardless of which object is on the left or right (:issue:`36759`)
@@ -716,6 +717,7 @@ Other Deprecations
 - Deprecated using ``epoch`` date format in :meth:`DataFrame.to_json` and :meth:`Series.to_json`, use ``iso`` instead. (:issue:`57063`)
 - Deprecated allowing ``fill_value`` that cannot be held in the original dtype (excepting NA values for integer and bool dtypes) in :meth:`Series.unstack` and :meth:`DataFrame.unstack` (:issue:`12189`, :issue:`53868`)
 - Deprecated allowing ``fill_value`` that cannot be held in the original dtype (excepting NA values for integer and bool dtypes) in :meth:`Series.shift` and :meth:`DataFrame.shift` (:issue:`53802`)
+- Deprecated option "future.no_silent_downcasting", as it is no longer used. In a future version accessing this option will raise (:issue:`59502`)
 - Deprecated slicing on a :class:`Series` or :class:`DataFrame` with a :class:`DatetimeIndex` using a ``datetime.date`` object, explicitly cast to :class:`Timestamp` instead (:issue:`35830`)
 
 .. ---------------------------------------------------------------------------
@@ -873,6 +875,7 @@ Other Removals
 - Removed the ``method`` keyword in ``ExtensionArray.fillna``, implement ``ExtensionArray._pad_or_backfill`` instead (:issue:`53621`)
 - Removed the attribute ``dtypes`` from :class:`.DataFrameGroupBy` (:issue:`51997`)
 - Enforced deprecation of ``argmin``, ``argmax``, ``idxmin``, and ``idxmax`` returning a result when ``skipna=False`` and an NA value is encountered or all values are NA values; these operations will now raise in such cases (:issue:`33941`, :issue:`51276`)
+- Enforced deprecation of storage option "pyarrow_numpy" for :class:`StringDtype` (:issue:`60152`)
 - Removed specifying ``include_groups=True`` in :class:`.DataFrameGroupBy.apply` and :class:`.Resampler.apply` (:issue:`7155`)
 
 .. ---------------------------------------------------------------------------
@@ -1012,12 +1015,13 @@ Strings
 ^^^^^^^
 - Bug in :meth:`Series.str.zfill` raising ``AttributeError`` for :class:`ArrowDtype` (:issue:`61485`)
 - Bug in :meth:`Series.value_counts` would not respect ``sort=False`` for series having ``string`` dtype (:issue:`55224`)
+- Bug in multiplication with a :class:`StringDtype` incorrectly allowing multiplying by bools; explicitly cast to integers instead (:issue:`62595`)
 
 Interval
 ^^^^^^^^
 - :meth:`Index.is_monotonic_decreasing`, :meth:`Index.is_monotonic_increasing`, and :meth:`Index.is_unique` could incorrectly be ``False`` for an ``Index`` created from a slice of another ``Index``. (:issue:`57911`)
+- Bug in :class:`Index`, :class:`Series`, :class:`DataFrame` constructors when given a sequence of :class:`Interval` subclass objects casting them to :class:`Interval` (:issue:`46945`)
 - Bug in :func:`interval_range` where start and end numeric types were always cast to 64 bit (:issue:`57268`)
--
 
 Indexing
 ^^^^^^^^

pandas/_libs/include/pandas/parser/pd_parser.h

@@ -37,10 +37,8 @@ typedef struct {
   int (*parser_trim_buffers)(parser_t *);
   int (*tokenize_all_rows)(parser_t *, const char *);
   int (*tokenize_nrows)(parser_t *, size_t, const char *);
-  int64_t (*str_to_int64)(const char *, int64_t, int64_t, TokenizerError *,
-                          char);
-  uint64_t (*str_to_uint64)(uint_state *, const char *, int64_t, uint64_t,
-                            TokenizerError *, char);
+  int64_t (*str_to_int64)(const char *, int *, char);
+  uint64_t (*str_to_uint64)(uint_state *, const char *, int *, char);
   double (*xstrtod)(const char *, char **, char, char, char, int, int *, int *);
   double (*precise_xstrtod)(const char *, char **, char, char, char, int, int *,
                             int *);
@@ -88,12 +86,10 @@ static PandasParser_CAPI *PandasParserAPI = NULL;
   PandasParserAPI->tokenize_all_rows((self), (encoding_errors))
 #define tokenize_nrows(self, nrows, encoding_errors)                           \
   PandasParserAPI->tokenize_nrows((self), (nrows), (encoding_errors))
-#define str_to_int64(p_item, int_min, int_max, error, t_sep)                   \
-  PandasParserAPI->str_to_int64((p_item), (int_min), (int_max), (error),       \
-                                (t_sep))
-#define str_to_uint64(state, p_item, int_max, uint_max, error, t_sep)          \
-  PandasParserAPI->str_to_uint64((state), (p_item), (int_max), (uint_max),     \
-                                 (error), (t_sep))
+#define str_to_int64(p_item, error, t_sep)                                     \
+  PandasParserAPI->str_to_int64((p_item), (error), (t_sep))
+#define str_to_uint64(state, p_item, error, t_sep)                             \
+  PandasParserAPI->str_to_uint64((state), (p_item), (error), (t_sep))
 #define xstrtod(p, q, decimal, sci, tsep, skip_trailing, error, maybe_int)     \
   PandasParserAPI->xstrtod((p), (q), (decimal), (sci), (tsep),                 \
                            (skip_trailing), (error), (maybe_int))

pandas/_libs/include/pandas/parser/tokenizer.h

@@ -14,6 +14,10 @@ See LICENSE for the license
 #define PY_SSIZE_T_CLEAN
 #include <Python.h>
 
+#define ERROR_NO_DIGITS 1
+#define ERROR_OVERFLOW 2
+#define ERROR_INVALID_CHARS 3
+
 #include <stdint.h>
 
 #define STREAM_INIT_SIZE 32
@@ -46,13 +50,6 @@ See LICENSE for the license
  *        duplication of some file I/O.
  */
 
-typedef enum {
-  TOKENIZER_OK,
-  ERROR_NO_DIGITS,
-  ERROR_OVERFLOW,
-  ERROR_INVALID_CHARS,
-} TokenizerError;
-
 typedef enum {
   START_RECORD,
   START_FIELD,
@@ -211,10 +208,9 @@ void uint_state_init(uint_state *self);
 
 int uint64_conflict(uint_state *self);
 
-uint64_t str_to_uint64(uint_state *state, const char *p_item, int64_t int_max,
-                       uint64_t uint_max, TokenizerError *error, char tsep);
-int64_t str_to_int64(const char *p_item, int64_t int_min, int64_t int_max,
-                     TokenizerError *error, char tsep);
+uint64_t str_to_uint64(uint_state *state, const char *p_item, int *error,
+                       char tsep);
+int64_t str_to_int64(const char *p_item, int *error, char tsep);
 double xstrtod(const char *p, char **q, char decimal, char sci, char tsep,
                int skip_trailing, int *error, int *maybe_int);
 double precise_xstrtod(const char *p, char **q, char decimal, char sci,

pandas/_libs/lib.pyx

@@ -2255,7 +2255,8 @@ cpdef bint is_interval_array(ndarray values):
     for i in range(n):
         val = values[i]
 
-        if isinstance(val, Interval):
+        if type(val) is Interval:
+            # GH#46945 catch Interval exactly, excluding subclasses
             if closed is None:
                 closed = val.closed
                 numeric = (

pandas/_libs/parsers.pyx

@@ -63,11 +63,6 @@ from numpy cimport (
 cnp.import_array()
 
 from pandas._libs cimport util
-from pandas._libs.util cimport (
-    INT64_MAX,
-    INT64_MIN,
-    UINT64_MAX,
-)
 
 from pandas._libs import lib
 
@@ -149,10 +144,7 @@ cdef extern from "pandas/parser/tokenizer.h":
         SKIP_LINE
         FINISHED
 
-    ctypedef enum TokenizerError:
-        TOKENIZER_OK,
-        ERROR_OVERFLOW,
-        ERROR_INVALID_CHARS
+    enum: ERROR_OVERFLOW, ERROR_INVALID_CHARS
 
     ctypedef enum BadLineHandleMethod:
         ERROR,
@@ -284,10 +276,8 @@ cdef extern from "pandas/parser/pd_parser.h":
     int tokenize_all_rows(parser_t *self, const char *encoding_errors) nogil
     int tokenize_nrows(parser_t *self, size_t nrows, const char *encoding_errors) nogil
 
-    int64_t str_to_int64(char *p_item, int64_t int_min,
-                         int64_t int_max, TokenizerError *error, char tsep) nogil
-    uint64_t str_to_uint64(uint_state *state, char *p_item, int64_t int_max,
-                           uint64_t uint_max, TokenizerError *error, char tsep) nogil
+    int64_t str_to_int64(char *p_item,  int *error, char tsep) nogil
+    uint64_t str_to_uint64(uint_state *state, char *p_item, int *error, char tsep) nogil
 
     double xstrtod(const char *p, char **q, char decimal,
                    char sci, char tsep, int skip_trailing,
@@ -1797,7 +1787,7 @@ cdef int _try_uint64_nogil(parser_t *parser, int64_t col,
                            const kh_str_starts_t *na_hashset,
                            uint64_t *data, uint_state *state) nogil:
     cdef:
-        TokenizerError error = TOKENIZER_OK
+        int error = 0
         Py_ssize_t i, lines = line_end - line_start
         coliter_t it
         const char *word = NULL
@@ -1813,15 +1803,13 @@ cdef int _try_uint64_nogil(parser_t *parser, int64_t col,
                 data[i] = 0
                 continue
 
-            data[i] = str_to_uint64(state, word, INT64_MAX, UINT64_MAX,
-                                    &error, parser.thousands)
+            data[i] = str_to_uint64(state, word, &error, parser.thousands)
             if error != 0:
                 return error
     else:
         for i in range(lines):
             COLITER_NEXT(it, word)
-            data[i] = str_to_uint64(state, word, INT64_MAX, UINT64_MAX,
-                                    &error, parser.thousands)
+            data[i] = str_to_uint64(state, word, &error, parser.thousands)
             if error != 0:
                 return error
 
@@ -1832,7 +1820,7 @@ cdef _try_int64(parser_t *parser, int64_t col,
                 int64_t line_start, int64_t line_end,
                 bint na_filter, kh_str_starts_t *na_hashset, bint raise_on_float):
     cdef:
-        TokenizerError error = TOKENIZER_OK
+        int error = 0
         int na_count = 0
         Py_ssize_t lines
         coliter_t it
@@ -1859,13 +1847,13 @@ cdef _try_int64(parser_t *parser, int64_t col,
     return result, na_count
 
 
-cdef TokenizerError _try_int64_nogil(parser_t *parser, int64_t col,
-                                     int64_t line_start,
-                                     int64_t line_end, bint na_filter,
-                                     const kh_str_starts_t *na_hashset, int64_t NA,
-                                     int64_t *data, int *na_count) nogil:
+cdef int _try_int64_nogil(parser_t *parser, int64_t col,
+                          int64_t line_start,
+                          int64_t line_end, bint na_filter,
+                          const kh_str_starts_t *na_hashset, int64_t NA,
+                          int64_t *data, int *na_count) nogil:
     cdef:
-        TokenizerError error = TOKENIZER_OK
+        int error = 0
         Py_ssize_t i, lines = line_end - line_start
         coliter_t it
         const char *word = NULL
@@ -1882,16 +1870,14 @@ cdef TokenizerError _try_int64_nogil(parser_t *parser, int64_t col,
                 data[i] = NA
                 continue
 
-            data[i] = str_to_int64(word, INT64_MIN, INT64_MAX,
-                                   &error, parser.thousands)
-            if error != TOKENIZER_OK:
+            data[i] = str_to_int64(word, &error, parser.thousands)
+            if error != 0:
                 return error
     else:
         for i in range(lines):
             COLITER_NEXT(it, word)
-            data[i] = str_to_int64(word, INT64_MIN, INT64_MAX,
-                                   &error, parser.thousands)
-            if error != TOKENIZER_OK:
+            data[i] = str_to_int64(word, &error, parser.thousands)
+            if error != 0:
                 return error
 
     return error