Merge remote-tracking branch 'upstream/main' into feature-plot-weighted-kde

Author: fbourgey

.github/workflows/unit-tests.yml

@@ -57,6 +57,10 @@ jobs:
             # Also install zh_CN (its encoding is gb2312) but do not activate it.
             # It will be temporarily activated during tests with locale.setlocale
             extra_loc: "zh_CN"
+          - name: "Future infer strings"
+            env_file: actions-311.yaml
+            pattern: "not slow and not network and not single_cpu"
+            pandas_future_infer_string: "1"
           - name: "Pypy"
             env_file: actions-pypy-39.yaml
             pattern: "not slow and not network and not single_cpu"
@@ -75,6 +79,7 @@ jobs:
       LANG: ${{ matrix.lang || 'C.UTF-8' }}
       LC_ALL: ${{ matrix.lc_all || '' }}
       PANDAS_CI: '1'
+      PANDAS_FUTURE_INFER_STRING: ${{ matrix.pandas_future_infer_string || '0' }}
       TEST_ARGS: ${{ matrix.test_args || '' }}
       PYTEST_WORKERS: 'auto'
       PYTEST_TARGET: ${{ matrix.pytest_target || 'pandas' }}

ci/code_checks.sh

@@ -304,7 +304,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.api.types.is_re PR07,SA01" \
         -i "pandas.api.types.is_re_compilable PR07,SA01" \
         -i "pandas.api.types.is_sparse SA01" \
-        -i "pandas.api.types.is_string_dtype SA01" \
         -i "pandas.api.types.is_timedelta64_ns_dtype SA01" \
         -i "pandas.api.types.pandas_dtype PR07,RT03,SA01" \
         -i "pandas.api.types.union_categoricals RT03,SA01" \

ci/run_tests.sh

@@ -16,5 +16,11 @@ if [[ "$PATTERN" ]]; then
   PYTEST_CMD="$PYTEST_CMD -m \"$PATTERN\""
 fi
 
+# temporarily let pytest always succeed (many tests are not yet passing in the
+# build enabling the future string dtype)
+if [[ "$PANDAS_FUTURE_INFER_STRING" == "1" ]]; then
+  PYTEST_CMD="$PYTEST_CMD || true"
+fi
+
 echo $PYTEST_CMD
 sh -c "$PYTEST_CMD"

doc/source/getting_started/comparison/comparison_with_sql.rst

@@ -505,7 +505,7 @@ DELETE
     DELETE FROM tips
     WHERE tip > 9;
 
-In pandas we select the rows that should remain instead of deleting them:
+In pandas we select the rows that should remain instead of deleting the rows that should be removed:
 
 .. ipython:: python
 

doc/source/whatsnew/v3.0.0.rst

@@ -583,6 +583,7 @@ I/O
 - Bug in :meth:`read_excel` raising ``ValueError`` when passing array of boolean values when ``dtype="boolean"``. (:issue:`58159`)
 - Bug in :meth:`read_json` not validating the ``typ`` argument to not be exactly ``"frame"`` or ``"series"`` (:issue:`59124`)
 - Bug in :meth:`read_stata` raising ``KeyError`` when input file is stored in big-endian format and contains strL data. (:issue:`58638`)
+- Bug in :meth:`read_stata` where extreme value integers were incorrectly interpreted as missing for format versions 111 and prior (:issue:`58130`)
 
 Period
 ^^^^^^

pandas/core/apply.py

@@ -486,20 +486,14 @@ def compute_dict_like(
                 cols = df[key]
 
                 if cols.ndim == 1:
-                    series_list = [obj._gotitem(key, ndim=1, subset=cols)]
+                    series = obj._gotitem(key, ndim=1, subset=cols)
+                    results.append(getattr(series, op_name)(how, **kwargs))
+                    keys.append(key)
                 else:
-                    series_list = []
-                    for index in range(cols.shape[1]):
-                        col = cols.iloc[:, index]
-
+                    for _, col in cols.items():
                         series = obj._gotitem(key, ndim=1, subset=col)
-                        series_list.append(series)
-
-                for series in series_list:
-                    result = getattr(series, op_name)(how, **kwargs)
-                    results.append(result)
-                    keys.append(key)
-
+                        results.append(getattr(series, op_name)(how, **kwargs))
+                        keys.append(key)
         else:
             results = [
                 getattr(obj._gotitem(key, ndim=1), op_name)(how, **kwargs)

pandas/core/config_init.py

@@ -858,7 +858,7 @@ def register_converter_cb(key: str) -> None:
 with cf.config_prefix("future"):
     cf.register_option(
         "infer_string",
-        False,
+        True if os.environ.get("PANDAS_FUTURE_INFER_STRING", "0") == "1" else False,
         "Whether to infer sequence of str objects as pyarrow string "
         "dtype, which will be the default in pandas 3.0 "
         "(at which point this option will be deprecated).",

pandas/core/dtypes/common.py

@@ -558,6 +558,11 @@ def is_string_dtype(arr_or_dtype) -> bool:
     boolean
         Whether or not the array or dtype is of the string dtype.
 
+    See Also
+    --------
+    api.types.is_string_dtype : Check whether the provided array or dtype
+                                is of the string dtype.
+
     Examples
     --------
     >>> from pandas.api.types import is_string_dtype

pandas/core/frame.py

@@ -531,6 +531,7 @@ class DataFrame(NDFrame, OpsMixin):
         will perform column selection instead.
     dtype : dtype, default None
         Data type to force. Only a single dtype is allowed. If None, infer.
+        If ``data`` is DataFrame then is ignored.
     copy : bool or None, default None
         Copy data from inputs.
         For dict data, the default of None behaves like ``copy=True``.  For DataFrame

pandas/core/series.py

@@ -256,6 +256,7 @@ class Series(base.IndexOpsMixin, NDFrame):  # type: ignore[misc]
         Data type for the output Series. If not specified, this will be
         inferred from `data`.
         See the :ref:`user guide <basics.dtypes>` for more usages.
+        If ``data`` is Series then is ignored.
     name : Hashable, default None
         The name to give to the Series.
     copy : bool, default False