Merge remote-tracking branch 'upstream/main' into bugfix/60678-pdNA-error

Author: akj2018

.github/workflows/unit-tests.yml

@@ -22,10 +22,11 @@ defaults:
 
 jobs:
   ubuntu:
-    runs-on: ubuntu-22.04
+    runs-on: ${{ matrix.platform }}
     timeout-minutes: 90
     strategy:
       matrix:
+        platform: [ubuntu-22.04, ubuntu-24.04-arm]
         env_file: [actions-310.yaml, actions-311.yaml, actions-312.yaml]
         # Prevent the include jobs from overriding other jobs
         pattern: [""]
@@ -35,9 +36,11 @@ jobs:
             env_file: actions-311-downstream_compat.yaml
             pattern: "not slow and not network and not single_cpu"
             pytest_target: "pandas/tests/test_downstream.py"
+            platform: ubuntu-22.04
           - name: "Minimum Versions"
             env_file: actions-310-minimum_versions.yaml
             pattern: "not slow and not network and not single_cpu"
+            platform: ubuntu-22.04
           - name: "Locale: it_IT"
             env_file: actions-311.yaml
             pattern: "not slow and not network and not single_cpu"
@@ -48,6 +51,7 @@ jobs:
             # Also install it_IT (its encoding is ISO8859-1) but do not activate it.
             # It will be temporarily activated during tests with locale.setlocale
             extra_loc: "it_IT"
+            platform: ubuntu-22.04
           - name: "Locale: zh_CN"
             env_file: actions-311.yaml
             pattern: "not slow and not network and not single_cpu"
@@ -58,25 +62,32 @@ jobs:
             # Also install zh_CN (its encoding is gb2312) but do not activate it.
             # It will be temporarily activated during tests with locale.setlocale
             extra_loc: "zh_CN"
+            platform: ubuntu-22.04
           - name: "Future infer strings"
             env_file: actions-312.yaml
             pandas_future_infer_string: "1"
+            platform: ubuntu-22.04
           - name: "Future infer strings (without pyarrow)"
             env_file: actions-311.yaml
             pandas_future_infer_string: "1"
+            platform: ubuntu-22.04
           - name: "Pypy"
             env_file: actions-pypy-39.yaml
             pattern: "not slow and not network and not single_cpu"
             test_args: "--max-worker-restart 0"
+            platform: ubuntu-22.04
           - name: "Numpy Dev"
             env_file: actions-311-numpydev.yaml
             pattern: "not slow and not network and not single_cpu"
             test_args: "-W error::DeprecationWarning -W error::FutureWarning"
+            platform: ubuntu-22.04
           - name: "Pyarrow Nightly"
             env_file: actions-311-pyarrownightly.yaml
             pattern: "not slow and not network and not single_cpu"
+            pandas_future_infer_string: "1"
+            platform: ubuntu-22.04
       fail-fast: false
-    name: ${{ matrix.name || format('ubuntu-latest {0}', matrix.env_file) }}
+    name: ${{ matrix.name || format('{0} {1}', matrix.platform, matrix.env_file) }}
     env:
       PATTERN: ${{ matrix.pattern }}
       LANG: ${{ matrix.lang || 'C.UTF-8' }}
@@ -91,7 +102,7 @@ jobs:
       REMOVE_PYARROW: ${{ matrix.name == 'Future infer strings (without pyarrow)' && '1' || '0' }}
     concurrency:
       # https://github.community/t/concurrecy-not-work-for-push/183068/7
-      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.extra_apt || '' }}-${{ matrix.pandas_future_infer_string }}
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.extra_apt || '' }}-${{ matrix.pandas_future_infer_string }}-${{ matrix.platform }}
       cancel-in-progress: true
 
     services:
@@ -419,20 +430,20 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Set up Python for Pyodide
+      - name: Set up Python for pyodide-build
         id: setup-python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.11.3'
+          python-version: '3.12'
 
       - name: Set up Emscripten toolchain
         uses: mymindstorm/setup-emsdk@v14
         with:
-          version: '3.1.46'
+          version: '3.1.58'
           actions-cache-folder: emsdk-cache
 
       - name: Install pyodide-build
-        run: pip install "pyodide-build==0.25.1"
+        run: pip install "pyodide-build>=0.29.2"
 
       - name: Build pandas for Pyodide
         run: |
@@ -441,10 +452,13 @@ jobs:
       - name: Set up Node.js
         uses: actions/setup-node@v4
         with:
-          node-version: '18'
+          node-version: '20'
 
       - name: Set up Pyodide virtual environment
+        env:
+          pyodide-version: '0.27.1'
         run: |
+          pyodide xbuildenv install ${{ env.pyodide-version }}
           pyodide venv .venv-pyodide
           source .venv-pyodide/bin/activate
           pip install dist/*.whl

.github/workflows/wheels.yml

@@ -94,6 +94,7 @@ jobs:
         buildplat:
         - [ubuntu-22.04, manylinux_x86_64]
         - [ubuntu-22.04, musllinux_x86_64]
+        - [ubuntu-24.04-arm, manylinux_aarch64]
         - [macos-13, macosx_x86_64]
         # Note: M1 images on Github Actions start from macOS 14
         - [macos-14, macosx_arm64]

.gitignore

@@ -137,3 +137,7 @@ doc/source/savefig/
 # Interactive terminal generated files #
 ########################################
 .jupyterlite.doit.db
+
+# Pyodide/WASM related files #
+##############################
+/.pyodide-xbuildenv-*

ci/code_checks.sh

@@ -72,21 +72,17 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         -i "pandas.Series.dt PR01" `# Accessors are implemented as classes, but we do not document the Parameters section` \
         -i "pandas.Period.freq GL08" \
         -i "pandas.Period.ordinal GL08" \
-        -i "pandas.RangeIndex.from_range PR01,SA01" \
         -i "pandas.Timedelta.max PR02" \
         -i "pandas.Timedelta.min PR02" \
         -i "pandas.Timedelta.resolution PR02" \
         -i "pandas.Timestamp.max PR02" \
         -i "pandas.Timestamp.min PR02" \
         -i "pandas.Timestamp.resolution PR02" \
         -i "pandas.Timestamp.tzinfo GL08" \
-        -i "pandas.arrays.ArrowExtensionArray PR07,SA01" \
-        -i "pandas.arrays.TimedeltaArray PR07,SA01" \
         -i "pandas.core.groupby.DataFrameGroupBy.plot PR02" \
         -i "pandas.core.groupby.SeriesGroupBy.plot PR02" \
         -i "pandas.core.resample.Resampler.quantile PR01,PR07" \
         -i "pandas.core.resample.Resampler.transform PR01,RT03,SA01" \
-        -i "pandas.plotting.andrews_curves RT03,SA01" \
         -i "pandas.tseries.offsets.BDay PR02,SA01" \
         -i "pandas.tseries.offsets.BQuarterBegin.is_on_offset GL08" \
         -i "pandas.tseries.offsets.BQuarterBegin.n GL08" \

ci/deps/actions-311-pyarrownightly.yaml

@@ -23,7 +23,7 @@ dependencies:
 
   - pip:
     - "tzdata>=2022.7"
-    - "--extra-index-url https://pypi.fury.io/arrow-nightlies/"
+    - "--extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple"
     - "--prefer-binary"
     - "--pre"
     - "pyarrow"

doc/source/getting_started/comparison/comparison_with_spss.rst

@@ -0,0 +1,229 @@
+.. _compare_with_spss:
+
+{{ header }}
+
+Comparison with SPSS
+********************
+For potential users coming from `SPSS <https://www.ibm.com/spss>`__, this page is meant to demonstrate
+how various SPSS operations would be performed using pandas.
+
+.. include:: includes/introduction.rst
+
+Data structures
+---------------
+
+General terminology translation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. csv-table::
+    :header: "pandas", "SPSS"
+    :widths: 20, 20
+
+    :class:`DataFrame`, data file
+    column, variable
+    row, case
+    groupby, split file
+    :class:`NaN`, system-missing
+
+:class:`DataFrame`
+~~~~~~~~~~~~~~~~~~
+
+A :class:`DataFrame` in pandas is analogous to an SPSS data file - a two-dimensional
+data source with labeled columns that can be of different types. As will be shown in this
+document, almost any operation that can be performed in SPSS can also be accomplished in pandas.
+
+:class:`Series`
+~~~~~~~~~~~~~~~
+
+A :class:`Series` is the data structure that represents one column of a :class:`DataFrame`. SPSS doesn't have a
+separate data structure for a single variable, but in general, working with a :class:`Series` is analogous
+to working with a variable in SPSS.
+
+:class:`Index`
+~~~~~~~~~~~~~~
+
+Every :class:`DataFrame` and :class:`Series` has an :class:`Index` -- labels on the *rows* of the data. SPSS does not
+have an exact analogue, as cases are simply numbered sequentially from 1. In pandas, if no index is
+specified, a :class:`RangeIndex` is used by default (first row = 0, second row = 1, and so on).
+
+While using a labeled :class:`Index` or :class:`MultiIndex` can enable sophisticated analyses and is ultimately an
+important part of pandas to understand, for this comparison we will essentially ignore the :class:`Index` and
+just treat the :class:`DataFrame` as a collection of columns. Please see the :ref:`indexing documentation<indexing>`
+for much more on how to use an :class:`Index` effectively.
+
+
+Copies vs. in place operations
+------------------------------
+
+.. include:: includes/copies.rst
+
+
+Data input / output
+-------------------
+
+Reading external data
+~~~~~~~~~~~~~~~~~~~~~
+
+Like SPSS, pandas provides utilities for reading in data from many formats. The ``tips`` dataset, found within
+the pandas tests (`csv <https://raw.githubusercontent.com/pandas-dev/pandas/main/pandas/tests/io/data/csv/tips.csv>`_)
+will be used in many of the following examples.
+
+In SPSS, you would use File > Open > Data to import a CSV file:
+
+.. code-block:: text
+
+    FILE > OPEN > DATA
+    /TYPE=CSV
+    /FILE='tips.csv'
+    /DELIMITERS=","
+    /FIRSTCASE=2
+    /VARIABLES=col1 col2 col3.
+
+The pandas equivalent would use :func:`read_csv`:
+
+.. code-block:: python
+
+   url = (
+       "https://raw.githubusercontent.com/pandas-dev"
+       "/pandas/main/pandas/tests/io/data/csv/tips.csv"
+   )
+   tips = pd.read_csv(url)
+   tips
+
+Like SPSS's data import wizard, ``read_csv`` can take a number of parameters to specify how the data should be parsed.
+For example, if the data was instead tab delimited, and did not have column names, the pandas command would be:
+
+.. code-block:: python
+
+   tips = pd.read_csv("tips.csv", sep="\t", header=None)
+
+   # alternatively, read_table is an alias to read_csv with tab delimiter
+   tips = pd.read_table("tips.csv", header=None)
+
+
+Data operations
+---------------
+
+Filtering
+~~~~~~~~~
+
+In SPSS, filtering is done through Data > Select Cases:
+
+.. code-block:: text
+
+    SELECT IF (total_bill > 10).
+    EXECUTE.
+
+In pandas, boolean indexing can be used:
+
+.. code-block:: python
+
+    tips[tips["total_bill"] > 10]
+
+
+Sorting
+~~~~~~~
+
+In SPSS, sorting is done through Data > Sort Cases:
+
+.. code-block:: text
+
+    SORT CASES BY sex total_bill.
+    EXECUTE.
+
+In pandas, this would be written as:
+
+.. code-block:: python
+
+    tips.sort_values(["sex", "total_bill"])
+
+
+String processing
+-----------------
+
+Finding length of string
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+In SPSS:
+
+.. code-block:: text
+
+    COMPUTE length = LENGTH(time).
+    EXECUTE.
+
+.. include:: includes/length.rst
+
+
+Changing case
+~~~~~~~~~~~~~
+
+In SPSS:
+
+.. code-block:: text
+
+    COMPUTE upper = UPCASE(time).
+    COMPUTE lower = LOWER(time).
+    EXECUTE.
+
+.. include:: includes/case.rst
+
+
+Merging
+-------
+
+In SPSS, merging data files is done through Data > Merge Files.
+
+.. include:: includes/merge_setup.rst
+.. include:: includes/merge.rst
+
+
+GroupBy operations
+------------------
+
+Split-file processing
+~~~~~~~~~~~~~~~~~~~~~
+
+In SPSS, split-file analysis is done through Data > Split File:
+
+.. code-block:: text
+
+    SORT CASES BY sex.
+    SPLIT FILE BY sex.
+    DESCRIPTIVES VARIABLES=total_bill tip
+      /STATISTICS=MEAN STDDEV MIN MAX.
+
+The pandas equivalent would be:
+
+.. code-block:: python
+
+    tips.groupby("sex")[["total_bill", "tip"]].agg(["mean", "std", "min", "max"])
+
+
+Missing data
+------------
+
+SPSS uses the period (``.``) for numeric missing values and blank spaces for string missing values.
+pandas uses ``NaN`` (Not a Number) for numeric missing values and ``None`` or ``NaN`` for string
+missing values.
+
+.. include:: includes/missing.rst
+
+
+Other considerations
+--------------------
+
+Output management
+-----------------
+
+While pandas does not have a direct equivalent to SPSS's Output Management System (OMS), you can
+capture and export results in various ways:
+
+.. code-block:: python
+
+   # Save summary statistics to CSV
+   tips.groupby('sex')[['total_bill', 'tip']].mean().to_csv('summary.csv')
+
+   # Save multiple results to Excel sheets
+   with pd.ExcelWriter('results.xlsx') as writer:
+       tips.describe().to_excel(writer, sheet_name='Descriptives')
+       tips.groupby('sex').mean().to_excel(writer, sheet_name='Means by Gender')

doc/source/getting_started/comparison/index.rst

@@ -14,3 +14,4 @@ Comparison with other tools
     comparison_with_spreadsheets
     comparison_with_sas
     comparison_with_stata
+    comparison_with_spss