Merge remote-tracking branch 'upstream/main' into lmoment

Author: mdhaber

.git-blame-ignore-revs

@@ -36,3 +36,5 @@ ecef3490da68a0c53ba543c618bab0c8e15dccee
 7b921fd28659b02544bfb46368ddadd1048b37aa
 # Style cleanup to always `import numpy as np`
 ceafa8e730887b81cf10d483ce375559ebd1de09
+# Clean-up for UP031, UP032 lint rules (gh-21029)
+d1b5af016e907e037136b7a38e485437165490f2

.github/PULL_REQUEST_TEMPLATE.md

@@ -5,7 +5,9 @@ http://scipy.github.io/devdocs/dev/contributor/development_workflow.html#checkli
 
 Also, please name and describe your PR as you would write a
 commit message:
-http://scipy.github.io/devdocs/dev/contributor/development_workflow.html#writing-the-commit-message
+http://scipy.github.io/devdocs/dev/contributor/development_workflow.html#writing-the-commit-message.
+However, please only include an issue number in the description, not the title,
+and please ensure that any code names containing underscores are enclosed in backticks.
 
 Depending on your changes, you can skip CI operations and save time and energy: 
 http://scipy.github.io/devdocs/dev/contributor/continuous_integration.html#skipping

.github/label-globs.yml

@@ -11,6 +11,11 @@ scipy.constants:
   - any-glob-to-any-file:
     - scipy/constants/**
 
+scipy.differentiate:
+- changed-files:
+  - any-glob-to-any-file:
+    - scipy/differentiate/**
+
 scipy.fft:
 - changed-files:
   - any-glob-to-any-file:
@@ -73,8 +78,11 @@ scipy.signal:
 
 scipy.sparse:
 - changed-files:
-  - any-glob-to-any-file:
+  - all-globs-to-any-file:
     - scipy/sparse/**
+    # don't match the `csgraph` or `linalg` submodules
+    - '!scipy/sparse/csgraph/**'
+    - '!scipy/sparse/linalg/**'
 
 scipy.sparse.csgraph:
 - changed-files:

.github/workflows/array_api.yml

@@ -15,6 +15,18 @@ permissions:
 env:
   CCACHE_DIR: "${{ github.workspace }}/.ccache"
   INSTALLDIR: "build-install"
+  XP_TESTS: >-
+    -t scipy.cluster
+    -t scipy.constants
+    -t scipy.fft
+    -t scipy.special.tests.test_support_alternative_backends
+    -t scipy._lib.tests.test_array_api
+    -t scipy._lib.tests.test__util
+    -t scipy.differentiate.tests.test_differentiate
+    -t scipy.integrate.tests.test_tanhsinh
+    -t scipy.optimize.tests.test_chandrupatla
+    -t scipy.stats
+    -t scipy.ndimage
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
@@ -93,11 +105,5 @@ jobs:
     - name: Test SciPy
       run: |
         export OMP_NUM_THREADS=2
-        # expand as new modules are added
-        python dev.py --no-build test -b all -s cluster -- --durations 3 --timeout=60
-        python dev.py --no-build test -b all -s constants -- --durations 3 --timeout=60
-        python dev.py --no-build test -b all -s fft -- --durations 3 --timeout=60
-        python dev.py --no-build test -b all -t scipy.special.tests.test_support_alternative_backends -- --durations 3 --timeout=60
-        python dev.py --no-build test -b all -t scipy._lib.tests.test_array_api -- --durations 3 --timeout=60
-        python dev.py --no-build test -b all -t scipy._lib.tests.test__util -- --durations 3 --timeout=60
-        python dev.py --no-build test -b all -s stats -- --durations 3 --timeout=60
+        # expand as more modules are supported by adding to `XP_TESTS` above
+        python dev.py --no-build test -b all $XP_TESTS -- --durations 3 --timeout=60

.github/workflows/free_threaded_wheels.yml

@@ -82,6 +82,12 @@ jobs:
         buildplat:
           - [ubuntu-22.04, manylinux, x86_64, "", ""]
           - [ubuntu-22.04, musllinux, x86_64, "", ""]
+          - [macos-12, macosx, x86_64, openblas, "10.9"]
+          - [macos-13, macosx, x86_64, accelerate, "14.0"]
+          # OpenBLAS builds with deployment target set to 12 are failing due
+          # to relocation issues of gfortran dylibs
+          # - [macos-14, macosx, arm64, openblas, "12.0"]
+          - [macos-14, macosx, arm64, accelerate, "14.0"]
           # TODO: build scipy and set up Windows and MacOS
           # cibuildwheel does not yet support Mac for free-threaded python
           # windows is supported but numpy doesn't build on the image yet
@@ -103,8 +109,56 @@ jobs:
         with:
           python-version: "3.x"
 
+      - name: Setup macOS
+        if: startsWith( matrix.buildplat[0], 'macos-' )
+        run: |
+          if [[ ${{ matrix.buildplat[3] }} == 'accelerate' ]]; then
+            echo CIBW_CONFIG_SETTINGS=\"setup-args=-Dblas=accelerate\" >> "$GITHUB_ENV"
+            # Always use preinstalled gfortran for Accelerate builds
+            ln -s $(which gfortran-13) gfortran
+            export PATH=$PWD:$PATH
+            echo "PATH=$PATH" >> "$GITHUB_ENV"
+            LIB_PATH=$(dirname $(gfortran --print-file-name libgfortran.dylib))
+          fi
+          if [[ ${{ matrix.buildplat[4] }} == '10.9' ]]; then
+            # Newest version of Xcode that supports macOS 10.9
+            XCODE_VER='13.4.1'
+          else
+            XCODE_VER='15.2'
+          fi
+          CIBW="sudo xcode-select -s /Applications/Xcode_${XCODE_VER}.app"
+          echo "CIBW_BEFORE_ALL=$CIBW" >> $GITHUB_ENV
+          # setting SDKROOT necessary when using the gfortran compiler
+          # installed in cibw_before_build_macos.sh
+          sudo xcode-select -s /Applications/Xcode_${XCODE_VER}.app
+          CIBW="MACOSX_DEPLOYMENT_TARGET=${{ matrix.buildplat[4] }}\
+            SDKROOT=$(xcrun --sdk macosx --show-sdk-path)\
+            PKG_CONFIG_PATH=${{ github.workspace }}"
+          echo "CIBW_ENVIRONMENT_MACOS=$CIBW" >> "$GITHUB_ENV"
+
+          echo "REPAIR_PATH=$LIB_PATH" >> "$GITHUB_ENV"
+
+          PREFIX=DYLD_LIBRARY_PATH="\$(dirname \$(gfortran --print-file-name libgfortran.dylib))"
+          # remove libgfortran from location used for linking (if any), to
+          # check wheel has bundled things correctly and all tests pass without
+          # needing installed gfortran
+          POSTFIX=" sudo rm -rf /opt/gfortran-darwin-x86_64-native &&\
+                    sudo rm -rf /usr/local/gfortran/lib"
+          CIBW="$PREFIX delocate-listdeps -d {wheel} && echo "-----------" &&\
+            $PREFIX delocate-wheel -v $EXCLUDE --require-archs \
+            {delocate_archs} -w {dest_dir} {wheel} && echo "-----------" &&\
+            delocate-listdeps -d {dest_dir}/*.whl && echo "-----------" &&\
+            $POSTFIX"
+
+          # Rename x86 Accelerate wheel to test on macOS 13 runner
+          if [[ ${{ matrix.buildplat[0] }} == 'macos-13' && ${{ matrix.buildplat[4] }} == '14.0' ]]; then
+            CIBW+=" && mv {dest_dir}/\$(basename {wheel}) \
+              {dest_dir}/\$(echo \$(basename {wheel}) | sed 's/14_0/13_0/')"
+          fi
+          echo "CIBW_REPAIR_WHEEL_COMMAND_MACOS=$CIBW" >> "$GITHUB_ENV"
+
       - name: Build wheels
-        uses: pypa/cibuildwheel@ba8be0d98853f5744f24e7f902c8adef7ae2e7f3  # v2.18.1
+        uses: pypa/cibuildwheel@v2.20.0
         env:
           CIBW_PRERELEASE_PYTHONS: True
           CIBW_FREE_THREADED_SUPPORT: True
@@ -114,25 +168,29 @@ jobs:
           # cibw_before_build.sh when a released cython can build numpy
           CIBW_BUILD_FRONTEND: "pip; args: --no-build-isolation"
 
+      - name: Rename after test (macOS x86 Accelerate only)
+        # Rename x86 Accelerate wheel back so it targets macOS >= 14
+        if: matrix.buildplat[0] == 'macos-13' && matrix.buildplat[4] == '14.0'
+        run: |
+          mv ./wheelhouse/*.whl $(find ./wheelhouse -type f -name '*.whl' | sed 's/13_0/14_0/')
+
       - uses: actions/upload-artifact@v4
         with:
           path: ./wheelhouse/*.whl
           name: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
             ${{ matrix.buildplat[2] }} ${{ matrix.buildplat[3] }}
             ${{ matrix.buildplat[4] }}
 
-      - uses: mamba-org/setup-micromamba@422500192359a097648154e8db4e39bdb6c6eed7
+      - uses: conda-incubator/setup-miniconda@v3
         with:
           # for installation of anaconda-client, required for upload to
           # anaconda.org
+          # default (and activated) environment name is test
           # Note that this step is *after* specific pythons have been used to
           # build and test the wheel
-          # for installation of anaconda-client, for upload to anaconda.org
-          # environment will be activated after creation, and in future bash steps
-          init-shell: bash
-          environment-name: upload-env
-          create-args: >-
-            anaconda-client
+          auto-update-conda: true
+          python-version: "3.10"
+          miniconda-version: "latest"
 
       - name: Upload wheels
         if: success()

.github/workflows/linux.yml

@@ -26,7 +26,7 @@ jobs:
     uses: ./.github/workflows/commit_message.yml
 
   test_meson:
-    name: mypy (py3.10) & dev deps (py3.12), fast, dev.py
+    name: mypy (py3.10) & dev deps (py3.13), fast, dev.py
     needs: get_commit_message
     # If using act to run CI locally the github object does not exist and
     # the usual skipping should not be enforced
@@ -36,7 +36,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python-version: ['3.10', '3.12'] # this run will use python dev versions when available
+        python-version: ['3.10', '3.13-dev'] # this run will use python dev versions when available
         maintenance-branch:
           - ${{ contains(github.ref, 'maintenance/') || contains(github.base_ref, 'maintenance/') }}
         exclude:
@@ -54,6 +54,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
         cache: 'pip'
         cache-dependency-path: 'environment.yml'
+        allow-prereleases: true
 
     - name: Install Ubuntu dependencies
       run: |
@@ -67,7 +68,7 @@ jobs:
         python -m pip install numpy cython pytest pytest-xdist pytest-timeout pybind11 mpmath gmpy2 pythran ninja meson click rich-click doit pydevtool pooch hypothesis
 
     - name: Install Python packages from repositories
-      if: matrix.python-version == '3.12' # this run will use python dev versions when available
+      if: matrix.python-version == '3.13-dev' # this run will use python dev versions when available
       run: |
         python -m pip install git+https://github.com/numpy/numpy.git
         python -m pip install ninja cython pytest pybind11 pytest-xdist pytest-timeout click rich-click doit pydevtool pooch hypothesis "setuptools<67.3"
@@ -123,6 +124,9 @@ jobs:
         python dev.py build --tags=runtime,python-runtime,devel,tests
         python tools/check_installation.py ${{ env.INSTALLDIR }}
 
+    - name: Check build-internal dependencies
+      run: ninja -C build -t missingdeps
+
     - name: Mypy
       if: matrix.python-version == '3.10'
       run: |
@@ -317,10 +321,7 @@ jobs:
     - name: Install Python packages
       run: |
         python -m pip install cython pythran ninja meson-python pybind11 click rich_click pydevtool
-        # Note: matplotlib not installed here to avoid issues around numpy 2.0
-        # (it can be put back after matplotlib has made a 2.0-compatible
-        # release on PyPI.
-        python -m pip install --pre --upgrade pytest pytest-cov pytest-xdist mpmath gmpy2 threadpoolctl pooch hypothesis
+        python -m pip install --pre --upgrade pytest pytest-cov pytest-xdist mpmath gmpy2 threadpoolctl pooch hypothesis matplotlib
         python -m pip install -r requirements/openblas.txt
         # Install numpy last, to ensure we get nightly (avoid possible <2.0 constraints).
         python -m pip install --pre --upgrade --timeout=60 -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy
@@ -471,13 +472,9 @@ jobs:
     - name: Install pre-release pip
       run: |
         pip install -U --pre pip
-    # TODO: remove cython nightly install when cython does a release
-    - name: Install nightly Cython
-      run: |
-        pip install git+https://github.com/cython/cython
-    - name: Install nightly NumPy
+    - name: Install nightly NumPy and Cython
       run: |
-        pip install -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy
+        pip install -i https://pypi.anaconda.org/scientific-python-nightly-wheels/simple cython numpy
     - name: Install Python dependencies
       run: |
         pip install git+https://github.com/serge-sans-paille/pythran

.github/workflows/linux_intel_oneAPI.yml

@@ -0,0 +1,98 @@
+name: Intel oneAPI tests
+
+on:
+  push:
+    branches:
+      - maintenance/**
+  pull_request:
+    branches:
+      - main
+      - maintenance/**
+
+permissions:
+   contents: read  # to fetch code (actions/checkout)
+
+# The environment variable LINUX_BASEKIT_URL and LINUX_HPCKIT_URL
+# store the URL for downloading Intel oneAPI.
+# Reference - https://github.com/oneapi-src/oneapi-ci/blob/b4a96bd1888c130fcb73524d2b77b90f43681cbc/.github/workflows/build_all.yml#L11-L12
+env:
+  LINUX_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/9a98af19-1c68-46ce-9fdd-e249240c7c42/l_BaseKit_p_2024.2.0.634_offline.sh
+  LINUX_HPCKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/d4e49548-1492-45c9-b678-8268cb0f1b05/l_HPCKit_p_2024.2.0.635_offline.sh
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  get_commit_message:
+    name: Get commit message
+    uses: ./.github/workflows/commit_message.yml
+
+  icx_icpx_ifx_mkl:
+    name: py3.12, dev.py
+    needs: get_commit_message
+    # Ensure (a) this doesn't run on forks by default, and
+    #        (b) it does run with Act locally (`github` doesn't exist there)
+    if: >
+      needs.get_commit_message.outputs.message == 1
+      && (github.repository == 'scipy/scipy' || github.repository == '')
+    runs-on: ubuntu-24.04
+    strategy:
+      matrix:
+        python-version: ["3.12"]
+
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+        with:
+          submodules: recursive
+
+      - name: Setup Conda
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+          channels: conda-forge
+          channel-priority: true
+          activate-environment: scipy-dev
+          use-only-tar-bz2: false
+          miniforge-variant: Mambaforge
+          miniforge-version: latest
+          use-mamba: true
+
+      - name: Update Conda Environment
+        run: |
+          mamba env update -n scipy-dev -f environment.yml
+
+      - name: cache install
+        id: cache-install
+        uses: actions/cache@v3
+        with:
+          path: |
+              /opt/intel/oneapi/
+          key: install-${{ env.LINUX_BASEKIT_URL }}-${{ env.LINUX_HPCKIT_URL }}-compiler
+      - name: Install oneAPI Base kit
+        if: steps.cache-install.outputs.cache-hit != 'true'
+        run: |
+          printenv LINUX_BASEKIT_URL
+          bash tools/install_intel_oneAPI_linux.sh $LINUX_BASEKIT_URL
+      - name: Install oneAPI HPC kit
+        if: steps.cache-install.outputs.cache-hit != 'true'
+        run: |
+          printenv LINUX_HPCKIT_URL
+          bash tools/install_intel_oneAPI_linux.sh $LINUX_HPCKIT_URL
+
+      - name: Initialise Intel oneAPI and Build SciPy
+        shell: bash -l {0}
+        run: |
+          . /opt/intel/oneapi/setvars.sh --force
+          conda activate scipy-dev
+          conda remove -y compilers
+          unset FFLAGS
+          CC=icx CXX=icpx FC=ifx python dev.py build -C-Dblas=mkl-dynamic-lp64-iomp -C-Dlapack=mkl-dynamic-lp64-iomp
+
+      - name: Test scipy
+        shell: bash -l {0}
+        run: |
+          . /opt/intel/oneapi/setvars.sh --force
+          conda activate scipy-dev
+          python dev.py test