Merge pull request numpy#24279 from rgommers/ci-win-py32bit

CI/BLD: fail by default if no BLAS/LAPACK, add 32-bit Python on Windows CI job

Author: charris

.circleci/config.yml

@@ -63,7 +63,7 @@ jobs:
             . venv/bin/activate
             pip install --progress-bar=off -r test_requirements.txt
             pip install --progress-bar=off -r doc_requirements.txt
-            pip install .
+            pip install . --config-settings=setup-args="-Dallow-noblas=true"
 
       - run:
           name: create release notes

.github/workflows/wheels.yml

@@ -186,12 +186,13 @@ jobs:
           python-version: "3.9"
       - name: Build sdist
         run: |
-          python setup.py sdist
+          python -m pip install -U pip build
+          python -m build --sdist -Csetup-args=-Dallow-noblas=true
       - name: Test the sdist
         run: |
           # TODO: Don't run test suite, and instead build wheels from sdist
           # Depends on pypa/cibuildwheel#1020
-          python -m pip install dist/*.gz
+          python -m pip install dist/*.gz -Csetup-args=-Dallow-noblas=true
           pip install ninja
           pip install -r test_requirements.txt
           cd .. # Can't import numpy within numpy src directory

.github/workflows/windows_meson.yml

@@ -17,10 +17,10 @@ permissions:
   contents: read # to fetch code (actions/checkout)
 
 jobs:
-  meson:
-    name: Meson windows build/test
+  msvc_64bit_python_openblas:
+    name: MSVC, x86-64, LP64 OpenBLAS
     runs-on: windows-2019
-    # if: "github.repository == 'numpy/numpy'"
+    if: "github.repository == 'numpy/numpy'"
     steps:
     - name: Checkout
       uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
@@ -86,3 +86,38 @@ jobs:
         echo "LASTEXITCODE is '$LASTEXITCODE'"
         python -c "import numpy, sys; sys.exit(numpy.test(verbose=3) is False)"
         echo "LASTEXITCODE is '$LASTEXITCODE'"
+
+  msvc_32bit_python_openblas:
+    name: MSVC, 32-bit Python, no BLAS
+    runs-on: windows-2019
+    if: "github.repository == 'numpy/numpy'"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+        with:
+          submodules: recursive
+          fetch-depth: 0
+
+      - name: Setup Python (32-bit)
+        uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
+        with:
+          python-version: '3.10'
+          architecture: 'x86'
+
+      - name: Setup MSVC (32-bit)
+        uses: bus1/cabuild/action/msdevshell@e22aba57d6e74891d059d66501b6b5aed8123c4d  # v1
+        with:
+          architecture: 'x86'
+
+      - name: Build and install
+        run: |
+          python -m pip install . -v -Ccompile-args="-j2" -Csetup-args="-Dallow-noblas=true"
+
+      - name: Install test dependencies
+        run: |
+          python -m pip install -r test_requirements.txt
+
+      - name: Run test suite (fast)
+        run: |
+          cd tools
+          python -m pytest --pyargs numpy -m "not slow" -n2

azure-pipelines.yml

@@ -227,6 +227,8 @@ stages:
             TEST_MODE: fast
             BITS: 64
             NPY_USE_BLAS_ILP64: '1'
+            # Broken - it builds but _multiarray_umath doesn't import - needs investigating
+            DISABLE_BLAS: '1'
 
     steps:
     - template: azure-steps-windows.yml

azure-steps-windows.yml

@@ -28,26 +28,33 @@ steps:
     mkdir  C:/opt/openblas/openblas_dll
     mkdir  C:/opt/32/lib/pkgconfig
     mkdir  C:/opt/64/lib/pkgconfig
-    # TBD: support 32 bit testing
     $target=$(python -c "import tools.openblas_support as obs; plat=obs.get_plat(); ilp64=obs.get_ilp64(); target=f'openblas_{plat}.zip'; obs.download_openblas(target, plat, ilp64);print(target)")
     unzip -o -d c:/opt/ $target
     echo "##vso[task.setvariable variable=PKG_CONFIG_PATH]c:/opt/64/lib/pkgconfig"
     copy C:/opt/64/bin/*.dll C:/opt/openblas/openblas_dll
   displayName: 'Download / Install OpenBLAS'
 
 - powershell: |
+    # Note: ensure the `pip install .` command remains the last one here, to
+    # avoid "green on failure" issues
     python -c "from tools import openblas_support; openblas_support.make_init('numpy')"
-    If ( Test-Path env:NPY_USE_BLAS_ILP64 ) {
-        python -m pip install . -Csetup-args="--vsenv" -Csetup-args="-Duse-ilp64=true" -Csetup-args="-Dblas-symbol-suffix=64_"
+    If ( Test-Path env:DISABLE_BLAS ) {
+        python -m pip install . -v -Csetup-args="--vsenv" -Csetup-args="-Dblas=none" -Csetup-args="-Dlapack=none" -Csetup-args="-Dallow-noblas=true"
+    }
+    elseif ( Test-Path env:NPY_USE_BLAS_ILP64 ) {
+        python -m pip install . -v -Csetup-args="--vsenv" -Csetup-args="-Duse-ilp64=true" -Csetup-args="-Dblas-symbol-suffix=64_"
     } else {
-        python -m pip install . -Csetup-args="--vsenv"
+        python -m pip install . -v -Csetup-args="--vsenv"
     }
+  displayName: 'Build NumPy'
+
+- powershell: |
     # copy from c:/opt/openblas/openblas_dll to numpy/.libs to ensure it can
     # get loaded when numpy is imported (no RPATH on Windows)
     $target = $(python -c "import sysconfig; print(sysconfig.get_path('platlib'))")
     mkdir $target/numpy/.libs
     copy C:/opt/openblas/openblas_dll/*.dll $target/numpy/.libs
-  displayName: 'Build NumPy'
+  displayName: 'Copy OpenBLAS DLL to site-packages'
 
 - script: |
     python -m pip install threadpoolctl

meson_options.txt

@@ -2,6 +2,8 @@ option('blas', type: 'string', value: 'openblas',
         description: 'option for BLAS library switching')
 option('lapack', type: 'string', value: 'openblas',
         description: 'option for LAPACK library switching')
+option('allow-noblas', type: 'boolean', value: false,
+        description: 'If set to true, allow building with (slow!) internal fallback routines')
 option('use-ilp64', type: 'boolean', value: false,
        description: 'Use ILP64 (64-bit integer) BLAS and LAPACK interfaces')
 option('blas-symbol-suffix', type: 'string', value: '',

numpy/core/tests/test_umath.py

@@ -1764,6 +1764,8 @@ def test_expm1(self):
         np.log, np.log2, np.log10, np.reciprocal, np.arccosh
     ]
 
+    @pytest.mark.skipif(sys.platform == "win32" and sys.maxsize < 2**31 + 1,
+                        reason='failures on 32-bit Python, see FIXME below')
     @pytest.mark.parametrize("ufunc", UFUNCS_UNARY_FP)
     @pytest.mark.parametrize("dtype", ('e', 'f', 'd'))
     @pytest.mark.parametrize("data, escape", (
@@ -1810,6 +1812,8 @@ def test_unary_spurious_fpexception(self, ufunc, dtype, data, escape):
         # FIXME: NAN raises FP invalid exception:
         #  - ceil/float16 on MSVC:32-bit
         #  - spacing/float16 on almost all platforms
+        # FIXME: skipped on MSVC:32-bit during switch to Meson, 10 cases fail
+        #        when SIMD support not present / disabled
         if ufunc in (np.spacing, np.ceil) and dtype == 'e':
             return
         array = np.array(data, dtype=dtype)

numpy/linalg/meson.build

@@ -14,7 +14,6 @@ lapack_lite_sources = [
 
 lapack_lite_module_src = ['lapack_litemodule.c']
 if not have_lapack
-  warning('LAPACK was not found, NumPy is using an unoptimized, naive build from sources!')
   lapack_lite_module_src += lapack_lite_sources
 endif
 

numpy/meson.build

@@ -53,6 +53,19 @@ endif
 # (see cibuildwheel settings in pyproject.toml), but used by CI jobs already
 blas_symbol_suffix = get_option('blas-symbol-suffix')
 
+use_ilp64 = get_option('use-ilp64')
+if not use_ilp64
+  # For now, keep supporting this environment variable too (same as in setup.py)
+  # `false is the default for the CLI flag, so check if env var was set
+  use_ilp64 = run_command(py,
+    [
+      '-c',
+      'import os; print(1) if os.environ.get("NPY_USE_BLAS_ILP64", "0") != "0" else print(0)'
+    ],
+    check: true
+  ).stdout().strip() == '1'
+endif
+
 
 # TODO: 64-bit (ILP64) BLAS and LAPACK support (e.g., check for more .pc files
 # so we detect `openblas64_.so` directly). Partially supported now, needs more
@@ -70,7 +83,12 @@ lapack_name = get_option('lapack')
 # pkg-config uses a lower-case name while CMake uses a capitalized name, so try
 # that too to make the fallback detection with CMake work
 if blas_name == 'openblas'
-  blas = dependency(['openblas', 'OpenBLAS'], required: false)
+  if use_ilp64
+    _openblas_names = ['openblas64', 'openblas', 'OpenBLAS']
+  else
+    _openblas_names = ['openblas', 'OpenBLAS']
+  endif
+  blas = dependency(_openblas_names, required: false)
 else
   blas = dependency(blas_name, required: false)
 endif
@@ -121,27 +139,22 @@ if have_blas
   endif
 endif
 
-use_ilp64 = get_option('use-ilp64')
-if not use_ilp64
-  # For now, keep supporting this environment variable too (same as in setup.py)
-  # `false is the default for the CLI flag, so check if env var was set
-  use_ilp64 = run_command(py,
-    [
-      '-c',
-      'import os; print(1) if os.environ.get("NPY_USE_BLAS_ILP64", "0") != "0" else print(0)'
-    ],
-    check: true
-  ).stdout().strip() == '1'
-endif
-
-# BLAS and LAPACK are optional dependencies for NumPy. We can only use a BLAS
-# which provides a CBLAS interface. So disable BLAS completely if CBLAS is not
-# found (lapack-lite will be used instead; xref gh-24200 for a discussion on
-# whether this silent disabling should stay as-is)
+# BLAS and LAPACK are dependencies for NumPy. Since NumPy 2.0, by default the
+# build will fail if they are missing; the performance impact is large, so
+# using fallback routines must be explicitly opted into by the user. xref
+# gh-24200 for a discussion on this.
+#
+# Note that we can only use a BLAS which provides a CBLAS interface. So disable
+# BLAS completely if CBLAS is not found.
+allow_noblas = get_option('allow-noblas')
 if have_blas
   _args_blas = []  # note: used for C and C++ via `blas_dep` below
   if have_cblas
     _args_blas += ['-DHAVE_CBLAS']
+  elif not allow_noblas
+    error('No CBLAS interface detected! Install a BLAS library with CBLAS ' + \
+          'support, or use the `allow-noblas` build option (note, this ' + \
+          'may be up to 100x slower for some linear algebra operations).')
   endif
   if use_ilp64
     _args_blas += ['-DHAVE_BLAS_ILP64']
@@ -154,15 +167,25 @@ if have_blas
     compile_args: _args_blas,
   )
 else
-  blas_dep = []
+  if allow_noblas
+    blas_dep = []
+  else
+    error('No BLAS library detected! Install one, or use the ' + \
+          '`allow-noblas` build option (note, this may be up to 100x slower ' + \
+          'for some linear algebra operations).')
+  endif
 endif
 
 if lapack_name == 'openblas'
   lapack_name = ['openblas', 'OpenBLAS']
 endif
 lapack_dep = dependency(lapack_name, required: false)
 have_lapack = lapack_dep.found()
-
+if not have_lapack and not allow_noblas
+  error('No LAPACK library detected! Install one, or use the ' + \
+        '`allow-noblas` build option (note, this may be up to 100x slower ' + \
+        'for some linear algebra operations).')
+endif
 
 # Copy the main __init__.py|pxd files to the build dir (needed for Cython)
 __init__py = fs.copyfile('__init__.py')

tools/ci/cirrus_macosx_arm64.yml

@@ -53,5 +53,5 @@ macos_arm64_test_task:
     pip install -r build_requirements.txt
     pip install pytest pytest-xdist hypothesis typing_extensions
     
-    spin build
+    spin build -- -Dallow-noblas=true
     spin test -j auto