Merge branch 'main' into fix-leaks-2

Author: seberg

.github/workflows/codeql.yml

@@ -45,7 +45,7 @@ jobs:
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@9fdb3e49720b44c48891d036bb502feb25684276 # v3.25.6
+        uses: github/codeql-action/init@f079b8493333aace61c81488f8bd40919487bd9f # v3.25.7
         with:
           languages: ${{ matrix.language }}
           # If you wish to specify custom queries, you can do so here or in a config file.
@@ -55,7 +55,7 @@ jobs:
       # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
       # If this step fails, then you should remove it and run the build manually (see below)
       - name: Autobuild
-        uses: github/codeql-action/autobuild@9fdb3e49720b44c48891d036bb502feb25684276 # v3.25.6
+        uses: github/codeql-action/autobuild@f079b8493333aace61c81488f8bd40919487bd9f # v3.25.7
 
       # ℹ️ Command-line programs to run using the OS shell.
       # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -68,6 +68,6 @@ jobs:
       #   ./location_of_script_within_repo/buildscript.sh
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@9fdb3e49720b44c48891d036bb502feb25684276 # v3.25.6
+        uses: github/codeql-action/analyze@f079b8493333aace61c81488f8bd40919487bd9f # v3.25.7
         with:
           category: "/language:${{matrix.language}}"

.github/workflows/scorecards.yml

@@ -50,6 +50,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@9fdb3e49720b44c48891d036bb502feb25684276 # v2.1.27
+        uses: github/codeql-action/upload-sarif@f079b8493333aace61c81488f8bd40919487bd9f # v2.1.27
         with:
           sarif_file: results.sarif

benchmarks/benchmarks/bench_ma.py

@@ -265,3 +265,47 @@ def time_where(self, mtype, msize):
             fun(self.nmxs > 2, self.nmxs, self.nmys)
         elif msize == 'big':
             fun(self.nmxl > 2, self.nmxl, self.nmyl)
+
+
+class Cov(Benchmark):
+    param_names = ["size"]
+    params = [["small", "large"]]
+
+    def setup(self, size):
+        # Set the proportion of masked values.
+        prop_mask = 0.2
+        # Set up a "small" array with 10 vars and 10 obs.
+        rng = np.random.default_rng()
+        data = rng.random((10, 10), dtype=np.float32)
+        self.small = np.ma.array(data, mask=(data <= prop_mask))
+        # Set up a "large" array with 100 vars and 100 obs.
+        data = rng.random((100, 100), dtype=np.float32)
+        self.large = np.ma.array(data, mask=(data <= prop_mask))
+
+    def time_cov(self, size):
+        if size == "small":
+            np.ma.cov(self.small)
+        if size == "large":
+            np.ma.cov(self.large)
+
+
+class Corrcoef(Benchmark):
+    param_names = ["size"]
+    params = [["small", "large"]]
+
+    def setup(self, size):
+        # Set the proportion of masked values.
+        prop_mask = 0.2
+        # Set up a "small" array with 10 vars and 10 obs.
+        rng = np.random.default_rng()
+        data = rng.random((10, 10), dtype=np.float32)
+        self.small = np.ma.array(data, mask=(data <= prop_mask))
+        # Set up a "large" array with 100 vars and 100 obs.
+        data = rng.random((100, 100), dtype=np.float32)
+        self.large = np.ma.array(data, mask=(data <= prop_mask))
+
+    def time_corrcoef(self, size):
+        if size == "small":
+            np.ma.corrcoef(self.small)
+        if size == "large":
+            np.ma.corrcoef(self.large)

doc/release/upcoming_changes/26285.change.rst

@@ -0,0 +1,13 @@
+``ma.corrcoef`` may return a slightly different result
+------------------------------------------------------
+A pairwise observation approach is currently used in `ma.corrcoef` to 
+calculate the standard deviations for each pair of variables. This has been 
+changed as it is being used to normalise the covariance, estimated using 
+`ma.cov`, which does not consider the observations for each variable in a 
+pairwise manner, rendering it unnecessary. The normalisation has been 
+replaced by the more appropriate standard deviation for each variable, 
+which significantly reduces the wall time, but will return slightly different 
+estimates of the correlation coefficients in cases where the observations 
+between a pair of variables are not aligned. However, it will return the same 
+estimates in all other cases, including returning the same correlation matrix 
+as `corrcoef` when using a masked array with no masked values.

doc/release/upcoming_changes/26285.performance.rst

@@ -0,0 +1,5 @@
+``ma.cov`` and ``ma.corrcoef`` are now significantly faster
+-----------------------------------------------------------
+The private function has been refactored along with `ma.cov` and 
+`ma.corrcoef`. They are now significantly faster, particularly on large, 
+masked arrays.

doc/source/reference/routines.ma.rst

@@ -416,10 +416,25 @@ Miscellanea
 
    ma.allequal
    ma.allclose
+   ma.amax
+   ma.amin
    ma.apply_along_axis
    ma.apply_over_axes
    ma.arange
    ma.choose
+   ma.compress_nd
+   ma.convolve
+   ma.correlate
    ma.ediff1d
+   ma.flatten_mask
+   ma.flatten_structured_array
+   ma.fromflex
    ma.indices
+   ma.left_shift
+   ma.ndim
+   ma.put
+   ma.putmask
+   ma.right_shift
+   ma.round_
+   ma.take
    ma.where

doc/source/reference/routines.strings.rst

@@ -31,6 +31,11 @@ String operations
    :toctree: generated/
 
    add
+   center
+   capitalize
+   decode
+   encode
+   expandtabs
    ljust
    lower
    lstrip

doc/source/user/c-info.ufunc-tutorial.rst

@@ -182,21 +182,16 @@ site-packages directory.
             $python setup.py install
             will install the module in your site-packages file.
 
-            See the distutils section of
-            'Extending and Embedding the Python Interpreter'
-            at docs.python.org for more information.
+            See the setuptools section 'Building Extension Modules'
+            at setuptools.pypa.io for more information.
         '''
 
+        from setuptools import setup, Extension
+        import numpy as np
 
-        from distutils.core import setup, Extension
+        module1 = Extension('spam', sources=['spammodule.c'])
 
-        module1 = Extension('spam', sources=['spammodule.c'],
-                                include_dirs=['/usr/local/lib'])
-
-        setup(name = 'spam',
-                version='1.0',
-                description='This is my spam package',
-                ext_modules = [module1])
+        setup(name='spam', version='1.0', ext_modules=[module1])
 
 
 Once the spam module is imported into python, you can call logit
@@ -355,8 +350,8 @@ using ``python setup.py build_ext --inplace``.
         '''
             setup.py file for single_type_logit.c
             Note that since this is a numpy extension
-            we use numpy.distutils instead of
-            distutils from the python standard library.
+            we add an include_dirs=[get_include()] so that the
+            extension is built with numpy's C/C++ header files.
 
             Calling
             $python setup.py build_ext --inplace
@@ -373,33 +368,26 @@ using ``python setup.py build_ext --inplace``.
             $python setup.py install
             will install the module in your site-packages file.
 
-            See the distutils section of
-            'Extending and Embedding the Python Interpreter'
-            at docs.python.org  and the documentation
-            on numpy.distutils for more information.
+            See the setuptools section 'Building Extension Modules'
+            at setuptools.pypa.io for more information.
         '''
 
+        from setuptools import setup, Extension
+        from numpy import get_include
 
-        def configuration(parent_package='', top_path=None):
-            from numpy.distutils.misc_util import Configuration
-
-            config = Configuration('npufunc_directory',
-                                   parent_package,
-                                   top_path)
-            config.add_extension('npufunc', ['single_type_logit.c'])
+        npufunc = Extension('npufunc',
+                            sources=['single_type_logit.c'],
+                            include_dirs=[get_include()])
 
-            return config
+        setup(name='npufunc', version='1.0', ext_modules=[npufunc])
 
-        if __name__ == "__main__":
-            from numpy.distutils.core import setup
-            setup(configuration=configuration)
 
 After the above has been installed, it can be imported and used as follows.
 
 >>> import numpy as np
 >>> import npufunc
 >>> npufunc.logit(0.5)
-0.0
+np.float64(0.0)
 >>> a = np.linspace(0,1,5)
 >>> npufunc.logit(a)
 array([       -inf, -1.09861229,  0.        ,  1.09861229,         inf])
@@ -607,8 +595,10 @@ or installed to site-packages via ``python setup.py install``.
         '''
             setup.py file for multi_type_logit.c
             Note that since this is a numpy extension
-            we use numpy.distutils instead of
-            distutils from the python standard library.
+            we add an include_dirs=[get_include()] so that the
+            extension is built with numpy's C/C++ header files.
+            Furthermore, we also have to include the npymath
+            lib for half-float d-type.
 
             Calling
             $python setup.py build_ext --inplace
@@ -625,38 +615,31 @@ or installed to site-packages via ``python setup.py install``.
             $python setup.py install
             will install the module in your site-packages file.
 
-            See the distutils section of
-            'Extending and Embedding the Python Interpreter'
-            at docs.python.org  and the documentation
-            on numpy.distutils for more information.
+            See the setuptools section 'Building Extension Modules'
+            at setuptools.pypa.io for more information.
         '''
 
+        from setuptools import setup, Extension
+        from numpy import get_include
+        from os import path
 
-        def configuration(parent_package='', top_path=None):
-            from numpy.distutils.misc_util import Configuration, get_info
-
-            #Necessary for the half-float d-type.
-            info = get_info('npymath')
+        path_to_npymath = path.join(get_include(), '..', 'lib')
+        npufunc = Extension('npufunc',
+                            sources=['multi_type_logit.c'],
+                            include_dirs=[get_include()],
+                            # Necessary for the half-float d-type.
+                            library_dirs=[path_to_npymath],
+                            libraries=["npymath"])
 
-            config = Configuration('npufunc_directory',
-                                    parent_package,
-                                    top_path)
-            config.add_extension('npufunc',
-                                    ['multi_type_logit.c'],
-                                    extra_info=info)
+        setup(name='npufunc', version='1.0', ext_modules=[npufunc])
 
-            return config
-
-        if __name__ == "__main__":
-            from numpy.distutils.core import setup
-            setup(configuration=configuration)
 
 After the above has been installed, it can be imported and used as follows.
 
 >>> import numpy as np
 >>> import npufunc
 >>> npufunc.logit(0.5)
-0.0
+np.float64(0.0)
 >>> a = np.linspace(0,1,5)
 >>> npufunc.logit(a)
 array([       -inf, -1.09861229,  0.        ,  1.09861229,         inf])
@@ -678,13 +661,17 @@ the line
 
     .. code-block:: python
 
-        config.add_extension('npufunc', ['single_type_logit.c'])
+        npufunc = Extension('npufunc',
+                            sources=['single_type_logit.c'],
+                            include_dirs=[get_include()])
 
 is replaced with
 
     .. code-block:: python
 
-        config.add_extension('npufunc', ['multi_arg_logit.c'])
+        npufunc = Extension('npufunc',
+                            sources=['multi_arg_logit.c'],
+                            include_dirs=[get_include()])
 
 The C file is given below. The ufunc generated takes two arguments ``A``
 and ``B``. It returns a tuple whose first element is ``A * B`` and whose second
@@ -809,13 +796,17 @@ the line
 
     .. code-block:: python
 
-        config.add_extension('npufunc', ['single_type_logit.c'])
+        npufunc = Extension('npufunc',
+                            sources=['single_type_logit.c'],
+                            include_dirs=[get_include()])
 
 is replaced with
 
     .. code-block:: python
 
-        config.add_extension('npufunc', ['add_triplet.c'])
+        npufunc = Extension('npufunc',
+                            sources=['add_triplet.c'],
+                            include_dirs=[get_include()])
 
 The C file is given below.
 
@@ -892,7 +883,7 @@ The C file is given below.
             NULL
         };
 
-        PyMODINIT_FUNC PyInit_struct_ufunc_test(void)
+        PyMODINIT_FUNC PyInit_npufunc(void)
         {
             PyObject *m, *add_triplet, *d;
             PyObject *dtype_dict;

environment.yml

@@ -35,8 +35,9 @@ dependencies:
   - scipy
   - pandas
   - matplotlib
-  - pydata-sphinx-theme=0.15.2
+  - pydata-sphinx-theme>=0.15.2
   - doxygen
+  - towncrier
   # NOTE: breathe 4.33.0 collides with sphinx.ext.graphviz
   - breathe>4.33.0
   # For linting

numpy/_core/src/multiarray/arraytypes.c.src

@@ -4222,6 +4222,7 @@ NPY_NO_EXPORT PyArray_Descr *
 PyArray_DescrFromType(int type)
 {
     PyArray_Descr *ret = NULL;
+    npy_bool is_stringdtype = (type == NPY_VSTRING || type == NPY_VSTRINGLTR);
 
     if (type < 0) {
         /*
@@ -4233,7 +4234,7 @@ PyArray_DescrFromType(int type)
          */
         ret = NULL;
     }
-    else if (type == NPY_VSTRING || type == NPY_VSTRINGLTR) {
+    else if  (is_stringdtype) {
         ret = (PyArray_Descr *)new_stringdtype_instance(NULL, 1);
     }
     // builtin legacy dtypes
@@ -4280,7 +4281,7 @@ PyArray_DescrFromType(int type)
         PyErr_SetString(PyExc_ValueError,
                 "Invalid data-type for array");
     }
-    else {
+    else if (!is_stringdtype) {
         Py_INCREF(ret);
     }