Merge pull request numpy#26680 from ev-br/smoke-docs-followup

remove doctesting from refguide-check, add `spin check-tutorials`

Author: mattip

.circleci/config.yml

@@ -95,14 +95,10 @@ jobs:
      #      destination: neps
 
       - run:
-          name: run doctests on documentation
+          name: run refguide-check
           command: |
             . venv/bin/activate
-            # Note: keep these two checks separate, because they seem to
-            # influence each other through changing global state (e.g., via
-            # `np.polynomial.set_default_printstyle`)
-            python tools/refguide_check.py --rst
-            python tools/refguide_check.py --doctests
+            python tools/refguide_check.py -v
 
       - persist_to_workspace:
           root: ~/repo

.github/workflows/linux.yml

@@ -181,8 +181,9 @@ jobs:
     - name: Check docstests
       shell: 'script -q -e -c "bash --noprofile --norc -eo pipefail {0}"'
       run: |
-        pip install scipy-doctest hypothesis matplotlib scipy pytz
+        pip install scipy-doctest hypothesis matplotlib scipy pytz pandas
         spin check-docs -v
+        spin check-tutorials -v
 
   sdist:
     needs: [smoke_test]

.spin/cmds.py

@@ -352,6 +352,75 @@ def check_docs(ctx, pytest_args, n_jobs, verbose, *args, **kwargs):
     ctx.forward(meson.test)
 
 
+@click.command()
+@click.argument("pytest_args", nargs=-1)
+@click.option(
+    "-j",
+    "n_jobs",
+    metavar='N_JOBS',
+    default="1",
+    help=("Number of parallel jobs for testing. "
+          "Can be set to `auto` to use all cores.")
+)
+@click.option(
+    '--verbose', '-v', is_flag=True, default=False
+)
+@click.pass_context
+def check_tutorials(ctx, pytest_args, n_jobs, verbose, *args, **kwargs):
+    """🔧 Run doctests of user-facing rst tutorials.
+
+    To test all tutorials in the numpy/doc/source/user/ directory, use
+
+      spin check-tutorials
+
+    To run tests on a specific RST file:
+
+     \b
+     spin check-tutorials numpy/doc/source/user/absolute-beginners.rst
+
+    \b
+    Note:
+    -----
+
+    \b
+     - This command only runs doctests and skips everything under tests/
+     - This command only doctests public objects: those which are accessible
+       from the top-level `__init__.py` file.
+
+    """  # noqa: E501
+    # handle all of
+    #   - `spin check-tutorials` (pytest_args == ())
+    #   - `spin check-tutorials path/to/rst`, and
+    #   - `spin check-tutorials path/to/rst -- --durations=3`
+    if (not pytest_args) or all(arg.startswith('-') for arg in pytest_args):
+        pytest_args = ('numpy/doc/source/user',) + pytest_args
+
+    # make all paths relative to the numpy source folder
+    pytest_args = tuple(
+        str(curdir / '..' / '..' / arg) if not arg.startswith('-') else arg
+        for arg in pytest_args
+   )
+
+    if (n_jobs != "1") and ('-n' not in pytest_args):
+        pytest_args = ('-n', str(n_jobs)) + pytest_args
+
+    if verbose:
+        pytest_args = ('-v',) + pytest_args
+
+    # turn doctesting on:
+    doctest_args = (
+        '--doctest-glob=*rst',
+    )
+
+    pytest_args = pytest_args + doctest_args
+
+    ctx.params['pytest_args'] = pytest_args
+
+    for extra_param in ('n_jobs', 'verbose'):
+        del ctx.params[extra_param]
+
+    ctx.forward(meson.test)
+
 
 # From scipy: benchmarks/benchmarks/common.py
 def _set_mem_rlimit(max_mem=None):

doc/TESTS.rst

@@ -74,6 +74,24 @@ Testing a subset of NumPy::
 
 For detailed info on testing, see :ref:`testing-builds`
 
+
+Running doctests
+----------------
+
+NumPy documentation contains code examples, "doctests". To check that the examples
+are correct, install the ``scipy-doctest`` package::
+
+  $ pip install scipy-doctest
+
+and run one of::
+
+  $ spin check-docs -v
+  $ spin check-docs numpy/linalg
+  $ spin check-docs -- -k 'det and not slogdet'
+
+Note that the doctests are not run when you use ``spin test``.
+
+
 Other methods of running tests
 ------------------------------
 

doc/source/dev/development_environment.rst

@@ -114,6 +114,15 @@ argument to pytest::
 
     $ spin test -v -t numpy/_core/tests/test_multiarray.py -- -k "MatMul and not vector"
 
+To run "doctests" -- to check that the code examples in the documentation is correct --
+use the `check-docs` spin command. It relies on the `scipy-docs` package, which
+provides several additional features on top of the standard library ``doctest``
+package. Install ``scipy-doctest`` and run one of::
+
+  $ spin check-docs -v
+  $ spin check-docs numpy/linalg
+  $ spin check-docs -v -- -k 'det and not slogdet'
+
 .. note::
 
     Remember that all tests of NumPy should pass before committing your changes.

doc/source/user/how-to-partition.rst

@@ -244,10 +244,10 @@ fully-dimensional result array.
 ::
 
    >>> np.ogrid[0:4, 0:6]
-   [array([[0],
+   (array([[0],
            [1],
            [2],
-           [3]]), array([[0, 1, 2, 3, 4, 5]])]
+           [3]]), array([[0, 1, 2, 3, 4, 5]]))
 
 All three methods described here can be used to evaluate function values on a
 grid.

numpy/conftest.py

@@ -210,7 +210,8 @@ def warnings_errors_and_rng(test=None):
         'c-info.ufunc-tutorial.rst': '',
         'basics.interoperability.rst': 'needs pandas',
         'basics.dispatch.rst': 'errors out in /testing/overrides.py',
-        'basics.subclassing.rst': '.. testcode:: admonitions not understood'
+        'basics.subclassing.rst': '.. testcode:: admonitions not understood',
+        'misc.rst': 'manipulates warnings',
     }
 
     # ignores are for things fail doctest collection (optionals etc)

pyproject.toml

@@ -211,6 +211,7 @@ cli = 'vendored-meson/meson/meson.py'
   ".spin/cmds.py:docs",
   ".spin/cmds.py:changelog",
   ".spin/cmds.py:notes",
-  ".spin/cmds.py:check_docs"
+  ".spin/cmds.py:check_docs",
+  ".spin/cmds.py:check_tutorials",
 ]
 "Metrics" = [".spin/cmds.py:bench"]