Check notebooks are in table of contents, bump some hooks (#4197)

* check notebooks are in table of contents

* check notebooks are in table of contents

* update notebook

* :doc:

* restrict check-toc to jupyter

* 🎨

Co-authored-by: Marco Gorelli <[email protected]>

Author: MarcoGorelli

.pre-commit-config.yaml

@@ -1,17 +1,17 @@
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v3.2.0
+  rev: v3.3.0
   hooks:
     -   id: end-of-file-fixer
     -   id: check-toml
 - repo: https://github.com/nbQA-dev/nbQA
-  rev: 0.3.2
+  rev: 0.3.5
   hooks:
     - id: nbqa-black
     - id: nbqa-isort
     - id: nbqa-pyupgrade
 - repo: https://github.com/asottile/pyupgrade
-  rev: v2.7.2
+  rev: v2.7.3
   hooks:
     - id: pyupgrade
       args: [--py36-plus]
@@ -26,3 +26,8 @@ repos:
       types: [jupyter]
       entry: python scripts/check_watermark.py
       language: python
+    - id: check-toc
+      name: Check all notebooks appear in table of contents
+      types: [jupyter]
+      entry: python scripts/check_toc_is_complete.py
+      language: python

docs/source/notebooks/blackbox_external_likelihood.ipynb

@@ -51,7 +51,7 @@
     "import theano\n",
     "import theano.tensor as tt\n",
     "\n",
-    "print(\"Running on PyMC3 v{}\".format(pm.__version__))"
+    "print(f\"Running on PyMC3 v{pm.__version__}\")"
    ]
   },
   {
@@ -697,7 +697,7 @@
     "            color=colors[i],\n",
     "            hist_kwargs={\"density\": True},\n",
     "            **hist2dkwargs,\n",
-    "            truths=[mtrue, ctrue]\n",
+    "            truths=[mtrue, ctrue],\n",
     "        )\n",
     "    else:\n",
     "        corner.corner(\n",
@@ -729,15 +729,15 @@
     "test_grad_op_func = theano.function([var], test_grad_op(var))\n",
     "grad_vals = test_grad_op_func([mtrue, ctrue])\n",
     "\n",
-    "print('Gradient returned by \"LogLikeGrad\": {}'.format(grad_vals))\n",
+    "print(f'Gradient returned by \"LogLikeGrad\": {grad_vals}')\n",
     "\n",
     "# test the gradient called through LogLikeWithGrad\n",
     "test_gradded_op = LogLikeWithGrad(my_loglike, data, x, sigma)\n",
     "test_gradded_op_grad = tt.grad(test_gradded_op(var), var)\n",
     "test_gradded_op_grad_func = theano.function([var], test_gradded_op_grad)\n",
     "grad_vals_2 = test_gradded_op_grad_func([mtrue, ctrue])\n",
     "\n",
-    "print('Gradient returned by \"LogLikeWithGrad\": {}'.format(grad_vals_2))\n",
+    "print(f'Gradient returned by \"LogLikeWithGrad\": {grad_vals_2}')\n",
     "\n",
     "# test the gradient that PyMC3 uses for the Normal log likelihood\n",
     "test_model = pm.Model()\n",
@@ -751,7 +751,7 @@
     "    gradfunc.set_extra_values({\"m_interval__\": mtrue, \"c_interval__\": ctrue})\n",
     "    grad_vals_pymc3 = gradfunc(np.array([mtrue, ctrue]))[1]  # get dlogp values\n",
     "\n",
-    "print('Gradient returned by PyMC3 \"Normal\" distribution: {}'.format(grad_vals_pymc3))"
+    "print(f'Gradient returned by PyMC3 \"Normal\" distribution: {grad_vals_pymc3}')"
    ]
   },
   {

docs/source/notebooks/table_of_contents_examples.js

@@ -34,6 +34,7 @@ Gallery.contents = {
     "gaussian_process": "Gaussian Processes",
     "conditional-autoregressive-model": "Gaussian Processes",
     "log-gaussian-cox-process":"Gaussian Processes",
+    "GP-Circular": "Gaussian Processes",
     "dependent_density_regression": "Mixture Models",
     "dp_mix": "Mixture Models",
     "gaussian-mixture-model-advi": "Mixture Models",
@@ -59,6 +60,7 @@ Gallery.contents = {
     "ODE_with_manual_gradients": "Inference in ODE models",
     "ODE_API_introduction": "Inference in ODE models",
     "probabilistic_matrix_factorization": "Case Studies",
+    "MLDA_introduction": "MCMC",
     "MLDA_simple_linear_regression": "MCMC",
     "MLDA_gravity_surveying": "MCMC",
     "MLDA_variance_reduction_linear_regression": "MCMC"

docs/source/notebooks/table_of_contents_tutorials.js

@@ -6,13 +6,15 @@ Gallery.contents = {
     "Gaussian_Processes.rst": "Basics",
     "data_container": "Basics",
     "sampling_compound_step": "Deep dives",
+    "sampling_callback": "Deep dives",
     "sampler-stats": "Deep dives",
     "Diagnosing_biased_Inference_with_Divergences": "Deep dives",
     "Advanced_usage_of_Theano_in_PyMC3.rst": "Deep dives",
     "getting_started": "Deep dives",
     "ODE_API_shapes_and_benchmarking": "Deep dives",
     "DEMetropolisZ_EfficiencyComparison": "Deep dives",
     "DEMetropolisZ_tune_drop_fraction": "Deep dives",
+    "factor_analysis": "Deep dives",
     "blackbox_external_likelihood": "How-To",
     "profiling": "How-To",
     "howto_debugging": "How-To",

pyproject.toml

@@ -13,6 +13,3 @@ pyupgrade = 1
 [tool.nbqa.addopts]
 isort = ["--treat-comment-as-code", "# %%"]
 pyupgrade = ["--py36-plus"]
-
-[tool.nbqa.ignore_cells]
-black = "%%cython"

scripts/check_toc_is_complete.py

@@ -0,0 +1,24 @@
+"""
+Check that given Jupyter notebooks all appear in the table of contents.
+
+This is intended to be used as a pre-commit hook, see `.pre-commit-config.yaml`.
+You can run it manually with `pre-commit run check-toc --all`.
+"""
+
+import json
+from pathlib import Path
+import argparse
+
+if __name__ == "__main__":
+    toc_examples = (Path("docs") / "source/notebooks/table_of_contents_examples.js").read_text()
+    toc_tutorials = (Path("docs") / "source/notebooks/table_of_contents_tutorials.js").read_text()
+    toc_keys = {
+        **json.loads(toc_examples[toc_examples.find("{") :]),
+        **json.loads(toc_tutorials[toc_tutorials.find("{") :]),
+    }.keys()
+    parser = argparse.ArgumentParser()
+    parser.add_argument("filenames", nargs="*")
+    args = parser.parse_args()
+    for file_ in args.filenames:
+        stem = Path(file_).stem
+        assert stem in toc_keys, f"Notebook '{stem}' not added to table of contents!"