Merge branch 'master' into inline_nodes

Author: akhmerov

.circleci/config.yml

@@ -18,7 +18,7 @@ jobs:
           name: Build documentation
           command: |
             cd doc
-            make html
+            make html-strict
 
       - store_artifacts:
           path: doc/build/html/

.github/workflows/tests.yml

@@ -6,10 +6,15 @@ jobs:
 
   tests:
 
-    runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.5, 3.6, 3.7, 3.8]
+        python-version: [3.6, 3.7, 3.8]
+        os: [ubuntu-latest]
+        include:
+          - os: windows-latest
+            python-version: 3.7
+    
+    runs-on: ${{ matrix.os }}
 
     steps:
     - uses: actions/checkout@v2

doc/Makefile

@@ -16,4 +16,11 @@ help:
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+# raise warnings to errors
+html-strict:
+	@$(SPHINXBUILD) -b html -nW --keep-going "$(SOURCEDIR)" "$(BUILDDIR)/html" $(SPHINXOPTS) $(O)
+
+clean:
+	rm -r $(BUILDDIR)

doc/source/index.rst

@@ -317,14 +317,18 @@ Pygments.
 
 Downloading the code as a script
 --------------------------------
+
 Jupyter Sphinx includes 2 roles that can be used to download the code embedded in a document:
-``:jupyter-download:script:`` (for a raw script file) and ``:jupyter-download:notebook:`` (for
-a Jupyter notebook). For example, to download all the code from this document as a script we
+``:jupyter-download:script:`` (for a raw script file) and ``:jupyter-download:notebook:`` or ``:jupyter-download:nb:`` (for
+a Jupyter notebook). 
+
+These roles are equivalent to the standard sphinx `download role <https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html#role-download>`__, **except** the extension of the file should not be given.
+For example, to download all the code from this document as a script we
 would use::
 
-    :jupyter-download:script:`index`
+    :jupyter-download:script:`click to download <index>`
 
-Which produces a link like this: :jupyter-download:script:`index`. The name that the role is
+Which produces a link like this: :jupyter-download:script:`click to download <index>`. The target that the role is
 applied to (``index`` in this case) is the name of the document for which you wish to download
 the code. If a document contains ``jupyter-kernel`` directives with ``:id:`` specified, then
 the name provided to ``:id:`` can be used to get the code for the cells belonging to the

jupyter_sphinx/__init__.py

@@ -8,6 +8,7 @@
 from sphinx.util.fileutil import copy_asset
 from sphinx.errors import ExtensionError
 from IPython.lib.lexers import IPythonTracebackLexer, IPython3Lexer
+from pathlib import Path
 
 from .ast import (
     JupyterCell,
@@ -19,7 +20,7 @@
     JupyterWidgetViewNode,
     JupyterWidgetStateNode,
     WIDGET_VIEW_MIMETYPE,
-    jupyter_download_role,
+    JupyterDownloadRole,
     CellOutputsToNodes,
 )
 from .execute import JupyterKernel, ExecuteJupyterCells
@@ -121,18 +122,20 @@ def build_finished(app, env):
     if app.builder.format != "html":
         return
 
+    module_dir = Path(__file__).parent
+    outdir = Path(app.outdir)
+
     # Copy stylesheet
-    src = os.path.join(os.path.dirname(__file__), "css")
-    dst = os.path.join(app.outdir, "_static")
+    src = module_dir / "css"
+    dst = outdir / "_static"
     copy_asset(src, dst)
 
     thebe_config = app.config.jupyter_sphinx_thebelab_config
     if not thebe_config:
         return
 
     # Copy all thebelab related assets
-    src = os.path.join(os.path.dirname(__file__), "thebelab")
-    dst = os.path.join(app.outdir, "_static")
+    src = module_dir / "thebelab"
     copy_asset(src, dst)
 
 
@@ -272,14 +275,15 @@ def setup(app):
     app.add_directive("jupyter-execute", JupyterCell)
     app.add_directive("jupyter-kernel", JupyterKernel)
     app.add_directive("thebe-button", ThebeButton)
-    app.add_role("jupyter-download:notebook", jupyter_download_role)
-    app.add_role("jupyter-download:script", jupyter_download_role)
+    app.add_role("jupyter-download:notebook", JupyterDownloadRole())
+    app.add_role("jupyter-download:nb", JupyterDownloadRole())
+    app.add_role("jupyter-download:script", JupyterDownloadRole())
     app.add_transform(ExecuteJupyterCells)
     app.add_post_transform(CellOutputsToNodes)
 
     # For syntax highlighting
-    app.add_lexer("ipythontb", IPythonTracebackLexer())
-    app.add_lexer("ipython", IPython3Lexer())
+    app.add_lexer("ipythontb", IPythonTracebackLexer)
+    app.add_lexer("ipython", IPython3Lexer)
 
     app.connect("builder-inited", builder_inited)
     app.connect("build-finished", build_finished)

jupyter_sphinx/_version.py

@@ -1,4 +1,4 @@
-version_info = (0, 2, 5, "alpha", 1)
+version_info = (0, 3, 0, "alpha", 1)
 
 _specifier_ = {"alpha": "a", "beta": "b", "candidate": "rc", "final": ""}
 

jupyter_sphinx/ast.py

@@ -2,11 +2,13 @@
 
 import os
 import json
+from pathlib import Path
 
 import docutils
 from docutils.parsers.rst import Directive, directives
-from docutils.nodes import math_block, image
+from docutils.nodes import math_block, image, literal
 from sphinx.util import parselinenos
+from sphinx.util.docutils import ReferenceRole
 from sphinx.addnodes import download_reference
 from sphinx.transforms import SphinxTransform
 from sphinx.environment.collectors.asset import ImageCollector
@@ -100,7 +102,7 @@ def run(self):
                     location=location,
                 )
             try:
-                with open(filename) as f:
+                with Path(filename).open() as f:
                     content = [line.rstrip() for line in f.readlines()]
             except (IOError, OSError):
                 raise IOError("File {} not found or reading it failed".format(filename))
@@ -227,7 +229,7 @@ def html(self):
         )
 
 
-def cell_output_to_nodes(outputs, data_priority, write_stderr, dir,
+def cell_output_to_nodes(outputs, data_priority, write_stderr, out_dir,
                          thebe_config, inline=False):
     """Convert a jupyter cell with outputs and filenames to doctree nodes.
 
@@ -238,7 +240,7 @@ def cell_output_to_nodes(outputs, data_priority, write_stderr, dir,
         Which media types to prioritize.
     write_stderr : bool
         If True include stderr in cell output
-    dir : string
+    out_dir : string
         Sphinx "absolute path" to the output folder, so it is a relative path
         to the source folder prefixed with ``/``.
     thebe_config: dict
@@ -317,19 +319,17 @@ def cell_output_to_nodes(outputs, data_priority, write_stderr, dir,
                 continue
             data = output["data"][mime_type]
             if mime_type.startswith("image"):
+                file_path = Path(output.metadata["filenames"][mime_type])
+                out_dir = Path(out_dir)
                 # Sphinx treats absolute paths as being rooted at the source
                 # directory, so make a relative path, which Sphinx treats
                 # as being relative to the current working directory.
-                filename = os.path.basename(output.metadata["filenames"][mime_type])
+                filename = file_path.name
 
-                # checks if file dir path is inside a subdir of dir
-                filedir = os.path.dirname(output.metadata["filenames"][mime_type])
-                subpaths = filedir.split(dir)
-                if subpaths and len(subpaths) > 1:
-                    subpath = subpaths[1]
-                    dir += subpath
+                if out_dir in file_path.parents:
+                    out_dir = file_path.parent
 
-                uri = os.path.join(dir, filename)
+                uri = (out_dir / filename).as_posix()
                 to_add.append(docutils.nodes.image(uri=uri))
             elif mime_type == "text/html":
                 to_add.append(
@@ -407,16 +407,19 @@ def attach_outputs(output_nodes, node, thebe_config, cm_language):
         node.children = node.children[::-1]
 
 
-def jupyter_download_role(name, rawtext, text, lineno, inliner):
-    _, filetype = name.split(":")
-    assert filetype in ("notebook", "script")
-    ext = ".ipynb" if filetype == "notebook" else ".py"
-    output_dir = sphinx_abs_dir(inliner.document.settings.env)
-    download_file = text + ext
-    node = download_reference(
-        download_file, download_file, reftarget=os.path.join(output_dir, download_file)
-    )
-    return [node], []
+class JupyterDownloadRole(ReferenceRole):
+    def run(self):
+        _, filetype = self.name.split(":")
+
+        assert filetype in ("notebook", "nb", "script")
+        ext = ".ipynb" if filetype in ("notebook", "nb") else ".py"
+        download_file = self.target + ext
+        reftarget = sphinx_abs_dir(self.env, download_file)
+        node = download_reference(self.rawtext, reftarget=reftarget)
+        self.set_source_info(node)
+        title = self.title if self.has_explicit_title else download_file
+        node += literal(self.rawtext, title, classes=["xref", "download"])
+        return [node], []
 
 
 def get_widgets(notebook):

jupyter_sphinx/execute.py

@@ -1,6 +1,7 @@
 """Execution and managing kernels."""
 
 import os
+from pathlib import Path
 
 from sphinx.transforms import SphinxTransform
 from sphinx.errors import ExtensionError
@@ -22,7 +23,6 @@
     default_notebook_names,
     output_directory,
     split_on,
-    sphinx_abs_dir,
     blank_nb,
 )
 from .ast import (
@@ -87,8 +87,9 @@ class ExecuteJupyterCells(SphinxTransform):
 
     def apply(self):
         doctree = self.document
-        doc_relpath = os.path.dirname(self.env.docname)  # relative to src dir
-        docname = os.path.basename(self.env.docname)
+        docname_path = Path(self.env.docname)
+        doc_dir_relpath = docname_path.parent  # relative to src dir
+        docname = docname_path.name
         default_kernel = self.config.jupyter_execute_default_kernel
         default_names = default_notebook_names(docname)
         thebe_config = self.config.jupyter_sphinx_thebelab_config
@@ -106,7 +107,7 @@ def apply(self):
             add_thebelab_library(doctree, self.env)
 
         js.logger.info("executing {}".format(docname))
-        output_dir = os.path.join(output_directory(self.env), doc_relpath)
+        output_dir = Path(output_directory(self.env)) / doc_dir_relpath
 
         # Start new notebook whenever a JupyterKernelNode is encountered
         jupyter_nodes = (JupyterCellNode, JupyterKernelNode)
@@ -206,7 +207,7 @@ def apply(self):
             # Write certain cell outputs (e.g. images) to separate files, and
             # modify the metadata of the associated cells in 'notebook' to
             # include the path to the output file.
-            write_notebook_output(notebook, output_dir, file_name)
+            write_notebook_output(notebook, str(output_dir), file_name, self.env.docname)
 
             try:
                 cm_language = notebook.metadata.language_info.codemirror_mode.name
@@ -239,7 +240,7 @@ def execute_cells(kernel_name, cells, execute_kwargs):
     return notebook
 
 
-def write_notebook_output(notebook, output_dir, notebook_name):
+def write_notebook_output(notebook, output_dir, notebook_name, location=None):
     """Extract output from notebook cells and write to files in output_dir.
 
     This also modifies 'notebook' in-place, adding metadata to each cell that
@@ -256,11 +257,20 @@ def write_notebook_output(notebook, output_dir, notebook_name):
         resources,
         os.path.join(output_dir, notebook_name + ".ipynb"),
     )
-    # Write a script too.
-    ext = notebook.metadata.language_info.file_extension
+    # Write a script too.  Note that utf-8 is the de facto
+    # standard encoding for notebooks. 
+    ext = notebook.metadata.get("language_info", {}).get("file_extension", None)
+    if ext is None:
+        ext = ".txt"
+        js.logger.warning(
+            "Notebook code has no file extension metadata, " "defaulting to `.txt`",
+            location=location,
+        )
     contents = "\n\n".join(cell.source for cell in notebook.cells)
-    with open(os.path.join(output_dir, notebook_name + ext), "w") as f:
-        f.write(contents)
+
+    notebook_file = notebook_name + ext
+    output_dir = Path(output_dir)
+    (output_dir / notebook_file).write_text(contents, encoding = "utf8")
 
 
 def contains_widgets(notebook):

jupyter_sphinx/thebelab.py

@@ -3,6 +3,7 @@
 import json
 import docutils
 from docutils.parsers.rst import Directive
+from pathlib import Path
 
 import jupyter_sphinx as js
 
@@ -86,23 +87,23 @@ def add_thebelab_library(doctree, env):
     if isinstance(thebe_config, dict):
         pass
     elif isinstance(thebe_config, str):
-        if os.path.isabs(thebe_config):
+        thebe_config = Path(thebe_config)
+        if thebe_config.is_absolute():
             filename = thebe_config
         else:
-            filename = os.path.join(os.path.abspath(env.app.srcdir), thebe_config)
+            filename = Path(env.app.srcdir).resolve() / thebe_config
 
-        if not os.path.exists(filename):
+        if not filename.exists():
             js.logger.warning("The supplied thebelab configuration file does not exist")
             return
 
-        with open(filename, "r") as config_file:
-            try:
-                thebe_config = json.load(config_file)
-            except ValueError:
-                js.logger.warning(
-                    "The supplied thebelab configuration file is not in JSON format."
-                )
-                return
+        try:
+            thebe_config = json.loads(filename.read_text())
+        except ValueError:
+            js.logger.warning(
+                "The supplied thebelab configuration file is not in JSON format."
+            )
+            return
     else:
         js.logger.warning(
             "The supplied thebelab configuration should be either a filename or a dictionary."

jupyter_sphinx/utils.py

@@ -4,6 +4,7 @@
 from sphinx.errors import ExtensionError
 import nbformat
 from jupyter_client.kernelspec import get_kernel_spec, NoSuchKernel
+from pathlib import Path
 
 
 def blank_nb(kernel_name):
@@ -69,17 +70,19 @@ def language_info(executor):
     return info_msg["content"]["language_info"]
 
 
-def sphinx_abs_dir(env):
+def sphinx_abs_dir(env, *paths):
     # We write the output files into
     # output_directory / jupyter_execute / path relative to source directory
     # Sphinx expects download links relative to source file or relative to
     # source dir and prepended with '/'. We use the latter option.
-    return "/" + os.path.relpath(
-        os.path.abspath(
-            os.path.join(output_directory(env), os.path.dirname(env.docname))
-        ),
-        os.path.abspath(env.app.srcdir),
-    )
+    out_path = (output_directory(env) /  Path(env.docname).parent / Path(*paths)).resolve()
+      
+    if os.name == "nt":
+        # Can't get relative path between drives on Windows
+        return out_path.as_posix()
+
+    # Path().relative_to() doesn't work when not a direct subpath
+    return "/" + os.path.relpath(out_path, env.app.srcdir)
 
 
 def output_directory(env):
@@ -90,6 +93,4 @@ def output_directory(env):
 
     # Note: we are using an implicit fact that sphinx output directories are
     # direct subfolders of the build directory.
-    return os.path.abspath(
-        os.path.join(env.app.outdir, os.path.pardir, "jupyter_execute")
-    )
+    return (Path(env.app.outdir) / os.path.pardir / "jupyter_execute").resolve()