Merge pull request #309 from minrk/stencila

stencila support

Author: choldgraf

.gitignore

@@ -9,6 +9,7 @@ MANIFEST
 
 .DS_Store
 .cache
+.pytest_cache
 .coverage
 htmlcov
 

.travis.yml

@@ -37,6 +37,7 @@ env:
   matrix:
   - REPO_TYPE=conda
   - REPO_TYPE=venv
+  - REPO_TYPE=stencila
   - REPO_TYPE=julia
   - REPO_TYPE=r
   - REPO_TYPE=dockerfile

repo2docker/buildpacks/base.py

@@ -65,9 +65,9 @@
 
 EXPOSE 8888
 
-{% if env -%}
-# Almost all environment variables
-{% for item in env -%}
+{% if build_env -%}
+# Environment variables required for build
+{% for item in build_env -%}
 ENV {{item[0]}} {{item[1]}}
 {% endfor -%}
 {% endif -%}
@@ -94,6 +94,14 @@
 COPY src/ ${HOME}
 RUN chown -R ${NB_USER}:${NB_USER} ${HOME}
 
+{% if env -%}
+# The rest of the environment
+{% for item in env -%}
+ENV {{item[0]}} {{item[1]}}
+{% endfor -%}
+{% endif -%}
+
+
 # Run assemble scripts! These will actually build the specification
 # in the repository into the image.
 {% for sd in assemble_script_directives -%}
@@ -182,6 +190,21 @@ def get_base_packages(self):
             "unzip",
         }
 
+    def get_build_env(self):
+        """
+        Ordered list of environment variables to be set for this image.
+
+        Ordered so that environment variables can use other environment
+        variables in their values.
+
+        Expects tuples, with the first item being the environment variable
+        name and the second item being the value.
+
+        These environment variables will be set prior to build.
+        Use .get_env() to set environment variables after build.
+        """
+        return []
+
     def get_env(self):
         """
         Ordered list of environment variables to be set for this image.
@@ -191,6 +214,8 @@ def get_env(self):
 
         Expects tuples, with the first item being the environment variable
         name and the second item being the value.
+
+        These variables will not be available to build.
         """
         return []
 
@@ -225,6 +250,30 @@ def get_build_script_files(self):
         """
         return {}
 
+
+    @property
+    def stencila_manifest_dir(self):
+        """Find the stencila manifest dir if it exists"""
+        if hasattr(self, '_stencila_manifest_dir'):
+            return self._stencila_manifest_dir
+
+        # look for a manifest.xml that suggests stencila could be used
+        # when we find one, stencila should be installed
+        # and set environment variables such that
+        # this file is located at:
+        # ${STENCILA_ARCHIVE_DIR}/${STENCILA_ARCHIVE}/manifest.xml
+
+        self._stencila_manifest_dir = None
+        for root, dirs, files in os.walk("."):
+            if "manifest.xml" in files:
+                self._stencila_manifest_dir = root.split(os.path.sep, 1)[1]
+                self.log.info(
+                    "Found stencila manifest.xml in %s",
+                    self._stencila_manifest_dir,
+                )
+                break
+        return self._stencila_manifest_dir
+
     def get_build_scripts(self):
         """
         Ordered list of shell script snippets to build the base image.
@@ -322,6 +371,7 @@ def render(self):
         return t.render(
             packages=sorted(self.get_packages()),
             path=self.get_path(),
+            build_env=self.get_build_env(),
             env=self.get_env(),
             labels=self.get_labels(),
             build_script_directives=build_script_directives,
@@ -388,11 +438,31 @@ def _filter_tar(tar):
 
 
 class BaseImage(BuildPack):
-    def get_env(self):
+    def get_build_env(self):
+        """Return env directives required for build"""
         return [
             ("APP_BASE", "/srv")
         ]
 
+    def get_env(self):
+        """Return env directives to be set after build"""
+        env = []
+        if self.stencila_manifest_dir:
+            # manifest_dir is the path containing the manifest.xml
+            # archive_dir is the directory containing archive directories (one level up)
+            # default archive is the name of the directory in the archive_dir
+            # such that
+            # ${STENCILA_ARCHIVE_DIR}/${STENCILA_ARCHIVE}/manifest.xml
+            # exists.
+
+            archive_dir, archive = os.path.split(self.stencila_manifest_dir)
+            env.extend([
+                ("STENCILA_ARCHIVE_DIR", "${HOME}/" + archive_dir),
+                ("STENCILA_ARCHIVE", archive),
+            ])
+        return env
+
+
     def detect(self):
         return True
 
@@ -425,6 +495,22 @@ def get_assemble_scripts(self):
             ))
         except FileNotFoundError:
             pass
+        if self.stencila_manifest_dir:
+            assemble_scripts.extend(
+                [
+                    (
+                        "${NB_USER}",
+                        r"""
+                        ${KERNEL_PYTHON_PREFIX}/bin/pip install --no-cache https://github.com/stencila/py/archive/f6a245fd.tar.gz && \
+                        ${KERNEL_PYTHON_PREFIX}/bin/python -m stencila register && \
+                        ${NB_PYTHON_PREFIX}/bin/pip install --no-cache nbstencilaproxy==0.1.1 && \
+                        jupyter serverextension enable --sys-prefix --py nbstencilaproxy && \
+                        jupyter nbextension install    --sys-prefix --py nbstencilaproxy && \
+                        jupyter nbextension enable     --sys-prefix --py nbstencilaproxy
+                        """,
+                    )
+                ]
+            )
         return assemble_scripts
 
     def get_post_build_scripts(self):

repo2docker/buildpacks/conda/__init__.py

@@ -18,14 +18,14 @@ class CondaBuildPack(BaseImage):
     Uses miniconda since it is more lightweight than Anaconda.
 
     """
-    def get_env(self):
+    def get_build_env(self):
         """Return environment variables to be set.
 
         We set `CONDA_DIR` to the conda install directory and
         the `NB_PYTHON_PREFIX` to the location of the jupyter binary.
 
         """
-        env = super().get_env() + [
+        env = super().get_build_env() + [
             ('CONDA_DIR', '${APP_BASE}/conda'),
             ('NB_PYTHON_PREFIX', '${CONDA_DIR}'),
         ]

repo2docker/buildpacks/julia.py

@@ -12,7 +12,7 @@ class JuliaBuildPack(CondaBuildPack):
     See https://github.com/JuliaPy/PyCall.jl/issues/410
 
     """
-    def get_env(self):
+    def get_build_env(self):
         """Get additional environment settings for Julia and Jupyter
 
         Returns:
@@ -31,7 +31,7 @@ def get_env(self):
             For example, a tuple may be `('JULIA_VERSION', '0.6.0')`.
 
         """
-        return super().get_env() + [
+        return super().get_build_env() + [
             ('JULIA_PATH', '${APP_BASE}/julia'),
             ('JULIA_HOME', '${JULIA_PATH}/bin'),
             ('JULIA_PKGDIR', '${JULIA_PATH}/pkg'),

repo2docker/buildpacks/r.py

@@ -82,15 +82,15 @@ def get_path(self):
             '/usr/lib/rstudio-server/bin/'
         ]
 
-    def get_env(self):
+    def get_build_env(self):
         """
         Return environment variables to be set.
 
         We want libraries to be installed in a path that users can write to
         without needing root. This is set via the `R_LIBS_USER` environment
         variable, so we set that here.
         """
-        return super().get_env() + [
+        return super().get_build_env() + [
             # This is the path where user libraries are installed
             ('R_LIBS_USER', '${APP_BASE}/rlibs')
         ]

tests/stencila/basic/archive/py-jupyter/bibliography.bibtex

@@ -0,0 +1,21 @@
+@article{kluyver2016jupyter,
+  title={Jupyter Notebooks-a publishing format for reproducible computational workflows.},
+  author={Kluyver, Thomas and Ragan-Kelley, Benjamin and P{\'e}rez, Fernando and Granger, Brian E and Bussonnier, Matthias and Frederic, Jonathan and Kelley, Kyle and Hamrick, Jessica B and Grout, Jason and Corlay, Sylvain and others},
+  journal={ELPUB},
+  pages={87--90},
+  year={2016}
+}
+@article{ragan2014jupyter,
+  title={The Jupyter/IPython architecture: a unified view of computational research, from interactive exploration to communication and publication.},
+  author={Ragan-Kelley, M and Perez, F and Granger, B and Kluyver, T and Ivanov, P and Frederic, J and Bussonnier, M},
+  journal={AGU Fall Meeting Abstracts},
+  year={2014}
+}
+@article{perez2015project,
+  title={Project Jupyter: Computational narratives as the engine of collaborative data science},
+  author={Perez, Fernando and Granger, Brian E},
+  journal={Retrieved September},
+  volume={11},
+  pages={207},
+  year={2015}
+}

tests/stencila/basic/archive/py-jupyter/manifest.xml

@@ -0,0 +1,6 @@
+<dar>
+  <documents>
+    <document id="py-jupyter.ipynb" name="py-jupyter.ipynb" type="article" path="py-jupyter.ipynb.jats.xml" src="py-jupyter.ipynb" />
+  </documents>
+  <assets/>
+</dar>