FEAT: Support for general static asset source locations

sphinxcontrib/jupyter/builders/jupyter.py

@@ -16,6 +16,7 @@
 import pdb
 import time
 from ..writers.utils import copy_dependencies
+from shutil import copyfile
 
 class JupyterBuilder(Builder):
     """
@@ -42,8 +43,8 @@ def init(self):
         self.executedir = self.outdir + '/executed'
         self.reportdir = self.outdir + '/reports/'
         self.errordir = self.outdir + "/reports/{}"
-        self.downloadsdir = self.outdir + "/_downloads"
-        self.downloadsExecutedir = self.downloadsdir + "/executed"
+        self.download_ipynb_dir = self.outdir + "/_downloads_ipynb"
+        self.downloads_ipynb_executed_dir = self.download_ipynb_dir + "/executed"
         self.client = None
 
         # Check default language is defined in the jupyter kernels
@@ -62,7 +63,7 @@ def init(self):
             instructions = overrides.split(",")
 
         for instruction in instructions:
-            if instruction:
+            if instruction:                                          #TODO: @AAKASH what is this doing?
                 if instruction == 'code_only':
                     self.config["jupyter_conversion_mode"] = "code"
                 else:
@@ -102,9 +103,16 @@ def init(self):
                 'delayed_notebooks': dict(),
                 'futures': [],
                 'delayed_futures': [],
-                'destination': self.downloadsExecutedir
+                'destination': self.downloads_ipynb_executed_dir
             }
 
+        self.image_library = {
+            'index' : [],
+        }
+        self.download_library = {
+            'index' : [],
+        }
+
     def get_outdated_docs(self):
         for docname in self.env.found_docs:
             if docname not in self.env.all_docs:
@@ -133,11 +141,10 @@ def prepare_writing(self, docnames):
         copy_dependencies(self)
 
         if (self.config["jupyter_execute_notebooks"]):
-             ## copies the dependencies to the executed folder
             copy_dependencies(self, self.executedir)
 
         if (self.config["jupyter_download_nb_execute"]):
-            copy_dependencies(self, self.downloadsExecutedir)
+            copy_dependencies(self, self.downloads_ipynb_executed_dir)
 
     def write_doc(self, docname, doctree):
         # work around multiple string % tuple issues in docutils;
@@ -147,7 +154,7 @@ def write_doc(self, docname, doctree):
         ### print an output for downloading notebooks as well with proper links if variable is set
         if "jupyter_download_nb" in self.config and self.config["jupyter_download_nb"]:
 
-            outfilename = os.path.join(self.downloadsdir, os_path(docname) + self.out_suffix)
+            outfilename = os.path.join(self.download_ipynb_dir, os_path(docname) + self.out_suffix)
             ensuredir(os.path.dirname(outfilename))
             self.writer._set_ref_urlpath(self.config["jupyter_download_nb_urlpath"])
             self.writer._set_jupyter_download_nb_image_urlpath((self.config["jupyter_download_nb_image_urlpath"]))
@@ -210,57 +217,103 @@ def update_Metadata(self, nb):
         return nb
 
     def copy_static_files(self):
-        # copy all static files
-        self.logger.info(bold("copying static files... "), nonl=True)
-        ensuredir(os.path.join(self.outdir, '_static'))
-        if (self.config["jupyter_execute_notebooks"]):
-            self.logger.info(bold("copying static files to executed folder... \n"), nonl=True)
-            ensuredir(os.path.join(self.executed_notebook_dir, '_static'))
-
-
-        # excluded = Matcher(self.config.exclude_patterns + ["**/.*"])
-        for static_path in self.config["jupyter_static_file_path"]:
-            entry = os.path.join(self.confdir, static_path)
-            if not os.path.exists(entry):
-                self.logger.warning(
-                    "jupyter_static_path entry {} does not exist"
-                    .format(entry))
+        """
+        Process Static Files including image and download libraries
+        """
+        self.process_image_library(self.outdir)
+        self.process_download_library(self.outdir)
+
+        if self.config['jupyter_execute_notebooks']:
+            self.process_image_library(self.executed_notebook_dir)
+            self.process_download_library(self.executed_notebook_dir)
+
+        # # copy all static files to build folder
+        # target = os.path.join(self.outdir, '_static')
+        # self.logger.info(bold("[builder] copying bulk static files to {}\n".format(target)), nonl=True)
+        # ensuredir(target)
+        # if self.config["jupyter_execute_notebooks"]:
+        #     target = os.path.join(self.executed_notebook_dir, '_static')
+        #     self.logger.info(bold("[builder] copying bulk static files to {}\n".format(target)), nonl=True)
+        #     ensuredir(target)
+
+        # # excluded = Matcher(self.config.exclude_patterns + ["**/.*"])
+        # for static_path in self.config["jupyter_static_file_path"]:
+        #     entry = os.path.join(self.confdir, static_path)
+        #     if not os.path.exists(entry):
+        #         self.logger.warning(
+        #             "[builder] jupyter_static_path entry {} does not exist"
+        #             .format(entry)
+        #             )
+        #     else:
+        #         copy_asset(entry, os.path.join(self.outdir, "_static"))
+        #         if self.config["jupyter_execute_notebooks"]:
+        #             copy_asset(entry, os.path.join(self.executed_notebook_dir, "_static"))
+
+    def process_image_library(self, context):
+        """
+        Action self.image_library
+        """
+        image_path = os.path.join(context, "_images")
+        self.logger.info(bold("[builder] copy images to {}".format(image_path)))
+        ensuredir(image_path)
+        for uri in self.image_library.keys():
+            if uri == "index":
+                continue
+            file, internal_image = self.image_library[uri]
+            if "../" in uri and internal_image == False:
+                num_steps = uri.count("../")
+                srcdir = "/".join(self.srcdir.split("/")[0:-1*num_steps])
             else:
-                copy_asset(entry, os.path.join(self.outdir, "_static"))
-                if (self.config["jupyter_execute_notebooks"]):
-                    copy_asset(entry, os.path.join(self.executed_notebook_dir, "_static"))
-        self.logger.info("done")
-
+                srcdir = self.srcdir
+            uri = uri.replace("../", "")
+            src = os.path.join(srcdir, uri)
+            target = os.path.join(image_path, file)
+            copyfile(src, target)
+
+    def process_download_library(self, context):
+        """
+        Action self.download_library
+        """
+        download_path = os.path.join(context, "_downloads")
+        self.logger.info(bold("[builder] copy downloads to {}".format(download_path)))
+        ensuredir(download_path)
+        for fl in self.download_library.keys():
+            if fl == "index":
+                continue
+            filename, internal_file, subfolder_depth = self.download_library[fl]
+            if "../" in fl and internal_file == False:
+                num_steps = fl.count("../")
+                srcdir = "/".join(self.srcdir.split("/")[0:-1*(num_steps - subfolder_depth)])
+            else:
+                srcdir = self.srcdir
+            fl = fl.replace("../", "")
+            src = os.path.join(srcdir, fl)
+            if not os.path.exists(src):
+                self.logger.info(bold("[builder] cannot find download: {}".format(src)))
+            else:
+                target = os.path.join(download_path, filename)
+                copyfile(src, target)
 
     def finish(self):
+        self.finish_tasks.add_task(self.copy_static_files)
 
-        if (self.config["jupyter_execute_notebooks"]):
+        if self.config["jupyter_execute_notebooks"]:
             self.finish_tasks.add_task(self.copy_static_files)
             self.save_executed_and_generate_coverage(self.execution_vars,'website', self.config['jupyter_make_coverage'])
 
-        if (self.config["jupyter_download_nb_execute"]):
+        if self.config["jupyter_download_nb_execute"]:
             self.finish_tasks.add_task(self.copy_static_files)
             self.save_executed_and_generate_coverage(self.download_execution_vars, 'downloads')
 
-        ### create a website folder
         if "jupyter_make_site" in self.config and self.config['jupyter_make_site']:
-            self._make_site_class.build_website(self)
-
-    def save_executed_and_generate_coverage(self, params, target, coverage = False):
-
-            # watch progress of the execution of futures
-            self.logger.info(bold("Starting notebook execution for %s and html conversion(if set in config)..."), target)
-            #progress(self.futures)
+            self._make_site_class.build_website()
 
+    def save_executed_and_generate_coverage(self, params, target, coverage=False):
+            self.logger.info(bold("[builder] Starting notebook execution for {} and html conversion...".format(target)))
             # save executed notebook
             error_results = self._execute_notebook_class.save_executed_notebook(self, params)
-
             ##generate coverage if config value set
             if coverage:
-                ## produces a JSON file of dask execution
-                self._execute_notebook_class.produce_dask_processing_report(self, params)
-
-                ## generate the JSON code execution reports file
+                self._execute_notebook_class.produce_dask_processing_report(self, params) 
                 error_results  = self._execute_notebook_class.produce_code_execution_report(self, error_results, params)
-
                 self._execute_notebook_class.create_coverage_report(self, error_results, params)

sphinxcontrib/jupyter/writers/make_site.py

@@ -6,82 +6,89 @@
 
 class MakeSiteWriter():
     """
-    Makes website for each package
+    Compile website from components
     """
     logger = logging.getLogger(__name__)
-    def __init__(self, builderSelf):
-        builddir = builderSelf.outdir
+
+    def __init__(self, builder):
+        self.builder = builder
 
         ## removing the /jupyter from path to get the top directory
-        index = builddir.rfind('/jupyter')
+        index = self.builder.outdir.rfind('/jupyter')
         if index > 0:
-            builddir = builddir[0:index]    
+            self.outdir = self.builder.outdir[0:index]
 
         ## defining directories
-        self.websitedir = builddir + "/jupyter_html/"
-        self.downloadipynbdir = self.websitedir + "/_downloads/ipynb/"
-
-    def build_website(self, builderSelf):
-        if os.path.exists(self.websitedir):
-            shutil.rmtree(self.websitedir)
-
-        builderSelf.themePath = builderSelf.config['jupyter_theme_path']
-        themeFolder = builderSelf.config['jupyter_theme']
-
-        if themeFolder is not None:
-            builderSelf.themePath = builderSelf.themePath + "/" + themeFolder
-
-        if os.path.exists(builderSelf.themePath):
-            pass
+        self.website_folder = self.outdir + "/jupyter_html/"
+        self.download_ipynb_folder = self.website_folder + "/_downloads/ipynb/"
+
+    def copy_theme_assets(self):
+        """ copies theme assets """
+        self.theme_path = self.builder.config['jupyter_theme_path']
+        self.theme_folder = self.builder.config['jupyter_theme']
+        if self.theme_folder is not None:
+            self.theme_path = self.theme_path + "/" + self.theme_folder
+        #-Check Theme Path-#
+        if not os.path.exists(self.theme_path):
+            self.logger.warning("[copy_theme_assets] the theme directory {} is not found".format(self.theme_path))
+            exit(1)
+        #-Copy HTML theme files to self.website_dir-#
+        self.html_assets_source = self.theme_path + "/html/"
+        if os.path.exists(self.html_assets_source):
+            copy_tree(self.html_assets_source, self.website_folder, preserve_symlinks=1)
         else:
-            self.logger.warning("theme directory not found")
-            exit()
-
-        htmlFolder = builderSelf.themePath + "/html/"
-        staticFolder = builderSelf.themePath + "/static"
-
-        ## copies the html and downloads folder
-        shutil.copytree(builderSelf.outdir + "/html/", self.websitedir, symlinks=True)
-
-        ## copies all the static files
-        shutil.copytree(builderSelf.outdir + "/_static/", self.websitedir + "_static/", symlinks=True)
-
-        ## copies all theme files to _static folder 
-        if os.path.exists(staticFolder):
-            copy_tree(staticFolder, self.websitedir + "_static/", preserve_symlinks=1)
-        else:
-            self.logger.warning("static folder not present in the themes directory")
-
-        ## copies the helper html files 
-        if os.path.exists(htmlFolder):
-            copy_tree(htmlFolder, self.websitedir, preserve_symlinks=1)
+            self.logger.warning("[copy_theme_assets] html folder not present in the themes directory")   #@AAKASH will there always be an html folder required in theme folder?
+        #-Copy other static assets to _static-#
+        self.static_theme_source = self.theme_path + "/static"
+        if os.path.exists(self.static_theme_source):
+            copy_tree(self.static_theme_source, self.website_folder + "_static/", preserve_symlinks=1)
         else:
-            self.logger.warning("html folder not present in the themes directory")
-
-
-        if "jupyter_coverage_dir" in builderSelf.config and builderSelf.config["jupyter_coverage_dir"]:
-            if os.path.exists(builderSelf.config['jupyter_coverage_dir']):
-                self.coveragedir = builderSelf.config['jupyter_coverage_dir']
+            self.logger.warning("[copy_theme_assets] static folder not present in the themes directory")
+
+    def copy_image_library(self):
+        """ copies image library """
+        image_path = os.path.join(self.builder.outdir, "_images")
+        if os.path.exists(image_path):
+            shutil.copytree(image_path, self.website_folder + "_images/", symlinks=True)
+
+    def copy_download_library(self):
+        """ copies download library """
+        download_path = os.path.join(self.builder.outdir, "_downloads")
+        if os.path.exists(download_path):
+            shutil.copytree(download_path, self.website_folder + "_downloads/", symlinks=True)
+
+    def build_website(self):
+        # clean old website
+        if os.path.exists(self.website_folder):
+            shutil.rmtree(self.website_folder)
+
+        # copies html and downloads folder
+        shutil.copytree(self.builder.outdir + "/html/", self.website_folder, symlinks=True)
+        self.copy_image_library()
+        self.copy_download_library()
+        self.copy_theme_assets()
+
+        ## copies all the static files (TODO: disable to debug!)
+        # shutil.copytree(self.builder.outdir + "/_static/", self.website_folder + "_static/", symlinks=True)
+
+        if "jupyter_coverage_dir" in self.builder.config and self.builder.config["jupyter_coverage_dir"]:
+            if os.path.exists(self.builder.config['jupyter_coverage_dir']):
+                self.coveragedir = self.builder.config['jupyter_coverage_dir']
                 ## copies the report of execution results
                 if os.path.exists(self.coveragedir + "/jupyter/reports/code-execution-results.json"):
-                    shutil.copy2(self.coveragedir + "/jupyter/reports/code-execution-results.json", self.websitedir + "_static/")
+                    shutil.copy2(self.coveragedir + "/jupyter/reports/code-execution-results.json", self.website_folder + "_static/")
             else:
-                self.logger.error("coverage directory not found. Please ensure to run coverage build before running website build")
-        else:
-            self.logger.error(" coverage directory nbot specified. Please specify coverage directory for creating website reports ")
-
-
-        ## copies the downloads folder
-        if "jupyter_download_nb" in builderSelf.config and builderSelf.config["jupyter_download_nb"]:
-            if builderSelf.config["jupyter_download_nb_execute"]:
-                sourceDownloads = builderSelf.outdir + "/_downloads/executed"
-            else: 
-                sourceDownloads = builderSelf.outdir + "/_downloads"
-            if os.path.exists(sourceDownloads):
-                shutil.copytree(sourceDownloads, self.downloadipynbdir, symlinks=True)
+                self.logger.error("[coverage] coverage directory {} not found. Please ensure to run coverage build \
+                                    before running website build".format(self.builder.config['jupyter_coverage_dir'])
+                                )
+
+        ## copies the downloadable ipynb assets to downloads ipynb support folder
+        if "jupyter_download_nb" in self.builder.config and self.builder.config["jupyter_download_nb"]:
+            if self.builder.config['jupyter_download_nb_execute']:
+                download_ipynb_source = self.outdir + "/jupyter/_downloads_ipynb/executed"
             else:
-                self.logger.warning("Downloads folder not created during build")
-
-
-
-
+                download_ipynb_source = self.outdir + "/jupyter/_downloads_ipynb"
+            if os.path.exists(download_ipynb_source):
+                shutil.copytree(download_ipynb_source, self.download_ipynb_folder, symlinks=True)
+            else:
+                self.logger.warning("[make_site] IPYNB downloads folder {} not created during build".format(download_ipynb_source))