Merge pull request #843 from krinsman/step3

Step 3

Author: parente

README.md

@@ -84,7 +84,7 @@ $ docker-compose up
 
 ### Local Installation
 
-The Notebook Viewer requires several binary packages to be installed on your system. The primary ones are `libmemcached-dev libcurl4-openssl-dev pandoc libevent-dev`. Package names may differ on your system, see [salt-states](https://github.com/rgbkrk/salt-states-nbviewer/blob/master/nbviewer/init.sls) for more details.
+The Notebook Viewer requires several binary packages to be installed on your system. The primary ones are `libmemcached-dev libcurl4-openssl-dev pandoc libevent-dev libgnutls28-dev`. Package names may differ on your system, see [salt-states](https://github.com/rgbkrk/salt-states-nbviewer/blob/master/nbviewer/init.sls) for more details.
 
 If they are installed, you can install the required Python packages via pip.
 

nbviewer/app.py

@@ -25,7 +25,7 @@
 
 from jinja2 import Environment, FileSystemLoader
 
-from traitlets import Unicode, Any, Set, default
+from traitlets import Any, Dict, Set, Unicode, default
 from traitlets.config import Application
 
 from .handlers import init_handlers
@@ -83,6 +83,9 @@ class NBViewer(Application):
 
     config_file = Unicode('nbviewer_config.py', help="The config file to load").tag(config=True)
 
+    # Use this to insert custom configuration of handlers for NBViewer extensions
+    handler_settings = Dict().tag(config=True)
+
     url_handler         = Unicode(default_value="nbviewer.providers.url.handlers.URLHandler",           help="The Tornado handler to use for viewing notebooks accessed via URL").tag(config=True)
     local_handler       = Unicode(default_value="nbviewer.providers.local.handlers.LocalFileHandler",   help="The Tornado handler to use for viewing notebooks found on a local filesystem").tag(config=True)
     github_blob_handler = Unicode(default_value="nbviewer.providers.github.handlers.GitHubBlobHandler", help="The Tornado handler to use for viewing notebooks stored as blobs on GitHub").tag(config=True)
@@ -243,7 +246,16 @@ def template_paths(self):
 
     def init_tornado_application(self):
         # handle handlers
-        handlers = init_handlers(self.formats, options.providers, self.base_url, options.localfiles)
+        handler_names = dict(
+                  url_handler=self.url_handler,
+                  github_blob_handler=self.github_blob_handler,
+                  github_tree_handler=self.github_tree_handler,
+                  local_handler=self.local_handler,
+                  gist_handler=self.gist_handler,
+                  user_gists_handler=self.user_gists_handler,
+        )
+        handler_kwargs = {'handler_names' : handler_names, 'handler_settings' : self.handler_settings}
+        handlers = init_handlers(self.formats, options.providers, self.base_url, options.localfiles, **handler_kwargs)
 
         # NBConvert config
         self.config.NbconvertApp.fileext = 'html'

nbviewer/handlers.py

@@ -75,31 +75,53 @@ def get_provider_rewrites(self):
 # Default handler URL mapping
 #-----------------------------------------------------------------------------
 
-def format_handlers(formats, handlers):
-    return [
+def format_handlers(formats, urlspecs, **handler_settings):
+    """
+    Tornado handler URLSpec of form (route, handler_class, initalize_kwargs)
+    https://www.tornadoweb.org/en/stable/web.html#tornado.web.URLSpec
+    kwargs passed to initialize are None by default but can be added
+    https://www.tornadoweb.org/en/stable/web.html#tornado.web.RequestHandler.initialize
+    """
+    urlspecs = [
         (prefix + url, handler, {
             "format": format,
             "format_prefix": prefix
         })
         for format in formats
-        for url, handler in handlers
+        for url, handler, initialize_kwargs in urlspecs
         for prefix in [format_prefix + format]
     ]
+    for handler_setting in handler_settings:
+        if handler_settings[handler_setting]:
+            # here we modify the URLSpec dict to have the key-value pairs from
+            # handler_settings in NBViewer.init_tornado_application
+            for urlspec in urlspecs:
+                urlspec[2][handler_setting] = handler_settings[handler_setting]
+    return urlspecs
+
+def init_handlers(formats, providers, base_url, localfiles, **handler_kwargs):
+    """
+    `handler_kwargs` is a dict of dicts: first dict is `handler_names`, which
+    specifies the handler_classes to load for the providers, the second
+    is `handler_settings` (see comments in format_handlers)
+    Only `handler_settings` should get added to the initialize_kwargs in the
+    handler URLSpecs, which is why we pass only it to `format_handlers`
+    but both it and `handler_names` to `provider_handlers`
+    """
+    handler_settings = handler_kwargs['handler_settings']
 
-
-def init_handlers(formats, providers, base_url, localfiles):
     pre_providers = [
-        ('/?', IndexHandler),
-        ('/index.html', IndexHandler),
-        (r'/faq/?', FAQHandler),
-        (r'/create/?', CreateHandler),
+        ('/?', IndexHandler, {}),
+        ('/index.html', IndexHandler, {}),
+        (r'/faq/?', FAQHandler, {}),
+        (r'/create/?', CreateHandler, {}),
 
         # don't let super old browsers request data-uris
-        (r'.*/data:.*;base64,.*', Custom404),
+        (r'.*/data:.*;base64,.*', Custom404, {}),
     ]
 
     post_providers = [
-        (r'/(robots\.txt|favicon\.ico)', web.StaticFileHandler)
+        (r'/(robots\.txt|favicon\.ico)', web.StaticFileHandler, {})
     ]
 
     # Add localfile handlers if the option is set
@@ -108,12 +130,12 @@ def init_handlers(formats, providers, base_url, localfiles):
         # https://github.com/jupyter/nbviewer/pull/727#discussion_r144448440.
         providers.insert(0, 'nbviewer.providers.local')
 
-    handlers = provider_handlers(providers)
+    handlers = provider_handlers(providers, **handler_kwargs)
 
     raw_handlers = (
         pre_providers +
         handlers +
-        format_handlers(formats, handlers) +
+        format_handlers(formats, handlers, **handler_settings) +
         post_providers
     )
 
@@ -122,6 +144,6 @@ def init_handlers(formats, providers, base_url, localfiles):
         pattern = url_path_join(base_url, handler[0])
         new_handler = tuple([pattern] + list(handler[1:]))
         new_handlers.append(new_handler)
-    new_handlers.append((r'.*', Custom404))
+    new_handlers.append((r'.*', Custom404, {}))
 
     return new_handlers

nbviewer/providers/__init__.py

@@ -12,16 +12,32 @@
                     for prov in ['gist', 'github', 'dropbox', 'url']]
 
 
-def provider_handlers(providers):
+def provider_handlers(providers, **handler_kwargs):
     """Load tornado URL handlers from an ordered list of dotted-notation modules
        which contain a `default_handlers` function
 
        `default_handlers` should accept a list of handlers and returns an
        augmented list of handlers: this allows the addition of, for
        example, custom URLs which should be intercepted before being
        handed to the basic `url` handler
+
+      `handler_kwargs` is a dict of dicts: first dict is `handler_names`, which
+      specifies the handler_classes to load for the providers, the second
+      is `handler_settings` (see comments in `format_handlers` in nbviewer/handlers.py)
     """
-    return _load_provider_feature('default_handlers', providers)
+    handler_names = handler_kwargs['handler_names']
+    handler_settings = handler_kwargs['handler_settings']
+
+    urlspecs = _load_provider_feature('default_handlers', providers, **handler_names)
+    for handler_setting in handler_settings:
+        if handler_settings[handler_setting]:
+            # here we modify the URLSpec dict to have the key-value pairs from
+            # handler_settings in NBViewer.init_tornado_application 
+            # kwargs passed to initialize are None by default but can be added
+            # https://www.tornadoweb.org/en/stable/web.html#tornado.web.RequestHandler.initialize
+            for urlspec in urlspecs:
+                urlspec[2][handler_setting] = handler_settings[handler_setting]
+    return urlspecs
 
 
 def provider_uri_rewrites(providers):
@@ -35,17 +51,59 @@ def provider_uri_rewrites(providers):
     return _load_provider_feature('uri_rewrites', providers)
 
 
-def _load_provider_feature(feature, providers):
+def _load_provider_feature(feature, providers, **handler_names):
     """Load the named feature from an ordered list of dotted-notation modules
        which each implements the feature.
 
        The feature will be passed a list of feature implementations and must
        return that list, suitably modified.
+
+      `handler_names` is the same as the `handler_names` attribute of the NBViewer class
     """
+
+    # Ex: provider = 'nbviewer.providers.url' 
+    # provider.rsplit(',', 1) = ['nbviewer.providers', 'url']
+    # provider_type = 'url'
+    provider_types = [provider.rsplit('.', 1)[-1] for provider in providers]
+
+    if 'github' in provider_types:
+        provider_types.append('github_blob')
+        provider_types.append('github_tree')
+        provider_types.remove('github')
+
+    provider_handlers = {}
+
+    # Ex: provider_type = 'url'
+    for provider_type in provider_types:
+        # Ex: provider_handler_key = 'url_handler'
+        provider_handler_key = provider_type + '_handler'
+        try:
+            # Ex: handler_names['url_handler']
+            handler_names[provider_handler_key]
+        except KeyError as e:
+            continue
+        else:
+            # Ex: provider_handlers['url_handler'] = handler_names['url_handler']
+            provider_handlers[provider_handler_key] = handler_names[provider_handler_key]
+
     features = []
 
+    # Ex: provider = 'nbviewer.providers.url'
     for provider in providers:
-        mod = __import__(provider, fromlist=[feature])
-        features = getattr(mod, feature)(features)
+        # Ex: module = __import__('nbviewer.providers.url', fromlist=['default_handlers'])
+        module = __import__(provider, fromlist=[feature])
+        # Ex: getattr(module, 'default_handlers') = the `default_handlers` function from 
+        # nbviewer.providers.url (in handlers.py of nbviewer/providers/url)
+        # so in example, features = nbviewer.providers.url.default_handlers(list_of_already_loaded_handlers, **handler_names)
+        # => features = list_of_already_loaded_handlers + [URLSpec of chosen URL handler]
+        features = getattr(module, feature)(features, **handler_names)
+    return features 
+
+def _load_handler_from_location(handler_location):
+    # Ex: handler_location = 'nbviewer.providers.url.URLHandler'
+    # module_name = 'nbviewer.providers.url', handler_name = 'URLHandler'
+    module_name, handler_name = tuple(handler_location.rsplit('.', 1))
 
-    return features
+    module = __import__(module_name, fromlist=[handler_name])
+    handler = getattr(module, handler_name)
+    return handler

nbviewer/providers/base.py

@@ -57,11 +57,14 @@ class CurlError(Exception): pass
 class BaseHandler(web.RequestHandler):
     """Base Handler class with common utilities"""
 
-    def initialize(self, format=None, format_prefix=""):
+    def initialize(self, format=None, format_prefix="", **handler_settings):
         self.format = format or self.default_format
         self.format_prefix = format_prefix
         self.http_client = httpclient.AsyncHTTPClient()
 
+        for handler_setting in handler_settings:
+            setattr(self, handler_setting, handler_settings[handler_setting])
+
     # Overloaded methods
     def redirect(self, url, *args, **kwargs):
         purl = urlparse(url)

nbviewer/providers/gist/handlers.py

@@ -21,6 +21,7 @@
 
 from ..github.handlers import GithubClientMixin
 
+from .. import _load_handler_from_location
 
 PROVIDER_CTX = {
     'provider_label': 'Gist',
@@ -207,15 +208,18 @@ def get(self, gist_id, file=''):
         self.redirect(self.from_base(new_url))
 
 
-def default_handlers(handlers=[]):
+def default_handlers(handlers=[], **handler_names):
     """Tornado handlers"""
 
+    gist_handler = _load_handler_from_location(handler_names['gist_handler'])
+    user_gists_handler = _load_handler_from_location(handler_names['user_gists_handler'])
+
     return handlers + [
-        (r'/gist/([^\/]+/)?([0-9]+|[0-9a-f]{20,})', GistHandler),
-        (r'/gist/([^\/]+/)?([0-9]+|[0-9a-f]{20,})/(?:files/)?(.*)', GistHandler),
-        (r'/([0-9]+|[0-9a-f]{20,})', GistRedirectHandler),
-        (r'/([0-9]+|[0-9a-f]{20,})/(.*)', GistRedirectHandler),
-        (r'/gist/([^\/]+)/?', UserGistsHandler),
+        (r'/gist/([^\/]+/)?([0-9]+|[0-9a-f]{20,})', gist_handler, {}),
+        (r'/gist/([^\/]+/)?([0-9]+|[0-9a-f]{20,})/(?:files/)?(.*)', gist_handler, {}),
+        (r'/([0-9]+|[0-9a-f]{20,})', GistRedirectHandler, {}),
+        (r'/([0-9]+|[0-9a-f]{20,})/(.*)', GistRedirectHandler, {}),
+        (r'/gist/([^\/]+)/?', user_gists_handler, {}),
     ]
 
 

nbviewer/providers/github/handlers.py

@@ -33,6 +33,7 @@
 
 from .client import AsyncGitHubClient
 
+from .. import _load_handler_from_location
 
 PROVIDER_CTX = {
     'provider_label': 'GitHub',
@@ -334,27 +335,31 @@ def get(self, user, repo, ref, path):
             self.cache_and_finish(filedata)
 
 
-def default_handlers(handlers=[]):
+def default_handlers(handlers=[], **handler_names):
     """Tornado handlers"""
 
+    blob_handler = _load_handler_from_location(handler_names['github_blob_handler'])
+    tree_handler = _load_handler_from_location(handler_names['github_tree_handler'])
+
     return [
         # ideally these URIs should have been caught by an appropriate
         # uri_rewrite rather than letting the url provider catch them and then
         # fixing it here.
         # There are probably links in the wild that depend on these, so keep
         # these handlers for backwards compatibility.
-        (r'/url[s]?/github\.com/(?P<url>.*)', GitHubRedirectHandler),
-        (r'/url[s]?/raw\.?github\.com/(?P<user>[^\/]+)/(?P<repo>[^\/]+)/(?P<path>.*)', RawGitHubURLHandler),
-        (r'/url[s]?/raw\.?githubusercontent\.com/(?P<user>[^\/]+)/(?P<repo>[^\/]+)/(?P<path>.*)', RawGitHubURLHandler),
+        (r'/url[s]?/github\.com/(?P<url>.*)', GitHubRedirectHandler, {}),
+        (r'/url[s]?/raw\.?github\.com/(?P<user>[^\/]+)/(?P<repo>[^\/]+)/(?P<path>.*)', RawGitHubURLHandler, {}),
+        (r'/url[s]?/raw\.?githubusercontent\.com/(?P<user>[^\/]+)/(?P<repo>[^\/]+)/(?P<path>.*)', RawGitHubURLHandler, {}),
     ] + handlers + [
-        (r'/github/([^\/]+)', AddSlashHandler),
-        (r'/github/(?P<user>[^\/]+)/', GitHubUserHandler),
-        (r'/github/([^\/]+)/([^\/]+)', AddSlashHandler),
-        (r'/github/(?P<user>[^\/]+)/(?P<repo>[^\/]+)/', GitHubRepoHandler),
-        (r'/github/([^\/]+)/([^\/]+)/(?:blob|raw)/([^\/]+)/(.*)/', RemoveSlashHandler),
-        (r'/github/(?P<user>[^\/]+)/(?P<repo>[^\/]+)/(?:blob|raw)/(?P<ref>[^\/]+)/(?P<path>.*)', GitHubBlobHandler),
-        (r'/github/([^\/]+)/([^\/]+)/tree/([^\/]+)', AddSlashHandler),
-        (r'/github/(?P<user>[^\/]+)/(?P<repo>[^\/]+)/tree/(?P<ref>[^\/]+)/(?P<path>.*)', GitHubTreeHandler),
+        (r'/github/([^\/]+)', AddSlashHandler, {}),
+        (r'/github/(?P<user>[^\/]+)/', GitHubUserHandler, {}),
+        (r'/github/([^\/]+)/([^\/]+)', AddSlashHandler, {}),
+        (r'/github/(?P<user>[^\/]+)/(?P<repo>[^\/]+)/', GitHubRepoHandler, {}),
+        (r'/github/([^\/]+)/([^\/]+)/(?:blob|raw)/([^\/]+)/(.*)/', RemoveSlashHandler, {}),
+        (r'/github/([^\/]+)/([^\/]+)/tree/([^\/]+)', AddSlashHandler, {}), 
+    ] + [
+        (r'/github/(?P<user>[^\/]+)/(?P<repo>[^\/]+)/tree/(?P<ref>[^\/]+)/(?P<path>.*)', tree_handler, {}),
+        (r'/github/(?P<user>[^\/]+)/(?P<repo>[^\/]+)/(?:blob|raw)/(?P<ref>[^\/]+)/(?P<path>.*)', blob_handler, {}),
     ]
 
 

nbviewer/providers/local/handlers.py

@@ -24,6 +24,7 @@
     RenderingHandler,
 )
 
+from .. import _load_handler_from_location
 
 class LocalFileHandler(RenderingHandler):
     """Renderer for /localfile
@@ -257,10 +258,12 @@ def show_dir(self, fullpath, path):
         return html
 
 
-def default_handlers(handlers=[]):
+def default_handlers(handlers=[], **handler_names):
     """Tornado handlers"""
 
+    local_handler = _load_handler_from_location(handler_names['local_handler'])
+
     return handlers + [
-        (r'/localfile/?(.*)', LocalFileHandler),
+        (r'/localfile/?(.*)', local_handler, {}),
     ]
 

nbviewer/providers/url/handlers.py

@@ -26,6 +26,7 @@
     RenderingHandler,
 )
 
+from .. import _load_handler_from_location
 
 class URLHandler(RenderingHandler):
     """Renderer for /url or /urls"""
@@ -89,11 +90,13 @@ def get(self, secure, netloc, url):
                                    format=self.format)
 
 
-def default_handlers(handlers=[]):
+def default_handlers(handlers=[], **handler_names):
     """Tornado handlers"""
 
+    url_handler = _load_handler_from_location(handler_names['url_handler'])
+
     return handlers + [
-        (r'/url(?P<secure>[s]?)/(?P<netloc>[^/]+)/(?P<url>.*)', URLHandler),
+        (r'/url(?P<secure>[s]?)/(?P<netloc>[^/]+)/(?P<url>.*)', url_handler, {}),
     ]