Merge pull request #9 from nteract/config-for-all

setup config for use in archiver

Author: rgbkrk

README.md

@@ -47,19 +47,16 @@ from bookstore import BookstoreContentsArchiver
 
 c.NotebookApp.contents_manager_class = BookstoreContentsArchiver
 
-c.Bookstore.workspace_prefix = "/workspace/kylek/notebooks"
-c.Bookstore.published_prefix = "/published/kylek/notebooks"
+c.BookstoreSettings.workspace_prefix = "/workspace/kylek/notebooks"
+c.BookstoreSettings.published_prefix = "/published/kylek/notebooks"
 
 # Optional, in case you're using a different contents manager
 # This defaults to notebook.services.contents.manager.ContentsManager
-# c.bookstore.Archiver.underlying_contents_manager_class = ADifferentContentsManager
 
-c.Bookstore.storage_class = BookstoreS3Settings
-
-c.BookstoreS3Settings.bucket = "<bucket-name>"
+c.BookstoreSettings.s3_bucket = "<bucket-name>"
 
 # Note: if bookstore is used from an EC2 instance with the right IAM role, you don't
 # have to specify these
-c.BookstoreS3Settings.access_key_id = <AWS Access Key ID / IAM Access Key ID>
-c.BookstoreS3Settings.secret_access_key = <AWS Secret Access Key / IAM Secret Access Key>
+c.BookstoreSettings.s3_access_key_id = <AWS Access Key ID / IAM Access Key ID>
+c.BookstoreSettings.s3_secret_access_key = <AWS Secret Access Key / IAM Secret Access Key>
 ```

bookstore/__init__.py

@@ -6,3 +6,5 @@
 from .jupyter_server_extension import load_jupyter_server_extension, _jupyter_server_extension_paths
 
 from .archive import BookstoreContentsArchiver
+
+from .bookstore_config import BookstoreSettings

bookstore/archive.py

@@ -1,7 +1,9 @@
-from s3contents.ipycompat import ContentsManager
-from s3contents.ipycompat import HasTraits, Unicode
+from notebook.services.contents.filemanager import FileContentsManager
+
+import s3fs
 
 from traitlets import (
+    HasTraits,
     Any,
     Bool,
     Dict,
@@ -12,16 +14,59 @@
     Unicode,
     validate,
     default,
+    Instance
 )
 
-class BookstoreContentsArchiver(ContentsManager, HasTraits):
+import json
+
+from .bookstore_config import BookstoreSettings
+
+class BookstoreContentsArchiver(FileContentsManager):
+    """
+    Archives notebooks to S3 on save
     """
-    Archives contents via one ContentsManager and passes through to
-    another ContentsManager.
 
-    Likely route:
+    def __init__(self, *args, **kwargs):
+        super(FileContentsManager, self).__init__(*args, **kwargs)
+        # opt ourselves into being part of the Jupyter App that should have Bookstore Settings applied
+        self.settings = BookstoreSettings(parent=self)
 
-    * Write directly to S3 on post_save_hook
+        self.fs = s3fs.S3FileSystem(key=self.settings.s3_access_key_id,
+                                    secret=self.settings.s3_secret_access_key,
+                                    client_kwargs={
+                                        "endpoint_url": self.settings.s3_endpoint_url,
+                                        "region_name": self.settings.s3_region_name
+                                    },
+                                    config_kwargs={},
+                                    s3_additional_kwargs={})
 
-    """
-    pass
+    @property
+    def delimiter(self):
+        """It's a slash! Normally this could be configurable. This leaves room for that later,
+        keeping it centralized for now"""
+        return "/"
+
+    @property
+    def full_prefix(self):
+        """Full prefix: bucket + workspace prefix"""
+        return self.delimiter.join([self.settings.s3_bucket, self.settings.workspace_prefix])
+
+    def s3_path(self, path):
+        """compute the s3 path based on the bucket, prefix, and the path to the notebook"""
+        return self.delimiter.join([self.full_prefix, path])
+
+    def run_pre_save_hook(self, model, path, **kwargs):
+        """Store notebook to S3 when saves happen
+        """
+        if model['type'] != 'notebook':
+            return
+
+        # TODO: store the hash of the notebook to not write on every save
+        notebook_contents = json.dumps(model['content'])
+
+        full_path = self.s3_path(path)
+
+        # write to S3
+        # TODO: Write to S3 asynchronously to not block other server operations
+        with self.fs.open(full_path, mode='wb') as f:
+            f.write(notebook_contents.encode('utf-8'))

bookstore/bookstore_config.py

@@ -13,26 +13,28 @@
 
 from traitlets.config import LoggingConfigurable
 
-class BookstoreS3Settings(LoggingConfigurable):
+class BookstoreSettings(LoggingConfigurable):
+    """The same settings to be shared across archival, publishing, and scheduling
+    """
+
+    workspace_prefix = Unicode("workspace", help="Prefix for the live workspace notebooks").tag(config=True)
+    published_prefix = Unicode("published", help="Prefix for published notebooks").tag(config=True)
+
+    ## S3 Settings for the S3 backed storage (other implementations can add on below)
     # Allowed to not set these as we can pick up IAM roles instead
-    access_key_id = Unicode(
+    s3_access_key_id = Unicode(
         help="S3/AWS access key ID", allow_none=True, default_value=None).tag(
             config=True, env="JPYNB_S3_ACCESS_KEY_ID")
-    secret_access_key = Unicode(
+    s3_secret_access_key = Unicode(
         help="S3/AWS secret access key", allow_none=True, default_value=None).tag(
             config=True, env="JPYNB_S3_SECRET_ACCESS_KEY")
 
-    endpoint_url = Unicode(
+    s3_endpoint_url = Unicode(
         "https://s3.amazonaws.com", help="S3 endpoint URL").tag(
             config=True, env="JPYNB_S3_ENDPOINT_URL")
-    region_name = Unicode(
+    s3_region_name = Unicode(
         "us-east-1", help="Region name").tag(
             config=True, env="JPYNB_S3_REGION_NAME")
-    bucket = Unicode(
-        "notebooks", help="Bucket name to store notebooks").tag(
+    s3_bucket = Unicode(
+        "bookstore", help="Bucket name to store notebooks").tag(
             config=True, env="JPYNB_S3_BUCKET")
-
-
-class Bookstore(LoggingConfigurable):
-    workspace_prefix = Unicode("workspace", help="Prefix for the live workspace notebooks").tag(config=True)
-    published_prefix = Unicode("published", help="Prefix for published notebooks").tag(config=True)

jupyter_notebook_config.py.example

@@ -3,8 +3,19 @@
 
 print("Welcome to the bookstore 📚")
 
-from bookstore import BookstoreContentsArchiver
+from bookstore import BookstoreContentsArchiver, BookstoreSettings
 
+# jupyter config
+# At ~/.jupyter/jupyter_notebook_config.py for user installs
+# At __ for system installs
 c = get_config()
 
 c.NotebookApp.contents_manager_class = BookstoreContentsArchiver
+
+c.BookstoreSettings.workspace_prefix = "works"
+
+# If using minio for development
+c.BookstoreSettings.s3_endpoint_url = "http://127.0.0.1:9000"
+c.BookstoreSettings.s3_bucket = "bookstore"
+c.BookstoreSettings.s3_access_key_id = "4MR9ON7H4UNVCT2LQTFX"
+c.BookstoreSettings.s3_secret_access_key = "o8CnAN5G9x87P9aLxSjohoSV0EsCLjksuY6wjK9N"

requirements.txt

@@ -1,4 +1,5 @@
 future
 futures ; python_version < "3.0"
 ipython >= 5.0
-s3contents
+notebook
+s3contents