run integration testing against minio

Closes #12.

* [x] Set up minio in a container on CircleCI
* [x] Create a packaged release for testing
* [x] Install packaged release into a virtualenv (full end-to-end integration test)
* [x] Fix bug in handlers registration (this can be moved to another PR if requested)
* [x] Write integration test that launches jupyter with bookstore enabled and tests that the notebook was written to disk and "s3" (minio) properly

Author: rgbkrk

.circleci/config.yml

@@ -6,14 +6,15 @@ version: 2
 jobs:
   build:
     docker:
-      # specify the version you desire here
-      # use `-browsers` prefix for selenium tests, e.g. `3.6.1-browsers`
-      - image: circleci/python:3.6.1
+      - image: circleci/python:3.6.7-node-browsers
+        ports: 9988:9988
 
-      # Specify service dependencies here if necessary
-      # CircleCI maintains a library of pre-built images
-      # documented at https://circleci.com/docs/2.0/circleci-images/
-      # - image: circleci/postgres:9.4
+      - image: minio/minio:RELEASE.2018-11-06T01-01-02Z
+        command: server /data
+        environment:
+          MINIO_ACCESS_KEY: ONLY_ON_CIRCLE
+          MINIO_SECRET_KEY: CAN_WE_DO_THIS
+        ports: 9000:9000
 
     working_directory: ~/repo
 
@@ -23,19 +24,41 @@ jobs:
       # Download and cache dependencies
       - restore_cache:
           keys:
-          - v1-dependencies-{{ checksum "requirements.txt" }}
+          - v2-dependencies-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt"}}
           # fallback to using the latest cache if no exact match is found
-          - v1-dependencies-
+          - v2-dependencies-
 
       - run:
           name: install dependencies
           command: |
             python3 -m venv venv
             . venv/bin/activate
+            pip install --upgrade pip setuptools wheel
             pip install -r requirements.txt
             pip install -r requirements-dev.txt
 
       - save_cache:
           paths:
             - ./venv
-          key: v1-dependencies-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }}
+          key: v2-dependencies-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }}
+
+      - run:
+          name: package up bookstore
+          command: |
+            . venv/bin/activate
+            # Package up the package
+            python setup.py sdist bdist_wheel
+      - run:
+          name: create virtual environment for packaged release
+          command: |
+            python3 -m venv venv_packaged_integration
+            . venv_packaged_integration/bin/activate
+            pip install --upgrade pip setuptools wheel
+            pip install -U --force-reinstall dist/bookstore*.whl
+      - run:
+          name: integration tests
+          command: |
+            . venv_packaged_integration/bin/activate
+            # Install the dependencies for our integration tester
+            npm i
+            node ci/integration.js

bookstore/__init__.py

@@ -3,7 +3,10 @@
 __version__ = get_versions()['version']
 del get_versions
 
-from .jupyter_server_extension import load_jupyter_server_extension, _jupyter_server_extension_paths
+from .handlers import load_jupyter_server_extension
+
+def _jupyter_server_extension_paths():
+    return [dict(module="bookstore")]
 
 from .archive import BookstoreContentsArchiver
 

bookstore/archive.py

@@ -31,6 +31,8 @@ def __init__(self, *args, **kwargs):
         # opt ourselves into being part of the Jupyter App that should have Bookstore Settings applied
         self.settings = BookstoreSettings(parent=self)
 
+        self.log.info("Archiving notebooks to {}".format(self.full_prefix))
+
         self.fs = s3fs.S3FileSystem(key=self.settings.s3_access_key_id,
                                     secret=self.settings.s3_secret_access_key,
                                     client_kwargs={

…ore/jupyter_server_extension/handlers.py bookstore/handlers.pybookstore/jupyter_server_extension/handlers.py renamed to bookstore/handlers.py bookstore/jupyter_server_extension/handlers.py renamed to bookstore/handlers.py

@@ -4,7 +4,7 @@
     AuthenticatedFileHandler
 from tornado import web
 
-from .._version import get_versions
+from ._version import get_versions
 version = get_versions()['version']
 
 class BookstoreVersionHandler(APIHandler):

bookstore/jupyter_server_extension/__init__.py

@@ -0,0 +1,236 @@
+const child_process = require("child_process");
+const { randomBytes } = require("crypto");
+const path = require("path");
+const fs = require("fs");
+
+global.XMLHttpRequest = require("xmlhttprequest").XMLHttpRequest;
+
+const { ajax } = require("rxjs/ajax");
+
+const rxJupyter = require("rx-jupyter");
+
+const _ = require("lodash");
+
+console.log("running bookstore integration tests");
+
+async function genToken(byteLength = 32) {
+  return new Promise((resolve, reject) => {
+    randomBytes(byteLength, (err, buffer) => {
+      if (err) {
+        reject(err);
+        return;
+      }
+
+      resolve(buffer.toString("hex"));
+    });
+  });
+}
+
+const sleep = timeout =>
+  new Promise((resolve, reject) => setTimeout(resolve, timeout));
+
+// Catch all rogue promise rejections to fail CI
+process.on("unhandledRejection", error => {
+  console.log("unhandledRejection", error.message);
+  process.exit(2);
+});
+
+var Minio = require("minio");
+
+const main = async () => {
+  const jupyterToken = await genToken();
+  const jupyterPort = 9988;
+  const jupyterEndpoint = `http://127.0.0.1:${jupyterPort}`;
+
+  const bucketName = "bookstore";
+  // Optional according to minio docs, likely here for AWS compat
+  const regionName = "us-east-1";
+
+  const s3Config = {
+    endPoint: "127.0.0.1",
+    port: 9000,
+    useSSL: false,
+    accessKey: "ONLY_ON_CIRCLE",
+    secretKey: "CAN_WE_DO_THIS"
+  };
+
+  // Instantiate the minio client with the endpoint
+  // and access keys as shown below.
+  var minioClient = new Minio.Client(s3Config);
+
+  const madeBucket = await new Promise((resolve, reject) => {
+    minioClient.makeBucket(bucketName, regionName, err => {
+      if (err) {
+        reject(err);
+        return;
+      }
+      resolve();
+    });
+  });
+  console.log(`Created bucket ${bucketName}`);
+
+  const jupyter = child_process.spawn(
+    "jupyter",
+    [
+      "notebook",
+      "--no-browser",
+      `--NotebookApp.token=${jupyterToken}`,
+      `--NotebookApp.disable_check_xsrf=True`,
+      `--port=${jupyterPort}`,
+      `--ip=127.0.0.1`
+    ],
+    { cwd: __dirname }
+  );
+
+  ////// Refactor me later, streams are a bit messy with async await
+  // Check to see that jupyter is up
+  let jupyterUp = false;
+
+  jupyter.stdout.on("data", data => {
+    const s = data.toString();
+    console.log(s);
+  });
+  jupyter.stderr.on("data", data => {
+    const s = data.toString();
+
+    console.error(s);
+    if (s.includes("Jupyter Notebook is running at")) {
+      jupyterUp = true;
+    }
+  });
+  jupyter.stdout.on("end", data => console.log("DONE WITH JUPYTER"));
+
+  jupyter.on("exit", code => {
+    if (code != 0) {
+      // Jupyter exited badly
+      console.error("jupyter errored", code);
+      process.exit(code);
+    }
+  });
+
+  await sleep(3000);
+
+  if (!jupyterUp) {
+    console.log("jupyter has not come up after 3 seconds, waiting 3 more");
+    await sleep(3000);
+
+    if (!jupyterUp) {
+      console.log("jupyter has not come up after 6 seconds, bailing");
+      process.exit(1);
+    }
+  }
+
+  const originalNotebook = {
+    cells: [
+      {
+        cell_type: "code",
+        execution_count: null,
+        metadata: {},
+        outputs: [],
+        source: ["import this"]
+      }
+    ],
+    metadata: {
+      kernelspec: {
+        display_name: "Python 3",
+        language: "python",
+        name: "python3"
+      },
+      language_info: {
+        codemirror_mode: {
+          name: "ipython",
+          version: 3
+        },
+        file_extension: ".py",
+        mimetype: "text/x-python",
+        name: "python",
+        nbconvert_exporter: "python",
+        pygments_lexer: "ipython3",
+        version: "3.7.0"
+      }
+    },
+    nbformat: 4,
+    nbformat_minor: 2
+  };
+
+  const xhr = await ajax({
+    url: `${jupyterEndpoint}/api/contents/ci-local-writeout.ipynb`,
+    responseType: "json",
+    createXHR: () => new XMLHttpRequest(),
+    method: "PUT",
+    body: {
+      type: "notebook",
+      content: originalNotebook
+    },
+    headers: {
+      "Content-Type": "application/json",
+      Authorization: `token ${jupyterToken}`
+    }
+  }).toPromise();
+
+  // Wait for minio to have the notebook
+  // Future iterations of this script should poll to get the notebook
+  await sleep(1000);
+
+  jupyter.kill();
+
+  //// Check the notebook we placed on S3
+  const rawNotebook = await new Promise((resolve, reject) =>
+    minioClient.getObject(
+      bucketName,
+      "ci-workspace/ci-local-writeout.ipynb",
+      (err, dataStream) => {
+        if (err) {
+          console.error("wat");
+          reject(err);
+          return;
+        }
+
+        const chunks = [];
+        dataStream.on("data", chunk => chunks.push(chunk));
+        dataStream.on("error", reject);
+        dataStream.on("end", () => {
+          resolve(Buffer.concat(chunks).toString("utf8"));
+        });
+      }
+    )
+  );
+
+  const notebook = JSON.parse(rawNotebook);
+
+  if (!_.isEqual(notebook, originalNotebook)) {
+    console.error("original");
+    console.error(originalNotebook);
+    console.error("from s3");
+    console.error(notebook);
+    throw new Error("Notebook on S3 does not match what we sent");
+  }
+
+  console.log("Notebook on S3 matches what we sent");
+
+  //// Check the notebook we placed on Disk
+  const diskNotebook = await new Promise((resolve, reject) =>
+    fs.readFile(
+      path.join(__dirname, "ci-local-writeout.ipynb"),
+      (err, data) => {
+        if (err) {
+          reject(err);
+        } else {
+          resolve(JSON.parse(data));
+        }
+      }
+    )
+  );
+
+  if (!_.isEqual(diskNotebook, originalNotebook)) {
+    console.error("original");
+    console.error(originalNotebook);
+    console.error("from disk");
+    console.error(diskNotebook);
+    throw new Error("Notebook on Disk does not match what we sent");
+  }
+
+  console.log("📚 Bookstore Integration Complete 📚");
+};
+
+main();

ci/integration.js

@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from bookstore import BookstoreContentsArchiver, BookstoreSettings
+
+# jupyter config
+# At ~/.jupyter/jupyter_notebook_config.py for user installs
+# At __ for system installs
+c = get_config()
+
+c.NotebookApp.contents_manager_class = BookstoreContentsArchiver
+
+c.BookstoreSettings.workspace_prefix = "ci-workspace"
+
+# If using minio for development
+c.BookstoreSettings.s3_endpoint_url = "http://127.0.0.1:9000"
+c.BookstoreSettings.s3_bucket = "bookstore"
+
+# Straight out of `circleci/config.yml`
+c.BookstoreSettings.s3_access_key_id = "ONLY_ON_CIRCLE"
+c.BookstoreSettings.s3_secret_access_key = "CAN_WE_DO_THIS"

ci/jupyter_notebook_config.py

@@ -0,0 +1,32 @@
+{
+  "name": "bookstore",
+  "version": "1.0.0",
+  "description": "[![Documentation Status](https://readthedocs.org/projects/bookstore/badge/?version=latest)](https://bookstore.readthedocs.io/en/latest/?badge=latest)",
+  "main": "index.js",
+  "directories": {
+    "doc": "docs"
+  },
+  "private": "true",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/nteract/bookstore.git"
+  },
+  "author": "Kyle Kelley <rgbkrk@gmail.com>",
+  "license": "BSD-3-Clause",
+  "bugs": {
+    "url": "https://github.com/nteract/bookstore/issues"
+  },
+  "homepage": "https://github.com/nteract/bookstore#readme",
+  "dependencies": {
+    "lodash": "^4.17.11",
+    "minio": "^7.0.1",
+    "rx-jupyter": "^4.0.0-alpha.0",
+    "rxjs": "^6.3.3",
+    "spawn-rx": "^3.0.0",
+    "xhr2": "^0.1.4",
+    "xmlhttprequest": "^1.8.0"
+  }
+}