Merge pull request #11 from nteract/run-minio-on-circleci

set up minio on circle

Author: rgbkrk

.circleci/config.yml

@@ -6,14 +6,15 @@ version: 2
 jobs:
   build:
     docker:
-      # specify the version you desire here
-      # use `-browsers` prefix for selenium tests, e.g. `3.6.1-browsers`
-      - image: circleci/python:3.6.1
+      - image: circleci/python:3.6.7-node-browsers
+        ports: 9988:9988
 
-      # Specify service dependencies here if necessary
-      # CircleCI maintains a library of pre-built images
-      # documented at https://circleci.com/docs/2.0/circleci-images/
-      # - image: circleci/postgres:9.4
+      - image: minio/minio:RELEASE.2018-11-06T01-01-02Z
+        command: server /data
+        environment:
+          MINIO_ACCESS_KEY: ONLY_ON_CIRCLE
+          MINIO_SECRET_KEY: CAN_WE_DO_THIS
+        ports: 9000:9000
 
     working_directory: ~/repo
 
@@ -23,19 +24,41 @@ jobs:
       # Download and cache dependencies
       - restore_cache:
           keys:
-          - v1-dependencies-{{ checksum "requirements.txt" }}
+          - v2-dependencies-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt"}}
           # fallback to using the latest cache if no exact match is found
-          - v1-dependencies-
+          - v2-dependencies-
 
       - run:
           name: install dependencies
           command: |
             python3 -m venv venv
             . venv/bin/activate
+            pip install --upgrade pip setuptools wheel
             pip install -r requirements.txt
             pip install -r requirements-dev.txt
 
       - save_cache:
           paths:
             - ./venv
-          key: v1-dependencies-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }}
+          key: v2-dependencies-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }}
+
+      - run:
+          name: package up bookstore
+          command: |
+            . venv/bin/activate
+            # Package up the package
+            python setup.py sdist bdist_wheel
+      - run:
+          name: create virtual environment for packaged release
+          command: |
+            python3 -m venv venv_packaged_integration
+            . venv_packaged_integration/bin/activate
+            pip install --upgrade pip setuptools wheel
+            pip install -U --force-reinstall dist/bookstore*.whl
+      - run:
+          name: integration tests
+          command: |
+            . venv_packaged_integration/bin/activate
+            # Install the dependencies for our integration tester
+            npm i
+            node ci/integration.js

bookstore/__init__.py

@@ -3,7 +3,10 @@
 __version__ = get_versions()['version']
 del get_versions
 
-from .jupyter_server_extension import load_jupyter_server_extension, _jupyter_server_extension_paths
+from .handlers import load_jupyter_server_extension
+
+def _jupyter_server_extension_paths():
+    return [dict(module="bookstore")]
 
 from .archive import BookstoreContentsArchiver
 

bookstore/archive.py

@@ -31,6 +31,8 @@ def __init__(self, *args, **kwargs):
         # opt ourselves into being part of the Jupyter App that should have Bookstore Settings applied
         self.settings = BookstoreSettings(parent=self)
 
+        self.log.info("Archiving notebooks to {}".format(self.full_prefix))
+
         self.fs = s3fs.S3FileSystem(key=self.settings.s3_access_key_id,
                                     secret=self.settings.s3_secret_access_key,
                                     client_kwargs={

…ore/jupyter_server_extension/handlers.py bookstore/handlers.pybookstore/jupyter_server_extension/handlers.py renamed to bookstore/handlers.py bookstore/jupyter_server_extension/handlers.py renamed to bookstore/handlers.py

@@ -4,7 +4,7 @@
     AuthenticatedFileHandler
 from tornado import web
 
-from .._version import get_versions
+from ._version import get_versions
 version = get_versions()['version']
 
 class BookstoreVersionHandler(APIHandler):

bookstore/jupyter_server_extension/__init__.py

@@ -0,0 +1,126 @@
+const path = require("path");
+const fs = require("fs");
+
+const _ = require("lodash");
+
+const s3 = require("./s3");
+const { JupyterServer } = require("./jupyter");
+
+const { sleep } = require("./sleep");
+
+// Catch all rogue promise rejections to fail CI
+process.on("unhandledRejection", error => {
+  console.log("unhandledRejection", error.message);
+  console.error(error.stack);
+  process.exit(2);
+});
+
+console.log("running bookstore integration tests");
+
+const main = async () => {
+  const bucketName = "bookstore";
+
+  const jupyterServer = new JupyterServer();
+  await jupyterServer.start();
+
+  const s3Config = {
+    endPoint: "127.0.0.1",
+    port: 9000,
+    useSSL: false,
+    accessKey: "ONLY_ON_CIRCLE",
+    secretKey: "CAN_WE_DO_THIS"
+  };
+
+  // Instantiate the minio client with the endpoint
+  // and access keys as shown below.
+  var s3Client = new s3.Client(s3Config);
+
+  await s3Client.makeBucket(bucketName);
+
+  console.log(`Created bucket ${bucketName}`);
+
+  const originalNotebook = {
+    cells: [
+      {
+        cell_type: "code",
+        execution_count: null,
+        metadata: {},
+        outputs: [],
+        source: ["import this"]
+      }
+    ],
+    metadata: {
+      kernelspec: {
+        display_name: "Python 3",
+        language: "python",
+        name: "python3"
+      },
+      language_info: {
+        codemirror_mode: {
+          name: "ipython",
+          version: 3
+        },
+        file_extension: ".py",
+        mimetype: "text/x-python",
+        name: "python",
+        nbconvert_exporter: "python",
+        pygments_lexer: "ipython3",
+        version: "3.7.0"
+      }
+    },
+    nbformat: 4,
+    nbformat_minor: 2
+  };
+
+  jupyterServer.writeNotebook("ci-local-writeout.ipynb", originalNotebook);
+
+  // Wait for minio to have the notebook
+  // Future iterations of this script should poll to get the notebook
+  await sleep(1000);
+
+  jupyterServer.shutdown();
+
+  /***** Check notebook from S3 *****/
+  const rawNotebook = await s3Client.getObject(
+    bucketName,
+    "ci-workspace/ci-local-writeout.ipynb"
+  );
+
+  const notebook = JSON.parse(rawNotebook);
+
+  if (!_.isEqual(notebook, originalNotebook)) {
+    console.error("original");
+    console.error(originalNotebook);
+    console.error("from s3");
+    console.error(notebook);
+    throw new Error("Notebook on S3 does not match what we sent");
+  }
+
+  console.log("Notebook on S3 matches what we sent");
+
+  /***** Check notebook from Disk *****/
+  const diskNotebook = await new Promise((resolve, reject) =>
+    fs.readFile(
+      path.join(__dirname, "ci-local-writeout.ipynb"),
+      (err, data) => {
+        if (err) {
+          reject(err);
+        } else {
+          resolve(JSON.parse(data));
+        }
+      }
+    )
+  );
+
+  if (!_.isEqual(diskNotebook, originalNotebook)) {
+    console.error("original");
+    console.error(originalNotebook);
+    console.error("from disk");
+    console.error(diskNotebook);
+    throw new Error("Notebook on Disk does not match what we sent");
+  }
+
+  console.log("📚 Bookstore Integration Complete 📚");
+};
+
+main();

ci/integration.js

@@ -0,0 +1,103 @@
+const child_process = require("child_process");
+const { genToken } = require("./token");
+const { sleep } = require("./sleep");
+
+// "Polyfill" XMLHttpRequest for rxjs' ajax to use
+global.XMLHttpRequest = require("xmlhttprequest").XMLHttpRequest;
+const { ajax } = require("rxjs/ajax");
+
+class JupyterServer {
+  constructor(config = {}) {
+    this.port = config.port || 9988;
+    this.ip = config.ip || "127.0.0.1";
+    this.scheme = config.scheme || "http";
+    this.token = null;
+
+    // Launch the server from the directory of this script by default
+    this.cwd = config.cwd || __dirname;
+
+    this.process = null;
+    this.up = false;
+  }
+
+  async start() {
+    if (!this.token) {
+      this.token = await genToken();
+    }
+
+    this.process = child_process.spawn(
+      "jupyter",
+      [
+        "notebook",
+        "--no-browser",
+        `--NotebookApp.token=${this.token}`,
+        `--NotebookApp.disable_check_xsrf=True`,
+        `--port=${this.port}`,
+        `--ip=${this.ip}`
+      ],
+      { cwd: this.cwd }
+    );
+
+    ////// Refactor me later, streams are a bit messy with async await
+    ////// Let's use spawn-rx in the future and make some clean rxjs with timeouts
+    this.process.stdout.on("data", data => {
+      const s = data.toString();
+      console.log(s);
+    });
+    this.process.stderr.on("data", data => {
+      const s = data.toString();
+
+      console.error(s);
+      if (s.includes("Jupyter Notebook is running at")) {
+        this.up = true;
+      }
+    });
+    this.process.stdout.on("end", data =>
+      console.log("jupyter server terminated")
+    );
+
+    await sleep(3000);
+
+    if (!this.up) {
+      console.log("jupyter has not come up after 3 seconds, waiting 3 more");
+      await sleep(3000);
+
+      if (!this.up) {
+        throw new Error("jupyter has not come up after 6 seconds, bailing");
+      }
+    }
+  }
+
+  async writeNotebook(path, notebook) {
+    // Once https://github.com/nteract/nteract/pull/3651 is merged, we can use
+    // rx-jupyter for writing a notebook to the contents API
+    const xhr = await ajax({
+      url: `${this.endpoint}/api/contents/${path}`,
+      responseType: "json",
+      createXHR: () => new XMLHttpRequest(),
+      method: "PUT",
+      body: {
+        type: "notebook",
+        content: notebook
+      },
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `token ${this.token}`
+      }
+    }).toPromise();
+
+    return xhr;
+  }
+
+  shutdown() {
+    this.process.kill();
+  }
+
+  get endpoint() {
+    return `${this.scheme}://${this.ip}:${this.port}`;
+  }
+}
+
+module.exports = {
+  JupyterServer
+};

ci/jupyter.js

@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from bookstore import BookstoreContentsArchiver, BookstoreSettings
+
+# jupyter config
+# At ~/.jupyter/jupyter_notebook_config.py for user installs
+# At __ for system installs
+c = get_config()
+
+c.NotebookApp.contents_manager_class = BookstoreContentsArchiver
+
+c.BookstoreSettings.workspace_prefix = "ci-workspace"
+
+# If using minio for development
+c.BookstoreSettings.s3_endpoint_url = "http://127.0.0.1:9000"
+c.BookstoreSettings.s3_bucket = "bookstore"
+
+# Straight out of `circleci/config.yml`
+c.BookstoreSettings.s3_access_key_id = "ONLY_ON_CIRCLE"
+c.BookstoreSettings.s3_secret_access_key = "CAN_WE_DO_THIS"