Integration of Jupyter through docker-compose (#1088)

Vismayak · lmarini · longshuicy · web-flow · commit f982515d3fe3 · 2024-07-11T15:46:04.000-05:00
* Working Jupyterhub Integration

* Linting

* Handling logout

* Updating docker-compose to keep jupyterhub version static

* Incorporating feedback

* Working Jupyterhub Integration

* Linting

* Handling logout

* Updating docker-compose to keep jupyterhub version static

* Incorporating feedback

* Fix conflict

* Fix config

* Creating a traefik setup for prod docker-compose

* Make it work for dev set up

* Updating .env ecample

* better file names

* Updated prod webpack and a shell script for prod setup

---------

Co-authored-by: Luigi Marini &lt;lmarini@illinois.edu&gt;
Co-authored-by: Chen Wang &lt;cwang138@illinois.edu&gt;
diff --git a/.gitignore b/.gitignore
@@ -82,3 +82,4 @@ secrets.yaml
 # faker
 official.csv
 fact.png
+jupyterhub/.env-dev
diff --git a/README.md b/README.md
@@ -36,7 +36,9 @@ There is a few other documentation links available on the [website](https://clow
 ## Installation
 
 The easiest way of running Clowder v2 is checking out the [code](https://github.com/clowder-framework/clowder2)
-and running `docker compose up` in the main directory.
+and running `docker compose up` in the main directory. If you would like to run Clowder with JupyterHub,
+you can use our script `docker-prod.sh` to start the services. Run `./docker-prod.sh prod up` to start the services
+and `./docker-prod.sh prod down` to stop them.
 
 Helm charts are available for running Clowder v2 on Kubernetes. See the [helm](https://github.com/clowder-framework/clowder2/tree/main/deployments/kubernetes/charts) directory for more information.
 
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
@@ -67,6 +67,8 @@ services:
 
   postgres:
       image: postgres
+      networks:
+        - clowder2
       volumes:
         - postgres_data:/var/lib/postgresql/data
       environment:
@@ -79,6 +81,8 @@ services:
       volumes:
         - ./scripts/keycloak/clowder-realm-dev.json:/opt/keycloak/data/import/realm.json:ro
         - ./scripts/keycloak/clowder-theme/:/opt/keycloak/themes/clowder-theme/:ro
+      networks:
+        - clowder2
       command:
         - start-dev
         - --http-relative-path /keycloak
@@ -172,6 +176,7 @@ services:
 
 networks:
   clowder2:
+    name: clowder2
 
 ## By default this config uses default local driver,
 ## For custom volumes replace with volume driver configuration.
diff --git a/docker-compose.jupyter-dev.yml b/docker-compose.jupyter-dev.yml
@@ -0,0 +1,30 @@
+version: '3'
+services:
+  jupyterhub:
+    build:
+        context: jupyterhub
+        dockerfile: Dockerfile.jupyterhub
+        args:
+              JUPYTERHUB_VERSION: 4
+    restart: always
+    networks:
+      - clowder2
+    volumes:
+      # The JupyterHub configuration file
+      - ./jupyterhub/jupyterhub_dev_config.py:/srv/jupyterhub/jupyterhub_config.py:ro
+      # Bind Docker socket on the hostso we can connect to the daemon from
+      # within the container
+      - /var/run/docker.sock:/var/run/docker.sock:rw
+      # Bind Docker volume on host for JupyterHub database and cookie secrets
+      - jupyterhub-data:/data
+    ports:
+      - "8765:8000"
+    env_file:
+        -   jupyterhub/.env-dev
+    command: jupyterhub -f /srv/jupyterhub/jupyterhub_config.py
+
+    depends_on:
+      - keycloak
+
+volumes:
+    jupyterhub-data:
diff --git a/docker-compose.jupyter.yml b/docker-compose.jupyter.yml
@@ -0,0 +1,33 @@
+version: '3'
+services:
+  jupyterhub:
+    build:
+        context: jupyterhub
+        dockerfile: Dockerfile.jupyterhub
+        args:
+              JUPYTERHUB_VERSION: 4
+    restart: always
+    networks:
+      - clowder2
+    volumes:
+      # The JupyterHub configuration file
+      - ./jupyterhub/jupyterhub_config.py:/srv/jupyterhub/jupyterhub_config.py:ro
+      # Bind Docker socket on the hostso we can connect to the daemon from
+      # within the container
+      - /var/run/docker.sock:/var/run/docker.sock:rw
+      # Bind Docker volume on host for JupyterHub database and cookie secrets
+      - jupyterhub_data:/data
+    env_file:
+        -   jupyterhub/.env
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.jupyterhub.rule=PathPrefix(`/jupyterhub`)"
+      - "traefik.http.services.jupyterhub.loadbalancer.server.port=8000"
+
+    command: jupyterhub -f /srv/jupyterhub/jupyterhub_config.py
+
+    depends_on:
+      - keycloak
+
+volumes:
+    jupyterhub_data:
diff --git a/docker-dev.sh b/docker-dev.sh
@@ -7,3 +7,12 @@ if [ "$1" = "down" ]
 then
   docker-compose -f docker-compose.dev.yml -p clowder2-dev down
 fi
+if [ "$1" = "jupyter" ] && [ "$2" = "up" ]
+then
+  docker-compose -f docker-compose.dev.yml -f docker-compose.jupyter-dev.yml -p clowder2-dev  up -d --build
+fi
+
+if [ "$1" = "jupyter" ] && [ "$2" = "down" ]
+then
+  docker-compose -f docker-compose.dev.yml -f docker-compose.jupyter-dev.yml -p clowder2-dev  down
+fi
diff --git a/docker-prod.sh b/docker-prod.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env sh
+if [ "$1" = "prod" ] && [ "$2" = "up" ]
+then
+  docker-compose -f docker-compose.yml -f docker-compose.jupyter.yml up -d
+fi
+
+if [ "$1" = "prod" ] && [ "$2" = "down" ]
+then
+  docker-compose -f docker-compose.yml -f docker-compose.jupyter.yml down
+fi
diff --git a/docs/docs/devs/getstarted.md b/docs/docs/devs/getstarted.md
@@ -47,6 +47,8 @@ section below).
 - Running `docker-compose logs -f` displays the live logs for all containers. To view the logs of individual containers,
   provide the container name. For example, for viewing the backend module logs, run `docker-compose logs -f backend`.
 - Running `./docker-dev.sh down` brings down the required services.
+- If you want to run the jupyterhub, you can run `./docker-dev.sh jupyter up`. The jupyterhub will be available at
+  `http://localhost:8765`. You can bring it down using `./docker-dev.sh jupyter down`.
 
 **Note:** `./docker-dev.sh` sets the project name flag to `-p clowder2-dev`. This is so that the dev containers
 don't get mixed with the production containers if the user is running both on the same machine using `docker-compose.yml`.
diff --git a/frontend/package.json b/frontend/package.json
@@ -12,7 +12,7 @@
 		"start-message": "babel-node tools/startMessage.js",
 		"prestart": "npm-run-all --parallel start-message",
 		"start": "npm-run-all --parallel open:src",
-		"start:dev": "export CLOWDER_REMOTE_HOSTNAME=http://localhost:8000 && npm run start",
+		"start:dev": "export CLOWDER_REMOTE_HOSTNAME=http://localhost:8000 && export JUPYTERHUB_URL=http://localhost:8765 && npm run start",
 		"open:src": "babel-node tools/srcServer.js",
 		"open:dist": "babel-node tools/distServer.js",
 		"lint:watch": "npm run lint --watch",
diff --git a/frontend/src/app.config.ts b/frontend/src/app.config.ts
@@ -9,6 +9,7 @@ interface Config {
 	hostname: string;
 	apikey: string;
 	GHIssueBaseURL: string;
+	jupyterHubURL: string;
 	KeycloakBaseURL: string;
 	KeycloakLogin: string;
 	KeycloakLogout: string;
@@ -66,6 +67,10 @@ config["KeycloakRegister"] = `${config.KeycloakBaseURL}/register`;
 config["searchEndpoint"] = `${hostname}/api/v2/elasticsearch`;
 config["publicSearchEndpoint"] = `${hostname}/api/v2/public_elasticsearch`;
 
+// jupterhub
+const localJupyterhubURL: string = `${config.hostname}/jupyterhub`;
+config["jupyterHubURL"] = process.env.JUPYTERHUB_URL || localJupyterhubURL;
+
 // refresh token time interval
 config["refreshTokenInterval"] = 1000 * 60; // 1 minute
 // updated extractor logs
diff --git a/frontend/src/components/Layout.tsx b/frontend/src/components/Layout.tsx
@@ -22,6 +22,7 @@ import { RootState } from "../types/data";
 import { AddBox, Explore } from "@material-ui/icons";
 import HistoryIcon from "@mui/icons-material/History";
 import GroupIcon from "@mui/icons-material/Group";
+import MenuBookIcon from "@mui/icons-material/MenuBook";
 import Gravatar from "react-gravatar";
 import PersonIcon from "@mui/icons-material/Person";
 import InfoOutlinedIcon from "@mui/icons-material/InfoOutlined";
@@ -40,6 +41,8 @@ import AdminPanelSettingsIcon from "@mui/icons-material/AdminPanelSettings";
 import { Footer } from "./navigation/Footer";
 import BuildIcon from "@mui/icons-material/Build";
 
+import config from "../app.config";
+
 const drawerWidth = 240;
 
 const Main = styled("main", { shouldForwardProp: (prop) => prop !== "open" })<{
@@ -422,6 +425,22 @@ export default function PersistentDrawerLeft(props) {
 						</ListItemButton>
 					</ListItem>
 				</List>
+				{/*TODO: Need to make link dynamic */}
+				<List>
+					<ListItem key={"jupyter"} disablePadding>
+						<ListItemButton
+							component={RouterLink}
+							to={config.jupyterHubURL}
+							target="_blank"
+							rel="noopener noreferrer"
+						>
+							<ListItemIcon>
+								<MenuBookIcon />
+							</ListItemIcon>
+							<ListItemText primary={"Jupyter Notebook"} />
+						</ListItemButton>
+					</ListItem>
+				</List>
 				<List>
 					<ListItem key={"feeds"} disablePadding>
 						<ListItemButton component={RouterLink} to="/feeds">
diff --git a/frontend/webpack.config.dev.js b/frontend/webpack.config.dev.js
@@ -8,6 +8,9 @@ import ESLintPlugin from "eslint-webpack-plugin";
 console.log(
 	`the current CLOWDER_REMOTE_HOSTNAME environment variable is ${process.env.CLOWDER_REMOTE_HOSTNAME}`
 );
+console.log(
+	`the JupyterHub URL is set to ${process.env.JUPYTERHUB_URL}`
+)
 
 export default {
 	mode: "development",
@@ -40,6 +43,7 @@ export default {
 				CLOWDER_REMOTE_HOSTNAME: JSON.stringify(
 					process.env.CLOWDER_REMOTE_HOSTNAME
 				),
+				JUPYTERHUB_URL: JSON.stringify(process.env.JUPYTERHUB_URL),
 				APIKEY: JSON.stringify(process.env.APIKEY),
 				KeycloakBaseURL: JSON.stringify(process.env.KeycloakBaseURL),
 			},
diff --git a/frontend/webpack.config.prod.js b/frontend/webpack.config.prod.js
@@ -11,6 +11,9 @@ import TerserPlugin from "terser-webpack-plugin";
 console.log(
 	`the current CLOWDER_REMOTE_HOSTNAME environment variable is ${process.env.CLOWDER_REMOTE_HOSTNAME}`
 );
+console.log(
+	`the JupyterHub URL is set to ${process.env.JUPYTERHUB_URL}`
+)
 
 export default {
 	mode: "production",
@@ -47,6 +50,7 @@ export default {
 				CLOWDER_REMOTE_HOSTNAME: JSON.stringify(
 					process.env.CLOWDER_REMOTE_HOSTNAME
 				),
+				JUPYTERHUB_URL: JSON.stringify(process.env.JUPYTERHUB_URL),
 				APIKEY: JSON.stringify(process.env.APIKEY),
 				KeycloakBaseURL: JSON.stringify(process.env.KeycloakBaseURL),
 			},
diff --git a/jupyterhub/.env-example b/jupyterhub/.env-example
@@ -0,0 +1,14 @@
+# Example configuration file for Clowder JupyterHub
+KEYCLOAK_HOSTNAME="keycloak:8080/keycloak"
+# Development mode use the following line instead
+#KEYCLOAK_HOSTNAME="keycloak:8080/keycloak"
+KEYCLOAK_AUDIENCE="clowder"
+KEYCLOAK_REALM="clowder"
+JUPYTERHUB_ADMIN="admin"
+#Change network name to the one created by docker-compose
+DOCKER_NETWORK_NAME="clowder2_clowder2"
+DOCKER_NOTEBOOK_IMAGE="quay.io/jupyter/base-notebook:latest"
+DOCKER_NOTEBOOK_DIR="/home/jovyan/work"
+JUPYTERHUB_CRYPT_KEY=""
+CLOWDER_URL="localhost"
+PROD_DEPLOYMENT="false"
diff --git a/jupyterhub/Dockerfile.jupyterhub b/jupyterhub/Dockerfile.jupyterhub
@@ -0,0 +1,14 @@
+ARG JUPYTERHUB_VERSION
+FROM quay.io/jupyterhub/jupyterhub:$JUPYTERHUB_VERSION
+
+# Install dockerspawner,
+# hadolint ignore=DL3013
+RUN python3 -m pip install --no-cache-dir \
+    dockerspawner
+
+# Install custom authenticator
+WORKDIR /tmp/authenticator/
+COPY authenticator /tmp/authenticator/
+RUN pip3 install /tmp/authenticator
+
+CMD ["jupyterhub", "-f", "/srv/jupyterhub/jupyterhub_config.py"]
diff --git a/jupyterhub/authenticator/customauthenticator/__init__.py b/jupyterhub/authenticator/customauthenticator/__init__.py
diff --git a/jupyterhub/authenticator/customauthenticator/custom.py b/jupyterhub/authenticator/customauthenticator/custom.py
diff --git a/jupyterhub/authenticator/setup.py b/jupyterhub/authenticator/setup.py
diff --git a/jupyterhub/authenticator/test_jwt.py b/jupyterhub/authenticator/test_jwt.py
diff --git a/jupyterhub/jupyterhub_config.py b/jupyterhub/jupyterhub_config.py
diff --git a/jupyterhub/jupyterhub_dev_config.py b/jupyterhub/jupyterhub_dev_config.py
