Add support for templated service ports on Python Kernels (#109)

kyujin-cho · achimnol · web-flow · commit cc7c34a37657 · 2020-03-17T21:06:03.000+09:00
* Add support for service port template
* Add .bin to Git LFS
* Apply service-defs in Python kernels

Co-authored-by: Joongi Kim &lt;joongi@lablup.com&gt;
diff --git a/.gitattributes b/.gitattributes
@@ -4,3 +4,4 @@
 *.tar.gz filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.tgz filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
diff --git a/python-ff/Dockerfile.19.09-py36-cuda9 b/python-ff/Dockerfile.19.09-py36-cuda9
@@ -29,11 +29,6 @@ RUN mkdir -p /tmp && \
     chown root:root /tmp && \
     chmod 1777 /tmp
 
-COPY ./code-server.${CODE_SERVER_VERSION}.tar.xz /tmp/code-server.tar.xz
-# TODO: Bundle ms-python extension with code-server when code-server v2 is released.
-# COPY code-server.ext.tar.gz /tmp/code-server.ext.tar.gz
-RUN tar xvJf /tmp/code-server.tar.xz -C /usr/local/bin
-
 # Backend.AI specifics
 LABEL ai.backend.kernelspec="1" \
       ai.backend.envs.corecount="OPENBLAS_NUM_THREADS,OMP_NUM_THREADS,NPROC" \
diff --git a/python/Dockerfile.3.6-ubuntu18.04 b/python/Dockerfile.3.6-ubuntu18.04
@@ -53,7 +53,14 @@ LABEL ai.backend.kernelspec="1" \
       ai.backend.base-distro="ubuntu16.04" \
       ai.backend.runtime-type="python" \
       ai.backend.runtime-path="/usr/bin/python3" \
-      ai.backend.service-ports="ipython:pty:3000,jupyter:http:8080,jupyterlab:http:8090"
+      ai.backend.service-ports="ipython:pty:3000,jupyter:http:8080,vscode:http:8180,jupyterlab:http:8090"
+
+COPY ./service-defs /etc/backend.ai/service-defs
 COPY policy.yml /etc/backend.ai/jail/policy.yml
+COPY ./vscode-exts.tar.gz /etc/backend.ai/vscode-exts.tar.gz
+
+RUN mkdir -p /etc/backend.ai/vscode-exts && \
+    tar xvzf /etc/backend.ai/vscode-exts.tar.gz -C /etc/backend.ai/vscode-exts && \
+    rm -rf /etc/backend.ai/vscode-exts.tar.gz
 
 # vim: ft=dockerfile
diff --git a/python/Dockerfile.3.7-anaconda2018.12 b/python/Dockerfile.3.7-anaconda2018.12
@@ -33,6 +33,7 @@ RUN /opt/conda/bin/python -m ipykernel install \
     cat /opt/conda/share/jupyter/kernels/python3/kernel.json
 
 # Backend.AI specifics
+COPY ./service-defs /etc/backend.ai/service-defs
 LABEL ai.backend.kernelspec="1" \
       ai.backend.envs.corecount="OPENBLAS_NUM_THREADS,OMP_NUM_THREADS,NPROC" \
       ai.backend.features="batch query uid-match user-input" \
diff --git a/python/Dockerfile.3.7-anaconda2019.07 b/python/Dockerfile.3.7-anaconda2019.07
@@ -41,8 +41,14 @@ LABEL ai.backend.kernelspec="1" \
       ai.backend.base-distro="ubuntu16.04" \
       ai.backend.runtime-type="python" \
       ai.backend.runtime-path="/opt/conda/bin/python" \
-      ai.backend.service-ports="ipython:pty:3000,jupyter:http:8080,jupyterlab:http:8090"
+      ai.backend.service-ports="ipython:pty:3000,jupyter:http:8080,vscode:http:8180,jupyterlab:http:8090"
 
+COPY ./service-defs /etc/backend.ai/service-defs
 COPY policy.yml /etc/backend.ai/jail/policy.yml
+COPY ./vscode-exts.tar.gz /etc/backend.ai/vscode-exts.tar.gz
+
+RUN mkdir -p /etc/backend.ai/vscode-exts && \
+    tar xvzf /etc/backend.ai/vscode-exts.tar.gz -C /etc/backend.ai/vscode-exts && \
+    rm -rf /etc/backend.ai/vscode-exts.tar.gz
 
 # vim: ft=dockerfile
diff --git a/python/Dockerfile.3.7-ubuntu18.04 b/python/Dockerfile.3.7-ubuntu18.04
@@ -55,6 +55,7 @@ RUN python3.7 -m ipykernel install --display-name "Python 3.7 on Backend.AI" &&
     cat /usr/local/share/jupyter/kernels/python3/kernel.json
 
 # Backend.AI specifics
+COPY ./service-defs /etc/backend.ai/service-defs
 LABEL ai.backend.kernelspec="1" \
       ai.backend.envs.corecount="OPENBLAS_NUM_THREADS,OMP_NUM_THREADS,NPROC" \
       ai.backend.features="batch query uid-match user-input" \
diff --git a/python/Dockerfile.3.8-ubuntu18.04 b/python/Dockerfile.3.8-ubuntu18.04
@@ -55,6 +55,7 @@ RUN python3.8 -m ipykernel install --display-name "Python 3.8 on Backend.AI" &&
     cat /usr/local/share/jupyter/kernels/python3/kernel.json
 
 # Backend.AI specifics
+COPY ./service-defs /etc/backend.ai/service-defs
 LABEL ai.backend.kernelspec="1" \
       ai.backend.envs.corecount="OPENBLAS_NUM_THREADS,OMP_NUM_THREADS,NPROC" \
       ai.backend.features="batch query uid-match user-input" \
diff --git a/python/Dockerfile.python2.7 b/python/Dockerfile.python2.7
@@ -43,6 +43,7 @@ RUN python -m ipykernel install --display-name "Python 2.7 on Backend.AI" && \
     cat /usr/local/share/jupyter/kernels/python2/kernel.json
 
 # Backend.AI specifics
+COPY ./service-defs /etc/backend.ai/service-defs
 LABEL ai.backend.kernelspec="1" \
       ai.backend.envs.corecount="OPENBLAS_NUM_THREADS,OMP_NUM_THREADS,NPROC" \
       ai.backend.features="batch query uid-match user-input" \
diff --git a/python/Dockerfile.python2.7-ubuntu18.04 b/python/Dockerfile.python2.7-ubuntu18.04
@@ -48,6 +48,7 @@ COPY custom.css /home/work/.jupyter/custom/custom.css
 COPY logo.svg /home/work/.jupyter/custom/logo.svg
 
 # Backend.AI specifics
+COPY ./service-defs /etc/backend.ai/service-defs
 LABEL ai.backend.kernelspec="1" \
       ai.backend.envs.corecount="OPENBLAS_NUM_THREADS,OMP_NUM_THREADS,NPROC" \
       ai.backend.features="batch query uid-match user-input" \
diff --git a/python/Dockerfile.python3.6 b/python/Dockerfile.python3.6
@@ -45,6 +45,7 @@ RUN python3 -m ipykernel install --display-name "Python 3.6 on Backend.AI" && \
     cat /usr/local/share/jupyter/kernels/python3/kernel.json
 
 # Backend.AI specifics
+COPY ./service-defs /etc/backend.ai/service-defs
 LABEL ai.backend.kernelspec="1" \
       ai.backend.envs.corecount="OPENBLAS_NUM_THREADS,OMP_NUM_THREADS,NPROC" \
       ai.backend.features="batch query uid-match user-input" \
diff --git a/python/Dockerfile.python3.6-ubuntu18.04 b/python/Dockerfile.python3.6-ubuntu18.04
@@ -45,6 +45,7 @@ RUN python3 -m ipykernel install --display-name "Python 3.6 on Backend.AI" && \
     cat /usr/local/share/jupyter/kernels/python3/kernel.json
 
 # Backend.AI specifics
+COPY ./service-defs /etc/backend.ai/service-defs
 LABEL ai.backend.kernelspec="1" \
       ai.backend.envs.corecount="OPENBLAS_NUM_THREADS,OMP_NUM_THREADS,NPROC" \
       ai.backend.features="batch query uid-match user-input" \
diff --git a/python/service-defs/digits.json b/python/service-defs/digits.json
@@ -0,0 +1,5 @@
+{
+    "command": [
+        "{runtime_path}", "-m", "digits"
+    ]
+}
diff --git a/python/service-defs/ipython.json b/python/service-defs/ipython.json
@@ -0,0 +1,5 @@
+{
+    "command": [
+        "{runtime_path}", "-m", "IPython"
+    ]
+}
diff --git a/python/service-defs/jupyter.json b/python/service-defs/jupyter.json
@@ -0,0 +1,20 @@
+{
+  "prestart": [
+    {
+      "action": "write_tempfile",
+      "args": {
+        "body": [
+          "c.NotebookApp.allow_root = True\n",
+          "c.NotebookApp.ip = \"0.0.0.0\"",
+          "c.NotebookApp.port = {ports[0]}",
+          "c.NotebookApp.token = \"\"",
+          "c.FileContentsManager.delete_to_trash = False"
+        ]
+      },
+      "ref": "jupyter_cfg"
+    }
+  ],
+  "command": [
+    "{runtime_path}", "-m", "notebook", "--no-browser", "--config", "{jupyter_cfg}"
+  ]
+}
diff --git a/python/service-defs/jupyterlab.json b/python/service-defs/jupyterlab.json
@@ -0,0 +1,20 @@
+{
+  "prestart": [
+    {
+      "action": "write_tempfile",
+      "args": {
+        "body": [
+          "c.NotebookApp.allow_root = True\n",
+          "c.NotebookApp.ip = \"0.0.0.0\"",
+          "c.NotebookApp.port = {ports[0]}",
+          "c.NotebookApp.token = \"\"",
+          "c.FileContentsManager.delete_to_trash = False"
+        ]
+      },
+      "ref": "jupyter_cfg"
+    }
+  ],
+  "command": [
+    "{runtime_path}", "-m", "jupyterlab", "--no-browser", "--config", "{jupyter_cfg}"
+  ]
+}
diff --git a/python/service-defs/sftp.json b/python/service-defs/sftp.json
@@ -0,0 +1,5 @@
+{
+    "command": [
+        "{runtime_path}", "-m", "sftpserver", "--port", "{ports[0]}"
+    ]
+}
diff --git a/python/service-defs/tensorboard.json b/python/service-defs/tensorboard.json
@@ -0,0 +1,18 @@
+{
+  "prestart": [
+    {
+      "action": "mkdir",
+      "args": {
+        "path": "/home/work/logs"
+      }
+    }
+  ],
+  "command": [
+    "{runtime_path}",
+    "-m", "tensorboard.main",
+    "--logdir", "/home/work/logs",
+    "--host", "0.0.0.0",
+    "--port", "{ports[0]}",
+    "--debugger_port", "6064"
+  ]
+}
diff --git a/python/service-defs/vscode.json b/python/service-defs/vscode.json
@@ -0,0 +1,17 @@
+{
+    "prestart": [],
+    "command": [
+      "/opt/kernel/code-server",
+      "--port", "{ports[0]}",
+      "--extensions-dir", "/etc/backend.ai/vscode-exts"
+    ],
+    "allowed_arguments": [
+      "--auth"
+    ],
+    "allowed_envs": [
+      "PASSWORD"
+    ],
+    "default_arguments": {
+      "--auth": "none"
+    }
+}
diff --git a/python/vscode-exts.tar.gz b/python/vscode-exts.tar.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e80612592cb33f80bf3635e60c85766b99ab0d71896ba2751d83e9ec644eebf5
+size 38377031
