Merge pull request #1531 from sgaist/podman_in_k8s

Podman in k8s

Author: manics

.github/workflows/test.yml

@@ -161,6 +161,20 @@ jobs:
           helm template --validate binderhub-test helm-chart/binderhub \
               --values tools/templates/lint-and-validate-values.yaml
 
+      - name: "Helm template --validate for dind (with lint-and-validate-values.yaml)"
+        if: matrix.test == 'helm'
+        run: |
+          helm template --validate binderhub-test helm-chart/binderhub \
+              --values tools/templates/lint-and-validate-values.yaml \
+              --set imageBuilderType=dind
+
+      - name: "Helm template --validate for pink (with lint-and-validate-values.yaml)"
+        if: matrix.test == 'helm'
+        run: |
+          helm template --validate binderhub-test helm-chart/binderhub \
+              --values tools/templates/lint-and-validate-values.yaml \
+              --set imageBuilderType=pink
+
       - name: Validate the chart against the k8s API
         if: matrix.test == 'helm'
         run: |

.github/workflows/watch-dependencies.yaml

@@ -29,6 +29,10 @@ jobs:
             registry: registry.hub.docker.com
             repository: library/docker
             values_path: dind.daemonset.image.tag
+          - name: podman
+            registry: quay.io
+            repository: podman/stable
+            values_path: pink.daemonset.image.tag
 
           # FIXME: After docker-image-cleaner 1.0.0 is released, we can enable
           #        this. So far, there isn't any available stable release, and
@@ -59,7 +63,7 @@ jobs:
         run: |
           latest_tag=$(
               docker run --rm quay.io/skopeo/stable list-tags docker://${{ matrix.registry }}/${{ matrix.repository }} \
-            | jq -r '[.Tags[] | select(. | match("^\\d+\\.\\d+\\.\\d+$") | .string)] | sort_by(split(".") | map(tonumber)) | last'
+            | jq -r '[.Tags[] | select(. | match("^v?\\d+\\.\\d+\\.\\d+$") | .string)] | sort_by(split(".") | map(ltrimstr("v") | tonumber)) | last'
           )
           echo "tag=$latest_tag" >> $GITHUB_OUTPUT
 

binderhub/build.py

@@ -259,6 +259,12 @@ def _default_namespace(self):
         {}, help="Node selector for the kubernetes build pod.", config=True
     )
 
+    extra_envs = Dict(
+        {},
+        help="Extra environment variables for the kubernetes build pod.",
+        config=True,
+    )
+
     log_tail_lines = Integer(
         100,
         help=(
@@ -380,7 +386,10 @@ def submit(self):
                 )
             )
 
-        env = []
+        env = [
+            client.V1EnvVar(name=key, value=value)
+            for key, value in self.extra_envs.items()
+        ]
         if self.git_credentials:
             env.append(
                 client.V1EnvVar(name="GIT_CREDENTIAL_ENV", value=self.git_credentials)

binderhub/tests/test_build.py

@@ -13,7 +13,7 @@
 from tornado.httputil import url_concat
 from tornado.queues import Queue
 
-from binderhub.build import Build, ProgressEvent
+from binderhub.build import Build, KubernetesBuildExecutor, ProgressEvent
 from binderhub.build_local import LocalRepo2dockerBuild, ProcessTerminated, _execute_cmd
 
 from .utils import async_requests
@@ -241,6 +241,47 @@ def test_git_credentials_passed_to_podspec_upon_submit():
     assert env["GIT_CREDENTIAL_ENV"] == git_credentials
 
 
+def test_extra_environment_variables_passed_to_podspec_upon_submit():
+    extra_environments = {
+        "CONTAINER_HOST": "unix:///var/run/docker.sock",
+        "REGISTRY_AUTH_FILE": "/root/.docker/config.json",
+    }
+
+    mock_k8s_api = _list_image_builder_pods_mock()
+
+    class EnvBuild(KubernetesBuildExecutor):
+        q = mock.MagicMock()
+        api = mock_k8s_api
+        name = "test_build"
+        repo_url = "repo"
+        ref = "ref"
+        image_name = "name"
+        extra_envs = extra_environments
+        namespace = "build_namespace"
+        push_secret = ""
+        build_image = "image"
+        memory_limit = 0
+        docker_host = "http://mydockerregistry.local"
+        node_selector = {}
+
+    build = EnvBuild()
+
+    with mock.patch.object(build.stop_event, "is_set", return_value=True):
+        build.submit()
+
+    call_args_list = mock_k8s_api.create_namespaced_pod.call_args_list
+    assert len(call_args_list) == 1
+
+    args = call_args_list[0][0]
+    pod = args[1]
+
+    assert len(pod.spec.containers) == 1
+
+    env = {env_var.name: env_var.value for env_var in pod.spec.containers[0].env}
+
+    assert env == extra_environments
+
+
 async def test_local_repo2docker_build():
     q = Queue()
     repo_url = "https://github.com/binderhub-ci-repos/cached-minimal-dockerfile"

docs/source/zero-to-binderhub/setup-binderhub.rst

@@ -355,7 +355,7 @@ You now have a functioning BinderHub at the above IP address.
 Customizing your Deployment
 ---------------------------
 
-The Helm chart used to install your BinderHub deployemnt exposes a number of
+The Helm chart used to install your BinderHub deployment exposes a number of
 optional features. Below we describe a few of the most common customizations
 and how you can configure them.
 
@@ -431,7 +431,7 @@ time at `the token administration page <https://github.com/settings/tokens>`_.
 GitLab
 ^^^^^^
 
-To access private GitLab repos, create an API token for your binderhub user
+To access private GitLab repos, create an API token for your BinderHub user
 under "User Settings" > "Access tokens". It at least needs the scopes "api" and
 "read_repository".
 
@@ -452,20 +452,20 @@ clone a repo.
 Use Docker-inside-Docker (DinD)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-By default, BinderHub will build pods with the host Docker installation.
+By default, BinderHub will build images with the host Docker installation.
 This often means you are stuck with whatever version of Docker provided by your
 cloud provider. BinderHub supports an alternative that uses `Docker-in-Docker
 (DinD) <https://hub.docker.com/_/docker>`_. To turn `dind` on, you'll need to set
 the following configuration in your ``config.yaml`` file::
 
+    imageBuilderType: dind
     dind:
-      enabled: true
       daemonset:
         image:
           name: docker
           tag: 18.09.2-dind
 
-If you plan to host multiple BinderHub deployments on the same kubernetes
+If you plan to host multiple BinderHub deployments on the same Kubernetes
 cluster, you'll also need to isolate the host socket and library directory
 for each DinD application::
 
@@ -474,4 +474,32 @@ for each DinD application::
       hostSocketDir: /var/run/dind/"<name of deployment, e.g. staging>"
 
 
+Use Podman-inside-Kubernetes (PinK)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In case Docker is not an option, Podman can be used as drop in replacement.
+Note that the implications about using a host installation of Podman are the same
+as with Docker. BinderHub supports an alternative that uses `Podman-in-Kubernetes
+(PinK) <https://www.redhat.com/sysadmin/podman-inside-kubernetes>`_. To turn
+`pink` on, you'll need to set the following configuration in your ``config.yaml`` file::
+
+    imageBuilderType: pink
+
+You can optionally override the default podman image:
+
+    pink:
+      daemonset:
+        image:
+          name: quay.io/podman/stable
+          tag: v4.2.0
+
+If you plan to host multiple BinderHub deployments on the same Kubernetes
+cluster, you'll also need to isolate the host socket and library directory
+for each DinD application::
+
+    pink:
+      hostStorageDir: /var/lib/pink/"<name of deployment, e.g. staging>"
+      hostSocketDir: /var/run/pink/"<name of deployment, e.g. staging>"
+
+
 For next steps, see :doc:`../debug` and :doc:`turn-off`.

helm-chart/binderhub/files/binderhub_config.py

@@ -46,11 +46,12 @@ def get_value(key, default=None):
 for section, sub_cfg in get_value("config", {}).items():
     c[section].update(sub_cfg)
 
-if get_value("dind.enabled", False) and get_value("dind.hostSocketDir"):
-    c.BinderHub.build_docker_host = "unix://{}/docker.sock".format(
-        get_value("dind.hostSocketDir")
-    )
-
+imageBuilderType = get_value("imageBuilderType")
+if imageBuilderType in ["dind", "pink"]:
+    hostSocketDir = get_value(f"{imageBuilderType}.hostSocketDir")
+    if hostSocketDir:
+        socketname = "docker" if imageBuilderType == "dind" else "podman"
+        c.BinderHub.build_docker_host = f"unix://{hostSocketDir}/{socketname}.sock"
 
 if c.BinderHub.auth_enabled:
     hub_url = urlparse(c.BinderHub.hub_url)

helm-chart/binderhub/schema.yaml

@@ -28,6 +28,8 @@ required:
   - jupyterhub
   - deployment
   - dind
+  - pink
+  - imageBuilderType
   - imageCleaner
   - ingress
   - initContainers
@@ -349,14 +351,21 @@ properties:
           See the [Kubernetes docs](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/)
           to learn more about labels.
 
+  imageBuilderType:
+    type: string
+    enum: ["local", "dind", "pink"]
+    default: "local"
+    description: |
+      Selected image builder type
+
   dind:
     type: object
     additionalProperties: false
     properties:
       enabled:
         type: boolean
         description: |
-          TODO
+          DEPRECATED: Use `imageBuilderType: dind`
       initContainers: &initContainers-spec
         type: array
         description: |
@@ -409,6 +418,29 @@ properties:
         description: |
           TODO
 
+  pink:
+    type: object
+    additionalProperties: false
+    properties:
+      initContainers: *initContainers-spec
+      daemonset:
+        type: object
+        additionalProperties: false
+        properties:
+          image: *image-spec
+          lifecycle: *lifecycle-spec
+          extraVolumes: *extraVolumes-spec
+          extraVolumeMounts: *extraVolumeMounts-spec
+      resources: *resources-spec
+      hostStorageDir:
+        type: string
+        description: |
+          Host path where the containers storage will be located
+      hostSocketDir:
+        type: string
+        description: |
+          Host path where the podman socket will be located
+
   imageCleaner:
     type: object
     additionalProperties: false

helm-chart/binderhub/templates/NOTES.txt

@@ -182,6 +182,14 @@ config:
 {{- $breaking = print $breaking "\n\nRENAMED: jupyterhub.custom.cors.allowOrigin has been renamed to jupyterhub.hub.config.BinderSpawner.cors_allow_origin" }}
 {{- end }}
 
+{{- if hasKey .Values.dind "enabled" }}
+{{- $breaking = print $breaking "\n\nCHANGED:" }}
+{{- $breaking = print $breaking "\n\ndind:" }}
+{{- $breaking = print $breaking "\n  enabled: true" }}
+{{- $breaking = print $breaking "\n\nmust as of version 0.3.0 be replace by" }}
+{{- $breaking = print $breaking "\n\nimageBuilderType: dind" }}
+{{- end }}
+
 {{- if $breaking }}
 {{- fail (print $breaking_title $breaking) }}
 {{- end }}