Merge pull request #1602 from manics/test-dind-pink

Test dind and pink

Author: consideRatio

.github/workflows/test.yml

@@ -54,7 +54,8 @@ jobs:
         test:
           - main
           - auth
-          - helm
+        test-variation:
+          - ""
         include:
           # Chart.yaml contains the chart's oldest supported k8s version, we
           # test against that and the oldest known supported helm cli version
@@ -64,25 +65,60 @@ jobs:
           - k3s-channel: v1.20
             helm-version: v3.5.0
             test: helm
+            test-variation: dind
+            local-chart-extra-args: >-
+              --values testing/k8s-binder-k8s-hub/binderhub-chart+dind.yaml
+              --set config.BinderHub.image_prefix=$REGISTRY_HOST/test/
+              --set registry.url=http://$REGISTRY_HOST
+          - k3s-channel: latest
+            test: helm
+            test-variation: pink
+            local-chart-extra-args: >-
+              --values testing/k8s-binder-k8s-hub/binderhub-chart+pink.yaml
+              --set config.BinderHub.image_prefix=$REGISTRY_HOST/test/
+              --set registry.url=http://$REGISTRY_HOST
           - k3s-channel: latest
             test: helm
             test-variation: upgrade
             # upgrade-from represents a release channel, see: https://jupyterhub.github.io/helm-chart/info.json
             upgrade-from: dev
             upgrade-from-extra-args: ""
+
+    services:
+      registry:
+        image: docker.io/library/registry:latest
+        ports:
+          - 5000:5000
+
     steps:
       - uses: actions/checkout@v3
         with:
           # chartpress requires the full history
           fetch-depth: 0
 
+      - name: Set registry host
+        if: matrix.test-variation == 'dind' || matrix.test-variation == 'pink'
+        run: |
+          REGISTRY_HOST=$(hostname -I | awk '{print $1}'):5000
+          echo REGISTRY_HOST="$REGISTRY_HOST" >> $GITHUB_ENV
+
+          # Allow k3s to pull from private registry
+          # https://docs.k3s.io/installation/private-registry
+          sudo mkdir -p /etc/rancher/k3s/
+          cat << EOF | sudo tee /etc/rancher/k3s/registries.yaml
+          mirrors:
+            "$REGISTRY_HOST":
+              endpoint:
+                - "http://$REGISTRY_HOST"
+          EOF
+
       - uses: jupyterhub/action-k3s-helm@v3
         with:
           k3s-channel: ${{ matrix.k3s-channel }}
           helm-version: ${{ matrix.helm-version }}
           metrics-enabled: false
           traefik-enabled: false
-          docker-enabled: true
+          docker-enabled: ${{ matrix.test-variation != 'dind' && matrix.test-variation != 'pink' }}
 
       - name: Setup OS level dependencies
         run: |
@@ -145,9 +181,17 @@ jobs:
         run: |
           export DOCKER_BUILDKIT=1
 
+          CHARTPRESS_ARGS=
+          if [ "${{ matrix.test-variation }}" = "dind" -o "${{ matrix.test-variation }}" = "pink" ]; then
+            CHARTPRESS_ARGS="--image-prefix localhost:5000/binderhub- --push"
+
+            # Allow the pods to push to the non-https GitHub workflow registry
+            envsubst < testing/k8s-binder-k8s-hub/cm-insecure-registries-${{ matrix.test-variation }}.yaml | kubectl apply -f -
+          fi
+
           # Use chartpress to create the helm chart and build its images
           helm dependency update ./helm-chart/binderhub
-          (cd helm-chart && chartpress)
+          (cd helm-chart && chartpress $CHARTPRESS_ARGS)
           git --no-pager diff --color=always
 
       - name: Generate values.schema.json from schema.yaml
@@ -286,7 +330,12 @@ jobs:
         uses: jupyterhub/action-k8s-namespace-report@v1
         if: always()
         with:
-          important-workloads: deploy/binder deploy/hub deploy/proxy
+          important-workloads: >
+            deploy/binder
+            deploy/hub
+            deploy/proxy
+            daemonset/binderhub-test-dind
+            daemonset/binderhub-test-pink
 
       # GitHub action reference: https://github.com/codecov/codecov-action
       - name: Upload coverage stats

testing/k8s-binder-k8s-hub/binderhub-chart+dind.yaml

@@ -0,0 +1,21 @@
+# Additional configuration for testing dind
+# You must create configmap/insecure-registries-dind first to allow testing with an
+# insecure http registry
+# https://docs.docker.com/registry/insecure/
+
+config:
+  BinderHub:
+    use_registry: true
+
+imageBuilderType: dind
+
+dind:
+  daemonset:
+    extraVolumeMounts:
+      - name: insecure-registries-dind
+        mountPath: /etc/docker/daemon.json
+        subPath: daemon.json
+    extraVolumes:
+      - name: insecure-registries-dind
+        configMap:
+          name: insecure-registries-dind

testing/k8s-binder-k8s-hub/binderhub-chart+pink.yaml

@@ -0,0 +1,22 @@
+# Additional configuration for testing podman
+# You must create configmap/insecure-registries-pink first to allow testing with an
+# insecure http registry
+
+config:
+  BinderHub:
+    use_registry: true
+
+imageBuilderType: pink
+
+pink:
+  daemonset:
+    extraArgs:
+      - --log-level=debug
+    extraVolumeMounts:
+      - name: insecure-registries-pink
+        mountPath: /etc/containers/registries.conf.d/100-insecure-registries-pink.conf
+        subPath: 100-insecure-registries-pink.conf
+    extraVolumes:
+      - name: insecure-registries-pink
+        configMap:
+          name: insecure-registries-pink

testing/k8s-binder-k8s-hub/binderhub-chart-config.yaml

@@ -38,7 +38,8 @@ ingress:
 # against which we haven't. We currently only test this through the use of
 # lint-and-validate-values.yaml and setting this value explicitly to make sure
 # our rendered templates are valid against a k8s api-server.
-imageBuilderType: "host"
+# This is already the default
+# imageBuilderType: "host"
 
 # NOTE: This is a mirror of the jupyterhub section in
 #       jupyterhub-chart-config.yaml in testing/local-binder-k8s-hub, keep these

testing/k8s-binder-k8s-hub/cm-insecure-registries-dind.yaml

@@ -0,0 +1,11 @@
+# REGISTRY_HOST='HOST:PORT' envsubst < cm-insecure-registries-dind.yaml | kubectl apply -f -
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: insecure-registries-dind
+data:
+  daemon.json: |
+    {
+      "insecure-registries": ["$REGISTRY_HOST"],
+      "debug": true
+    }

testing/k8s-binder-k8s-hub/cm-insecure-registries-pink.yaml

@@ -0,0 +1,12 @@
+# REGISTRY_HOST='HOST:PORT' envsubst < cm-insecure-registries-pink.yaml | kubectl apply -f -
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: insecure-registries-pink
+data:
+  100-insecure-registries-pink.conf: |
+    # https://www.redhat.com/sysadmin/manage-container-registries
+    unqualified-search-registries = ["docker.io"]
+    [[registry]]
+    location="$REGISTRY_HOST"
+    insecure=true