Deploy optional local registry with self-signed CA

.github/workflows/test_k3s.yml

@@ -25,41 +25,47 @@ jobs:
             metrics-enabled: "true"
             traefik-enabled: "true"
             docker-enabled: "false"
+            registry-enabled: "true"
 
           - k3s-version: ""
             k3s-channel: latest
             helm-version: ""
             metrics-enabled: "false"
             traefik-enabled: "false"
             docker-enabled: "true"
+            registry-enabled: "false"
 
           - k3s-version: ""
             k3s-channel: latest
             helm-version: ""
             metrics-enabled: "true"
             traefik-enabled: "true"
             docker-enabled: "true"
+            registry-enabled: "true"
 
           - k3s-version: ""
             k3s-channel: latest
             helm-version: ""
             metrics-enabled: "false"
             traefik-enabled: "false"
             docker-enabled: "false"
+            registry-enabled: "false"
 
           - k3s-version: v1.24.7+k3s1
             k3s-channel: ""
             helm-version: v3.5.0
             metrics-enabled: "true"
             traefik-enabled: "true"
             docker-enabled: "false"
+            registry-enabled: "false"
 
           - k3s-version: v1.24.7+k3s1
             k3s-channel: ""
             helm-version: v3.5.0
             metrics-enabled: "false"
             traefik-enabled: "false"
             docker-enabled: "true"
+            registry-enabled: "false"
 
     steps:
       - uses: actions/checkout@v4
@@ -73,6 +79,7 @@ jobs:
           metrics-enabled: ${{ matrix.metrics-enabled }}
           traefik-enabled: ${{ matrix.traefik-enabled }}
           docker-enabled: ${{ matrix.docker-enabled }}
+          registry-enabled: ${{ matrix.registry-enabled }}
 
       - name: Verify action's outputs and env
         run: |
@@ -82,6 +89,7 @@ jobs:
           echo "k8s-version=${{ steps.k3s.outputs.k8s-version }}"
           echo "calico-version=${{ steps.k3s.outputs.calico-version }}"
           echo "helm-version=${{ steps.k3s.outputs.helm-version }}"
+          echo "registry-host=${{ steps.k3s.outputs.registry-host }}"
           echo "---"
 
           EXIT=0
@@ -178,6 +186,18 @@ jobs:
       - name: Run netpol enforcement test chart's tests
         run: helm test test-netpol-enforcement --logs
 
+      - name: Test registry
+        if: matrix.registry-enabled == 'true'
+        run: |
+          IMAGE="${{ steps.k3s.outputs.registry-host }}/nginx"
+          echo "IMAGE=$IMAGE"
+          docker pull nginx
+          docker tag nginx $IMAGE
+          docker push $IMAGE
+
+          sed "s%IMAGE%$IMAGE%" addons/registry-test.yaml | kubectl apply -f -
+          kubectl rollout status --watch --timeout=5m deployment/registry-test
+
       # ref: https://github.com/jupyterhub/action-k8s-namespace-report
       - name: Kubernetes namespace report (kube-system)
         uses: jupyterhub/action-k8s-namespace-report@v1
@@ -189,6 +209,8 @@ jobs:
             deploy/calico-kube-controllers
             deploy/metrics-server
             deploy/traefik
+            deployment/registry
+            deployment/registry-test
 
   # Provides a single status_all check that can be used in GitHub branch
   # protection rules instead of having to list each matrix job

action.yml

@@ -46,6 +46,10 @@ inputs:
     description: Enable K3s to use the Docker daemon
     required: false
     default: "false"
+  registry-enabled:
+    description: Install a local registry with a self-signed certificate
+    required: false
+    default: "false"
   extra-setup-args:
     description: Addition arguments to be passed to the K3S setup script
     required: false
@@ -67,6 +71,9 @@ outputs:
   helm-version:
     description: "Installed helm version, such as v3.13.0"
     value: "${{ steps.set-output.outputs.helm-version }}"
+  registry-host:
+    description: "Local registry host:port"
+    value: "${{ steps.set-output.outputs.registry-host }}"
 
 runs:
   using: "composite"
@@ -90,6 +97,15 @@ runs:
         echo "::endgroup::"
       shell: bash
 
+    # This should add the CA to the system store, so K3s should pick it up
+    - name: Setup self-signed CA
+      if: inputs.registry-enabled == 'true'
+      run: |
+        echo "::group::Create self-signed certificate"
+        ${{ github.action_path }}/addons/self-signed-ca.sh
+        echo "::endgroup::"
+      shell: bash
+
     # NOTE: We apply a workaround as of version 3.0.1 by passing
     #       --egress-selector-mode=disabled by default as not doing so following
     #       modern versions of k3s has led to issues with `kubectl exec` and
@@ -183,6 +199,19 @@ runs:
         echo "::endgroup::"
       shell: bash
 
+    - name: Setup local registry
+      if: inputs.registry-enabled == 'true'
+      run: |
+        echo "::group::Setup registry"
+        kubectl create ns registry
+        kubectl create secret tls registry-cert \
+          -n registry \
+          --cert=certs/registry.crt \
+          --key=certs/registry.key
+        kubectl apply -f ${{ github.action_path }}/addons/registry.yaml -n registry
+        echo "::endgroup::"
+      shell: bash
+
     - name: Set version output
       id: set-output
       run: |
@@ -192,7 +221,12 @@ runs:
         echo "k8s-version=$(k3s --version | grep --max-count=1 'k3s' | sed 's/.*\(v[0-9][^+]*\).*/\1/')" >> $GITHUB_OUTPUT
         echo "calico-version=$(cat /tmp/calico.yaml | grep --max-count=1 'calico/cni:v' | sed 's/.*calico\/cni:\(.*\)/\1/')" >> $GITHUB_OUTPUT
         echo "helm-version=$(helm version --short | sed 's/\([^+]*\).*/\1/')" >> $GITHUB_OUTPUT
-        echo "::endgroup::"
+
+        if [[ "${{ inputs.registry-enabled }}" == true ]]; then
+          echo "registry-host=$CERTIFICATE_IP:5000" >> $GITHUB_OUTPUT
+        else
+          echo "registry-host=" >> $GITHUB_OUTPUT
+        fi
       shell: bash
 
     - name: Wait for calico, coredns, metrics server, traefik
@@ -225,4 +259,10 @@ runs:
           kubectl rollout status --watch --timeout=5m deployment/traefik -n kube-system
         fi
         echo "::endgroup::"
+
+        if [[ "${{ inputs.registry-enabled }}" == true ]]; then
+          echo "::group::Wait for deployment/registry"
+          kubectl rollout status --watch --timeout=5m deployment/registry -n registry
+          echo "::endgroup::"
+        fi
       shell: bash

addons/registry-test.yaml

@@ -0,0 +1,19 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: registry-test
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      run: registry-test
+  template:
+    metadata:
+      labels:
+        run: registry-test
+    spec:
+      containers:
+        - name: web
+          # Replace this with CERTIFICATE_IP:PORT/image to test the local registry
+          image: IMAGE

addons/registry.yaml

@@ -0,0 +1,35 @@
+# No PVC since this is just for testing
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: registry
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      run: registry
+  template:
+    metadata:
+      labels:
+        run: registry
+    spec:
+      containers:
+        - name: registry
+          image: registry:2
+          ports:
+            - containerPort: 5000
+              hostPort: 5000
+          env:
+            - name: REGISTRY_HTTP_TLS_CERTIFICATE
+              value: "/certs/tls.crt"
+            - name: REGISTRY_HTTP_TLS_KEY
+              value: "/certs/tls.key"
+          volumeMounts:
+            - name: registry-certs
+              mountPath: "/certs"
+              readOnly: true
+      volumes:
+        - name: registry-certs
+          secret:
+            secretName: registry-cert

addons/self-signed-ca.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+set -eu
+
+# This should be a non-localhost IPv4
+CERTIFICATE_IP=$(hostname -I | cut -d" " -f1)
+
+mkdir -p certs
+openssl req \
+ -newkey rsa:4096 -nodes -sha256 -keyout certs/registry.key \
+ -subj "/CN=self-signed-ca" \
+ -addext "subjectAltName = IP:$CERTIFICATE_IP" \
+ -x509 -days 365 -out certs/registry.crt
+
+sudo cp certs/registry.crt /usr/local/share/ca-certificates/registry.crt
+sudo update-ca-certificates
+
+sudo systemctl restart docker
+
+echo "CERTIFICATE_IP=$CERTIFICATE_IP" >> "$GITHUB_ENV"