e2e test

Signed-off-by: Pat O'Connor <[email protected]>

Author: chipspeak

.github/workflows/e2e_tests.yaml

@@ -63,10 +63,113 @@ jobs:
       - name: Setup NVidia GPU environment for KinD
         uses: ./common/github-actions/nvidia-gpu-setup
 
-      - name: Setup and start KinD cluster
-        uses: ./common/github-actions/kind
+      - name: Init directories
+        run: |
+          TEMP_DIR="$(pwd)/tmp"
+          mkdir -p "${TEMP_DIR}"
+          echo "TEMP_DIR=${TEMP_DIR}" >> $GITHUB_ENV
+
+          mkdir -p "$(pwd)/bin"
+          echo "$(pwd)/bin" >> $GITHUB_PATH
+
+      - name: Container image registry
+        run: |
+          podman run -d -p 5000:5000 --name registry registry:2.8.1
+
+          export NODE_IMAGE="kindest/node:v1.26.0@sha256:691e24bd2417609db7e589e1a479b902d2e209892a10ce375fab60a8407c7352"
+
+          export REGISTRY_ADDRESS=$(hostname -i):5000
+          echo "REGISTRY_ADDRESS=${REGISTRY_ADDRESS}" >> $GITHUB_ENV
+          echo "Container image registry started at ${REGISTRY_ADDRESS}"
+
+          KIND_CONFIG_FILE=${TEMP_DIR}/kind.yaml
+
+          # Create KIND config with 1 worker node
+          cat > ${KIND_CONFIG_FILE} <<EOF
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+containerdConfigPatches:
+- |-
+  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."${REGISTRY_ADDRESS}"]
+    endpoint = ["http://${REGISTRY_ADDRESS}"]
+nodes:
+- role: control-plane
+  image: ${NODE_IMAGE}
+  kubeadmConfigPatches:
+  - |
+    kind: InitConfiguration
+    nodeRegistration:
+      kubeletExtraArgs:
+        node-labels: "ingress-ready=true"
+  extraPortMappings:
+  - containerPort: 80
+    hostPort: 80
+    protocol: TCP
+  - containerPort: 443
+    hostPort: 443
+    protocol: TCP
+- role: worker
+  image: ${NODE_IMAGE}
+  labels:
+    worker-1: true
+  extraMounts:
+    - hostPath: /dev/null
+      containerPath: /var/run/nvidia-container-devices/all
+EOF
+
+          echo "KIND_CONFIG_FILE=${KIND_CONFIG_FILE}" >> $GITHUB_ENV
+
+          sudo --preserve-env=REGISTRY_ADDRESS sh -c 'cat > /etc/containers/registries.conf.d/local.conf <<EOF
+[[registry]]
+prefix = "$REGISTRY_ADDRESS"
+insecure = true
+location = "$REGISTRY_ADDRESS"
+EOF'
+
+      - name: Setup KinD cluster
+        uses: helm/[email protected]
         with:
-          worker-nodes: 1
+          cluster_name: cluster
+          version: v0.17.0
+          config: ${{ env.KIND_CONFIG_FILE }}
+
+      - name: Print cluster info
+        run: |
+          echo "KinD cluster:"
+          kubectl cluster-info
+          kubectl describe nodes
+
+      - name: Install Ingress controller
+        run: |
+          VERSION=controller-v1.9.6
+          echo "Deploying Ingress controller into KinD cluster"
+          curl https://raw.githubusercontent.com/kubernetes/ingress-nginx/"${VERSION}"/deploy/static/provider/kind/deploy.yaml | sed "s/--publish-status-address=localhost/--report-node-internal-ip-address\\n        - --status-update-interval=10/g" | kubectl apply -f -
+          kubectl annotate ingressclass nginx "ingressclass.kubernetes.io/is-default-class=true"
+          # Turn on SSL Passthrough
+          kubectl patch deploy --type json --patch '[{"op":"add","path": "/spec/template/spec/containers/0/args/-","value":"--enable-ssl-passthrough"}]' ingress-nginx-controller -n ingress-nginx
+
+          kubectl -n ingress-nginx wait --timeout=300s --for=condition=Available deployments --all
+
+      - name: Setup Dnsmasq to resolve hostnames with domain name kind
+        run: |
+          # Based on https://sixfeetup.com/blog/local-development-with-wildcard-dns-on-linux
+          sudo apt-get install -y dnsmasq
+
+          sudo sed -i -E "s/#DNS=/DNS=127.0.0.2/" /etc/systemd/resolved.conf
+          sudo sed -i -E "s/#Domains=/Domains=~kind/" /etc/systemd/resolved.conf
+          sudo systemctl restart systemd-resolved
+
+          sudo sed -i -E "s/#IGNORE_RESOLVCONF=yes/IGNORE_RESOLVCONF=yes/" /etc/default/dnsmasq
+          sudo sed -i -E "s/#listen-address=/listen-address=127.0.0.2/" /etc/dnsmasq.conf
+          sudo sed -i -E "s/#bind-interfaces/bind-interfaces/" /etc/dnsmasq.conf
+          sudo sed -i -E "s|#(address=).*|\1/kind/127.0.0.1|" /etc/dnsmasq.conf
+          sudo systemctl restart dnsmasq
+          systemctl status dnsmasq
+
+      - name: Set env variables for tests to properly leverage KinD cluster  
+        run: |
+          echo "CLUSTER_TYPE=KIND" >> $GITHUB_ENV
+          echo "CLUSTER_HOSTNAME=kind" >> $GITHUB_ENV
 
       - name: Install NVidia GPU operator for KinD
         uses: ./common/github-actions/nvidia-gpu-operator