Adding Multus and Calico CNI to Minikube (#51)

* Adding Multus and Calico CNI to Minikube

* PR review Jun

* addind doc for Multus

* PR review Jun and Salah

* Fix yamllint

* fix

* PR review Jun

* Fix

* namespace improvement

* Moving to docker,  improved reliability, and support for multi-namespaces

* fix minikube

* merge

* PR review Jun and fix for minikube bug

* Wait for pods to be terminated after scale in

Co-authored-by: Jun Chen <[email protected]>
Co-authored-by: Salaheddine Hamadi <[email protected]>

Author: 3 people

.github/actions/start-minikube/action.yaml

@@ -8,7 +8,7 @@ inputs:
   minikube_driver:
     default: docker
   minikube_extra_options:
-    default: '--embed-certs --nodes 3'
+    default: '--embed-certs --nodes 3 --cni=calico'
   oc_kcm_timeout:
     default: 5m
 

Makefile

@@ -5,6 +5,7 @@
 # Deploys the partner and test pods and the operator
 install:
 	./scripts/fix-minikube-labels.sh
+	./scripts/deploy-multus-network.sh
 	./scripts/deploy-partner-pods.sh
 	./scripts/deploy-test-pods.sh
 	./scripts/deploy-hpa.sh

README.md

@@ -80,18 +80,22 @@ In order to run the local test setup, the following dependencies are needed:
 * [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/)
 
 ## Setup
+Install the latest minikube by following the instructions at:
+```https://minikube.sigs.k8s.io/docs/start/```
 
 To start minikube, issue the following command:
 
 ```shell-script
-minikube start --embed-certs --driver="virtualbox" --nodes 3
+minikube start --embed-certs --driver="virtualbox" --nodes 3 --network-plugin=cni --cni=calico 
 ```
 
 The `--embed-certs` flag will cause minikube to embed certificates directly in its kubeconfig file.  
 This will allow the minikube cluster to be reached directly from the container without the need of binding additional volumes for certificates.
 
 The `--nodes 3` flag creates a cluster with one master node and two worker nodes. This is to support anti-affinity and pod-recreation test case.
 
+The  `--network-plugin=cni --cni=calico` flags configure CNI support and installs Calico. This is required to install Multus later on.
+
 To avoid having to specify this flag, set the `embed-certs` configuration key:
 
 ```shell-script

scripts/deploy-debug-ds.sh

@@ -7,8 +7,7 @@ source $SCRIPT_DIR/init-env.sh
 export REDHAT_RHEL_REGISTRY="${REDHAT_RHEL_REGISTRY:-registry.redhat.io/rhel8}"
 # check if we're using minikube by looking for kube-apiserver-minikube pod
 # if it's minikube, don't install debug partner
-res=`oc version | grep  Server`
-if [[ -z "$res" ]]
+if $TNF_NON_OCP_CLUSTER
 then
   echo "minikube detected, skip installing debug daemonSet"
 else

scripts/deploy-minikube.sh

@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
 set -x
 minikube delete
-minikube start --driver=virtualbox --embed-certs --nodes 3
+
+# Minikube configures the Calico CNI to deploy multus 
+minikube start --driver=docker --embed-certs --nodes 3 --cni=calico --feature-gates="LocalStorageCapacityIsolation=false"

scripts/deploy-multus-network.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+# Initialization
+SCRIPT_DIR=$(dirname "$0")
+source $SCRIPT_DIR/init-env.sh
+
+if $TNF_NON_OCP_CLUSTER
+then
+  echo "minikube detected, deploying Multus, Calico CNI needed in Minikube for this to work"
+
+  rm -rf ./temp
+  git clone --depth 1 https://github.com/k8snetworkplumbingwg/multus-cni.git ./temp/multus-cni
+
+  # fix for dimensioning bug
+  sed 's/memory: "50Mi"/memory: "100Mi"/g' temp/multus-cni/deployments/multus-daemonset-thick-plugin.yml -i
+
+  # Deploy Multus
+  oc apply -f ./temp/multus-cni/deployments/multus-daemonset-thick-plugin.yml
+  
+  # Wait for all calico and multus daemonset pods to be running
+  oc rollout status daemonset calico-node -n kube-system  --timeout=$TNF_DEPLOYMENT_TIMEOUT
+  oc rollout status daemonset kube-multus-ds -n kube-system  --timeout=$TNF_DEPLOYMENT_TIMEOUT
+
+  # Creates the network attachment on eth0 (bridge) on partner namespace
+  mkdir -p ./temp
+  cat ./test-target/multus.yaml | RANGE_START="192.168.1.2" RANGE_END="192.168.1.49" $SCRIPT_DIR/mo > ./temp/rendered-multus.yaml
+  oc apply -f ./temp/rendered-multus.yaml
+  
+  # Creating a network attachment for the default namespace as well
+  cat ./test-target/multus.yaml | TNF_EXAMPLE_CNF_NAMESPACE=default RANGE_START="192.168.1.50" RANGE_END="192.168.1.99" $SCRIPT_DIR/mo > ./temp/rendered-multus.yaml
+  oc apply -f ./temp/rendered-multus.yaml
+
+  rm ./temp/rendered-multus.yaml
+  sleep 3
+else 
+  echo "Minukube not detected, Skipping Multus installation"
+fi

scripts/deploy-operator.sh

@@ -15,9 +15,6 @@ fi
 # Installing OLM
 $SCRIPT_DIR/install-olm.sh
 
-# Create namespace if it does not exist
-oc create namespace ${TNF_EXAMPLE_CNF_NAMESPACE} 2>/dev/null
-
 $SCRIPT_DIR/delete-operator.sh
 
 # Creates a secret if a pem file exists

scripts/deploy-partner-pods.sh

@@ -5,9 +5,18 @@ SCRIPT_DIR=$(dirname "$0")
 source $SCRIPT_DIR/init-env.sh
 
 mkdir -p ./temp
-cat ./test-partner/partner-deployment.yaml | $SCRIPT_DIR/mo > ./temp/rendered-partner-template.yaml
+cat ./test-partner/partner-deployment.yaml | MULTUS_ANNOTATION=$MULTUS_ANNOTATION $SCRIPT_DIR/mo > ./temp/rendered-partner-template.yaml
 oc apply -f ./temp/rendered-partner-template.yaml
 rm ./temp/rendered-partner-template.yaml
 sleep 3
 
 oc wait deployment tnfpartner -n default --for=condition=available --timeout=$TNF_DEPLOYMENT_TIMEOUT
+
+# Minikube bug needs a scale in/out to reflect the multus changes, bug?
+oc scale --replicas=0 deployment tnfpartner -n default
+# wait for the pods to be really terminated
+kubectl wait  --for=delete pod --selector=app=tnfpartner --timeout=$TNF_DEPLOYMENT_TIMEOUT
+# sacle out again
+oc scale --replicas=1 deployment tnfpartner -n default
+# wait for the deployment to be available
+oc wait deployment tnfpartner -n default --for=condition=available --timeout=$TNF_DEPLOYMENT_TIMEOUT

scripts/deploy-test-pods.sh

@@ -4,13 +4,15 @@
 SCRIPT_DIR=$(dirname "$0")
 source $SCRIPT_DIR/init-env.sh
 
-# Create namespace if it does not exist
-oc create namespace ${TNF_EXAMPLE_CNF_NAMESPACE} 2>/dev/null
-
 mkdir -p ./temp
-cat ./test-target/local-pod-under-test.yaml | $SCRIPT_DIR/mo > ./temp/rendered-local-pod-under-test-template.yaml
+cat ./test-target/local-pod-under-test.yaml | MULTUS_ANNOTATION=$MULTUS_ANNOTATION $SCRIPT_DIR/mo > ./temp/rendered-local-pod-under-test-template.yaml
 oc apply -f ./temp/rendered-local-pod-under-test-template.yaml
 rm ./temp/rendered-local-pod-under-test-template.yaml
 sleep 3
 
-oc wait deployment test -n $TNF_EXAMPLE_CNF_NAMESPACE --for=condition=available --timeout=$TNF_DEPLOYMENT_TIMEOUT
+oc wait deployment test -n $TNF_EXAMPLE_CNF_NAMESPACE --for=condition=available --timeout=$TNF_DEPLOYMENT_TIMEOUT
+oc scale --replicas=0 deployment test -n $TNF_EXAMPLE_CNF_NAMESPACE 
+
+# Minikube bug needs another scale out?
+oc wait deployment test -n $TNF_EXAMPLE_CNF_NAMESPACE --for=condition=available --timeout=$TNF_DEPLOYMENT_TIMEOUT
+oc scale --replicas=2 deployment test -n $TNF_EXAMPLE_CNF_NAMESPACE 

scripts/fix-minikube-labels.sh

@@ -4,8 +4,7 @@
 SCRIPT_DIR=$(dirname "$0")
 source $SCRIPT_DIR/init-env.sh
 
-res=`oc version | grep  Server`
-if [ -z "$res" ]
+if $TNF_NON_OCP_CLUSTER
 then
   echo "minikube detected, applying worker labels on all nodes"
   oc get nodes -oname | xargs -I{} oc label  {}  node-role.kubernetes.io/worker=worker --overwrite