Add support for deploying test k8s cluster natively on arm64 architectures (#433)

* Add support for deploying test k8s cluster on arm64 architectures
* Addressing comments from Brandon and David
* Fix Typo
* revert bumping operator version by accident
* Addressing comment from Brandon, re-adding TNF_CONTAINER_CLIENT="docker"

Author: edcdavid

scripts/deploy-cr-scale-operator.sh

@@ -11,7 +11,7 @@ source "$SCRIPT_DIR"/logging.sh
 # Variables for deployment
 CR_SCALE_OPERATOR_GIT_REPO="https://github.com/test-network-function/cr-scale-operator.git"
 TAG="main"
-IMG="quay.io/testnetworkfunction/cr-scale-operator"
+IMG="quay.io/testnetworkfunction/cr-scale-operator:latest"
 CR_SCALE_OPERATOR_DIR=cr-scale-operator
 
 # Clone the repo.

scripts/deploy-multus-network.sh

@@ -24,10 +24,16 @@ if $TNF_NON_OCP_CLUSTER; then
 	# Install macvlan and other default plugins
 	echo "## install CNIs"
 	pushd temp/multus-cni/e2e || exit
-	./get_tools.sh
+	if [ "$CPU_ARCH" != arm64 ]; then
+		./get_tools.sh
+	fi
 	./generate_yamls.sh
 	popd || exit
 
+	if [ "$CPU_ARCH" == arm64 ]; then
+		sed -i 's/amd64/arm64/g' temp/multus-cni/e2e/yamls/cni-install.yml
+	fi
+
 	# Temporarily commenting this out as we are currently not running this in a non-allowlisted environment
 	# sed 's/alpine/quay.io\/jitesoft\/alpine:latest/g' temp/multus-cni/e2e/yamls/cni-install.yml -i
 	kubectl create -f temp/multus-cni/e2e/yamls/cni-install.yml

scripts/deploy-statefulset-test-pods.sh

@@ -15,6 +15,7 @@ NUM_NODES="${NUM_NODES%"${NUM_NODES##*[![:space:]]}"}"
 if [[ $NUM_NODES == 1 ]]; then
 	REPLICAS=1
 fi
+
 APP="testss" RESOURCE_TYPE="StatefulSet" MULTUS_ANNOTATION=$MULTUS_ANNOTATION REPLICAS=$REPLICAS "$SCRIPT_DIR"/mo ./test-target/local-pod-under-test.yaml >./temp/rendered-local-statefulset-pod-under-test-template.yaml
 oc apply --filename ./temp/rendered-local-statefulset-pod-under-test-template.yaml
 rm ./temp/rendered-local-statefulset-pod-under-test-template.yaml

scripts/deploy-test-pods.sh

@@ -16,6 +16,7 @@ if [[ $NUM_NODES == 1 ]]; then
 	REPLICAS=1
 fi
 "$SCRIPT_DIR"/mo ./test-target/test-service-account.yaml >./temp/rendered-test-service-account-template.yaml
+
 APP="testdp" RESOURCE_TYPE="Deployment" MULTUS_ANNOTATION=$MULTUS_ANNOTATION REPLICAS=$REPLICAS "$SCRIPT_DIR"/mo ./test-target/local-pod-under-test.yaml >./temp/rendered-local-pod-under-test-template.yaml
 oc apply --filename ./temp/rendered-test-service-account-template.yaml
 oc apply --filename ./temp/rendered-local-pod-under-test-template.yaml

scripts/init-env.sh

@@ -73,7 +73,7 @@ oc version | grep Server >/dev/null || {
 # create Multus annotations
 create_multus_annotation() {
 	for ((NUM = 0; NUM < MULTUS_IF_NUM; NUM++)); do
-		MULTUS_ANNOTATION="${MULTUS_ANNOTATION}{ \"name\" : \"${NET_NAME}-$1-${NUM}\" },"
+		MULTUS_ANNOTATION="${MULTUS_ANNOTATION}${NET_NAME}-$1-${NUM},"
 	done
 }
 
@@ -86,9 +86,12 @@ if $TNF_NON_OCP_CLUSTER; then
 
 	# IPv6
 	create_multus_annotation ipv6
-	if [ "$NUM" -ge 0 ]; then
-		# Remove last comma. Works on bash 3.2.4 on OS X:
-		#  https://unix.stackexchange.com/questions/144298/delete-the-last-character-of-a-string-using-string-manipulation-in-shell-script
-		export MULTUS_ANNOTATION="'[ ${MULTUS_ANNOTATION::-1} ]'"
-	fi
+
+	# Remove the last character (comma)
+	#  https://unix.stackexchange.com/questions/144298/delete-the-last-character-of-a-string-using-string-manipulation-in-shell-script
+	MULTUS_ANNOTATION="${MULTUS_ANNOTATION%?}"
 fi
+
+CPU_ARCH="$(uname -m)"
+export CPU_ARCH
+echo "CPU_ARCH=$CPU_ARCH"

scripts/preload-images.sh

@@ -17,7 +17,7 @@ set -e
 
 # Create array for images to preload
 IMAGES_TO_PRELOAD=(
-	httpd:2.4.58
+	docker.io/library/httpd:2.4.58
 	"$COMMUNITY_OPERATOR_IMAGEREPO"/"$COMMUNITY_OPERATOR_BASE":"$COMMUNITY_OPERATOR_IMAGEVERSION"
 	quay.io/testnetworkfunction/cnf-test-partner:latest
 	quay.io/testnetworkfunction/debug-partner:latest
@@ -26,7 +26,6 @@ IMAGES_TO_PRELOAD=(
 	quay.io/calico/node:v3.28.0
 	quay.io/testnetworkfunction/nginx-operator-bundle:v0.0.1
 	ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot
-	ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot-thick
 	registry.access.redhat.com/ubi9/ubi:latest
 	registry.access.redhat.com/ubi9/ubi-minimal:latest
 	quay.io/operator-framework/configmap-operator-registry:latest
@@ -37,6 +36,6 @@ IMAGES_TO_PRELOAD=(
 
 # Preload images
 for image in "${IMAGES_TO_PRELOAD[@]}"; do
-	docker pull "$image"
-	kind load docker-image "$image"
+	${CONTAINER_CLIENT} pull "$image"
+	${CONTAINER_CLIENT} save "$image" -o image.tar && kind load image-archive image.tar && rm image.tar
 done

test-target/local-pod-under-test.yaml

@@ -20,7 +20,7 @@ spec:
       annotations:
         test-network-function.com/container_tests: '["PRIVILEGED_POD","PRIVILEGED_ROLE"]' # optional
         test-network-function.com/defaultnetworkinterface: '"eth0"' # optional
-        k8s.v1.cni.cncf.io/networks: {{ MULTUS_ANNOTATION }}
+        k8s.v1.cni.cncf.io/networks: '{{ MULTUS_ANNOTATION }}'
       name: test
     spec:
       terminationGracePeriodSeconds: 30