[CI] Add Cyclonus to merge validation (#860)

Author: matmerr

.github/workflows/cyclonus-netpol-test.yaml

@@ -0,0 +1,54 @@
+name: Cyclonus Network Policy Test
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+  pull_request:
+  schedule:
+    # run once a day at midnight
+    - cron: '0 0 * * *'
+
+jobs:
+  cyclonus-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Make NPM image
+        run: |
+          VERSION=cyclonus make azure-npm-image
+
+      - name: Setup Kind
+        uses: engineerd/[email protected]
+        with:
+          version: "v0.9.0"
+          config: ./test/kind/kind.yaml
+          name: npm-kind
+      
+      - name: Install Azure NPM
+        run: |
+          sed -i 's/mcr.microsoft.com\/containernetworking\/azure-npm:v1.3.1/acnpublic.azurecr.io\/azure-npm:cyclonus/' ./npm/azure-npm.yaml
+          kind load docker-image acnpublic.azurecr.io/azure-npm:cyclonus --name npm-kind
+          kubectl apply -f ./npm/azure-npm.yaml
+
+      - name: Run Cyclonus network policy test
+        run: make test-cyclonus
+
+      - name: Fetch logs
+        if: always()
+        run: |
+          kubectl logs -n kube-system -l k8s-app=azure-npm --tail -1 --prefix > npm-logs.txt
+          mv ./test/cyclonus/cyclonus-test.txt ./cyclonus-test.txt
+
+      - name: 'Upload Logs'
+        uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: logs
+          path: |
+            ./npm-logs.txt
+            ./cyclonus-test.txt
+

Makefile

@@ -461,4 +461,14 @@ test-all:
 # run all tests
 .PHONY: test-integration
 test-integration:
-	go test -coverpkg=./... -v -race -covermode atomic -coverprofile=coverage.out -tags=integration ./test/integration...
+	go test -coverpkg=./... -v -race -covermode atomic -coverprofile=coverage.out -tags=integration ./test/integration...
+
+.PHONY: test-cyclonus
+test-cyclonus:
+	cd test/cyclonus && bash ./test-cyclonus.sh
+	cd ..
+
+.PHONY: kind
+kind:
+	kind create cluster --config ./test/kind/kind.yaml
+	

README.md

@@ -1,7 +1,13 @@
 # Microsoft Azure Container Networking
 
 [![Build Status](https://msazure.visualstudio.com/One/_apis/build/status/Custom/Networking/ContainerNetworking/Azure.azure-container-networking?branchName=master)](https://msazure.visualstudio.com/One/_build/latest?definitionId=95007&branchName=master) [![Go Report Card](https://goreportcard.com/badge/github.com/Azure/azure-container-networking)](https://goreportcard.com/report/github.com/Azure/azure-container-networking) ![GitHub release](https://img.shields.io/github/release/Azure/azure-container-networking.svg)
-[![codecov](https://codecov.io/gh/Azure/azure-container-networking/branch/master/graph/badge.svg)](https://codecov.io/gh/Azure/azure-container-networking)
+
+| Azure Network Policy Manager Conformance      |  |
+| ----------- | ----------- |
+| Cyclonus Network Policy Suite      | [![Cyclonus Network Policy Test](https://github.com/Azure/azure-container-networking/actions/workflows/cyclonus-netpol-test.yaml/badge.svg?branch=master)](https://github.com/Azure/azure-container-networking/actions/workflows/cyclonus-netpol-test.yaml)       |
+| Kubernetes Network Policy E2E  | [![Build Status](https://dev.azure.com/msazure/One/_apis/build/status/Custom/Networking/ContainerNetworking/NPM%20Conformance%20Tests?branchName=master)](https://dev.azure.com/msazure/One/_build/latest?definitionId=195725&branchName=master)  |
+
+
 
 ## Overview
 This repository contains container networking services and plugins for Linux and Windows containers running on Azure:

npm/azure-npm.yaml

@@ -6,7 +6,7 @@ metadata:
   labels:
     addonmanager.kubernetes.io/mode: EnsureExists
 ---
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
   name: azure-npm
@@ -33,7 +33,7 @@ rules:
       - list
       - watch
 ---
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
   name: azure-npm-binding
@@ -77,12 +77,10 @@ spec:
         effect: NoSchedule
       - key: CriticalAddonsOnly
         operator: Exists
-      nodeSelector:
-        beta.kubernetes.io/os: linux
-        kubernetes.io/role: agent
+
       containers:
         - name: azure-npm
-          image: mcr.microsoft.com/containernetworking/azure-npm:v1.1.5
+          image: mcr.microsoft.com/containernetworking/azure-npm:v1.3.1
           resources:
             limits:
               cpu: 250m

test/cyclonus/install-cyclonus.yaml

@@ -0,0 +1,28 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: cyclonus
+  namespace: kube-system
+  labels:
+    app: cyclonus
+spec:
+  template:
+    spec:
+      restartPolicy: Never
+      containers:
+        - command:
+            - ./cyclonus
+            - generate
+            - --noisy=true
+            - --ignore-loopback=true
+            - --cleanup-namespaces=true
+            - --perturbation-wait-seconds=5
+            - --pod-creation-timeout-seconds=20
+            - --job-timeout-seconds=2
+            - --server-protocol=TCP,UDP
+            - --server-port=80
+          name: cyclonus
+          imagePullPolicy: IfNotPresent
+          image: mfenwick100/cyclonus:v0.4.4
+      serviceAccount: cyclonus
+      

test/cyclonus/test-cyclonus.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+set -xv
+
+kubectl delete --ignore-not-found=true clusterrolebinding cyclonus 
+kubectl delete --ignore-not-found=true sa cyclonus -n kube-system
+kubectl delete --ignore-not-found=true -f ./install-cyclonus.yaml
+kubectl delete --ignore-not-found=true ns x y z
+
+sleep 5
+
+# set up cyclonus
+kubectl create clusterrolebinding cyclonus --clusterrole=cluster-admin --serviceaccount=kube-system:cyclonus
+kubectl create sa cyclonus -n kube-system
+kubectl create -f ./install-cyclonus.yaml
+
+time kubectl wait --for=condition=ready --timeout=1m pod -n kube-system -l job-name=cyclonus
+
+#!/bin/bash
+{ kubectl logs -f -n kube-system job.batch/cyclonus;  } &
+{ time kubectl wait --for=condition=completed --timeout=600m pod -n kube-system -l job-name=cyclonus;  } &
+wait -n
+pkill -P $$
+echo done
+
+# grab the job logs
+LOG_FILE=cyclonus-test.txt
+kubectl logs -n kube-system job.batch/cyclonus | tee "$LOG_FILE"
+cat "$LOG_FILE"
+
+kubectl delete --ignore-not-found=true clusterrolebinding cyclonus 
+kubectl delete --ignore-not-found=true sa cyclonus -n kube-system
+kubectl delete --ignore-not-found=true -f ./install-cyclonus.yaml
+
+# if 'failure' is in the logs, fail; otherwise succeed
+rc=0
+
+cat "$LOG_FILE" | grep "failed" > /dev/null 2>&1 || rc=$?
+echo $rc
+if [ $rc -eq 0 ]; then
+    exit 1
+fi

test/kind/kind.yaml

@@ -0,0 +1,13 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+  - role: control-plane
+    image: kindest/node:v1.19.1
+  - role: worker
+    image: kindest/node:v1.19.1
+  - role: worker
+    image: kindest/node:v1.19.1
+networking:
+  ipFamily: ipv4
+  podSubnet: "10.10.0.0/16"
+  serviceSubnet: "10.11.0.0/16"