test: Add Swift Testing Pipeline (#712)

* Add new Swift test scenarios

Author: matmerr

.pipelines/Dockerfile

@@ -21,4 +21,5 @@ RUN go get github.com/gorilla/mux
 RUN go get github.com/jstemmer/go-junit-report
 RUN go get github.com/axw/gocov/gocov
 RUN go get github.com/AlekSi/gocov-xml
-RUN go get -u gopkg.in/matm/v1/gocov-html
+RUN go get -u gopkg.in/matm/v1/gocov-html
+RUN go get -u github.com/onsi/ginkgo/ginkgo

.pipelines/pipeline.yaml

@@ -16,7 +16,9 @@ stages:
         displayName: CNI, NPM, CNS, CNM
         pool:
           name: Networking-ContainerNetworking
-          demands: agent.os -equals Linux
+          demands: 
+          - agent.os -equals Linux
+          - Role -equals Build
 
         container:
           image: "$(BUILD_IMAGE)" # build image set as variable in pipeline runtime for flexibility
@@ -58,11 +60,7 @@ stages:
               echo "##vso[task.setvariable variable=StorageID;isOutput=true]$(echo ${BUILD_NUMBER//./-})"
               echo "##vso[task.setvariable variable=CommitHash;isOutput=true]$(git rev-parse HEAD)"
               echo "##vso[task.setvariable variable=Tag;isOutput=true]$(git describe --tags --always --dirty)"
-              export BRANCH=$(Build.SourceBranchName)
-              echo Building branch $BRANCH from $(Build.SourceBranch) 
-              if [[ "$BRANCH" == "master" ]]; then
-                echo "##vso[task.setvariable variable=Tag;isOutput=true]$(git describe --tags --abbrev=0)"
-              fi
+              echo "##vso[task.setvariable variable=ImageTag;isOutput=true]$(git describe --tags --always --dirty)-test"
             workingDirectory: "$(modulePath)"
             name: "EnvironmentalVariables"
             displayName: "Set environmental variables"
@@ -75,13 +73,12 @@ stages:
               go get github.com/axw/gocov/gocov
               go get github.com/AlekSi/gocov-xml
               go get github.com/matm/gocov-html
+              go get github.com/rakyll/gotest
             workingDirectory: "$(modulePath)"
             name: "GoDependencies"
             displayName: "Install Go dependencies"
 
           - script: |
-              echo Building in $(pwd)
-              echo Build tag is $(EnvironmentalVariables.Tag)
               export GOOS=windows
               make all-binaries VERSION=$(EnvironmentalVariables.Tag)
             workingDirectory: "$(modulePath)"
@@ -109,11 +106,11 @@ stages:
             displayName: "Run Tests"
 
           - script: |
-              make all-binaries VERSION=$(EnvironmentalVariables.Tag)
-              sudo make azure-npm-image VERSION=$(EnvironmentalVariables.Tag)
+              sudo make hack-images VERSION=$(EnvironmentalVariables.Tag)
+              sudo make all-images VERSION=$(EnvironmentalVariables.Tag)
             workingDirectory: "$(modulePath)"
-            name: "BuildNPMImage"
-            displayName: "Build NPM Image"
+            name: "BuildImages"
+            displayName: "Build Images"
 
           - bash: |
               export CODECOV_TOKEN=$(CODECOV_TOKEN)
@@ -130,6 +127,12 @@ stages:
               docker tag $IMAGE_REGISTRY/azure-npm:$(EnvironmentalVariables.Tag) $IMAGE_REGISTRY/azure-npm:$(EnvironmentalVariables.Tag)-test
               docker push $IMAGE_REGISTRY/azure-npm:$(EnvironmentalVariables.Tag)-test
 
+              docker tag $IMAGE_REGISTRY/azure-cns:$(EnvironmentalVariables.Tag) $IMAGE_REGISTRY/azure-cns:$(EnvironmentalVariables.Tag)-test
+              docker push $IMAGE_REGISTRY/azure-cns:$(EnvironmentalVariables.Tag)-test
+
+              docker tag $IMAGE_REGISTRY/azure-cni-manager:$(EnvironmentalVariables.Tag) $IMAGE_REGISTRY/azure-cni-manager:$(EnvironmentalVariables.Tag)-test
+              docker push $IMAGE_REGISTRY/azure-cni-manager:$(EnvironmentalVariables.Tag)-test
+
               function auto-retry()
               {
                   export i="1"
@@ -139,7 +142,10 @@ stages:
                     printf "Attempt $i/$attempts - " && "$@" && break ||  sleep 3 &&  i=$[$i+1] && false
                   done
               }
+
               auto-retry docker pull $IMAGE_REGISTRY/azure-npm:$(EnvironmentalVariables.Tag)-test
+              auto-retry docker pull $IMAGE_REGISTRY/azure-cns:$(EnvironmentalVariables.Tag)-test
+              auto-retry docker pull $IMAGE_REGISTRY/azure-cni-manager:$(EnvironmentalVariables.Tag)-test
             workingDirectory: "$(modulePath)"
             name: "mcrreplication"
             displayName: "Push NPM Image and Wait for Repository"
@@ -194,7 +200,13 @@ stages:
             displayName: Create artifact storage container
             condition: succeeded()
 
-  - template: e2e-job-template.yaml
+  - template: singletenancy/aks-swift/e2e-job-template.yaml
+    parameters:
+      name: "aks_on_swift_e2e"
+      displayName: AKS on Swift
+      pipelineBuildImage: "$(BUILD_IMAGE)"
+
+  - template: singletenancy/aks-engine/e2e-job-template.yaml
     parameters:
       name: "ubuntu_16_04_linux_e2e"
       displayName: Ubuntu 16.04
@@ -204,7 +216,7 @@ stages:
       clusterDefinitionCniBuildOS: "linux"
       clusterDefinitionCniBuildExt: ".tgz"
 
-  - template: e2e-job-template.yaml
+  - template: singletenancy/aks-engine/e2e-job-template.yaml
     parameters:
       name: "ubuntu_18_04_linux_e2e"
       displayName: Ubuntu 18.04
@@ -214,7 +226,7 @@ stages:
       clusterDefinitionCniBuildOS: "linux"
       clusterDefinitionCniBuildExt: ".tgz"
 
-  - template: e2e-job-template.yaml
+  - template: singletenancy/aks-engine/e2e-job-template.yaml
     parameters:
       name: "windows_18_09_e2e"
       displayName: "Windows 1809"
@@ -224,7 +236,7 @@ stages:
       clusterDefinitionCniBuildOS: "windows"
       clusterDefinitionCniBuildExt: ".zip"
 
-  - template: e2e-job-template.yaml
+  - template: singletenancy/aks-engine/e2e-job-template.yaml
     parameters:
       name: "windows_19_03_e2e"
       displayName: "Windows 1903"
@@ -237,6 +249,7 @@ stages:
   - stage: cleanup
     displayName: Cleanup
     dependsOn:
+      - "aks_on_swift_e2e"
       - "ubuntu_16_04_linux_e2e"
       - "ubuntu_18_04_linux_e2e"
       - "windows_18_09_e2e"
@@ -260,5 +273,7 @@ stages:
                 BUILD_NUMBER=${BUILD_NUMBER//./-}
                 echo Deleting storage container with name acn-$BUILD_NUMBER and account name $(STORAGE_ACCOUNT_NAME)
                 az storage container delete -n acn-$BUILD_NUMBER --account-name $(STORAGE_ACCOUNT_NAME)
+                echo Pruning old docker images...
+                sudo docker system prune -f
               workingDirectory: "$(modulePath)"
             displayName: Cleanup remote Azure storage container

.pipelines/e2e-job-template.yaml renamed to .pipelines/singletenancy/aks-engine/e2e-job-template.yaml

@@ -17,7 +17,9 @@ stages:
         timeoutInMinutes: 120
         pool:
           name: Networking-ContainerNetworking
-          demands: agent.os -equals Linux
+          demands: 
+          - agent.os -equals Linux
+          - Role -equals Build
         container:
           image: ${{ parameters.pipelineBuildImage }}
         variables:

.pipelines/e2e-step-template.yaml renamed to .pipelines/singletenancy/aks-engine/e2e-step-template.yaml

@@ -0,0 +1,29 @@
+parameters:
+  name: ""
+  displayName: ""
+  pipelineBuildImage: "$(BUILD_IMAGE)"
+
+stages:
+  - stage: ${{ parameters.name }}
+    displayName: E2E - ${{ parameters.displayName }}
+    dependsOn: build_and_test
+    jobs:
+      - job: ${{ parameters.name }}
+        displayName: Singletenancy AKS on Swift Suite - (${{ parameters.name }})
+        timeoutInMinutes: 120
+        pool:
+          name: Networking-ContainerNetworking
+          demands: 
+          - agent.os -equals Linux
+          - Role -equals $(CUSTOM_E2E_ROLE)
+        container:
+          image: "$(BUILD_IMAGE)" # build image set as variable in pipeline runtime for flexibilit
+
+        variables:
+          GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path
+          GOBIN: "$(GOPATH)/bin" # Go binaries path
+          ImageTag: $[ stagedependencies.build_and_test.unit_tests.outputs['EnvironmentalVariables.ImageTag'] ]
+        steps:
+          - template: e2e-step-template.yaml
+            parameters:
+              name: ${{ parameters.name }}

.pipelines/singletenancy/aks-swift/e2e-job-template.yaml

@@ -0,0 +1,46 @@
+parameters:
+  name: ""
+
+steps:
+  - bash: |
+      echo $UID
+      sudo rm -rf $(System.DefaultWorkingDirectory)/*
+    displayName: "Set up OS environment"
+
+  - checkout: self
+
+  - bash: |
+      go version
+      go env
+      mkdir -p '$(GOBIN)'
+      mkdir -p '$(GOPATH)/pkg'
+      mkdir -p '$(modulePath)'
+      echo '##vso[task.prependpath]$(GOBIN)'
+      echo '##vso[task.prependpath]$(GOROOT)/bin'
+    name: "GoEnv"
+    displayName: "Set up the Go environment"
+
+  - bash: |
+      echo $(ImageTag)
+    displayName: Echo Tag
+
+  - task: AzureCLI@1
+    inputs:
+      azureSubscription: $(ARTIFACT_SUBSCRIPTION)
+      scriptLocation: "inlineScript"
+      addSpnToEnvironment: true
+      inlineScript: |
+        mkdir -p ~/.kube/
+        az keyvault secret show --name $(KV_AKS_SWIFT_SECRET_NAME) --vault-name $(KV_NAME) --query value | sed 's/\\n/\n/g' | sed -e 's/^"//' -e 's/"$//'
+        az keyvault secret show --name $(KV_AKS_SWIFT_SECRET_NAME) --vault-name $(KV_NAME) --query value | sed 's/\\n/\n/g' | sed -e 's/^"//' -e 's/"$//' > ~/.kube/config
+    displayName: Pull Kubeconfig from Keyvault
+    condition: succeeded()
+
+  - script: |
+      ls -lah
+      pwd
+      sudo -E env "PATH=$PATH" VERSION=$(ImageTag) INSTALL_CNI=true INSTALL_CNS=true make test-integration
+    name: "aksswifte2e"
+    displayName: "Run AKS Swift E2E"
+
+

.pipelines/singletenancy/aks-swift/e2e-step-template.yaml

@@ -181,12 +181,13 @@ all-binaries: azure-cnm-plugin azure-cni-plugin azure-cns
 endif
 
 ifeq ($(GOOS),linux)
-all-images: azure-npm-image azure-vnet-telemetry-image
+all-images: azure-npm-image azure-cns-aks-swift-image 
 else
 all-images:
 	@echo "Nothing to build. Skip."
 endif
 
+
 # Clean all build artifacts.
 .PHONY: clean
 clean:
@@ -233,7 +234,7 @@ $(NPM_BUILD_DIR)/azure-npm$(EXE_EXT): $(NPMFILES)
 .PHONY: all-containerized
 all-containerized:
 	pwd && ls -l
-	docker build -f Dockerfile.build -t $(BUILD_CONTAINER_IMAGE):$(VERSION) .
+	docker build -f Dockerfile.build -t $(BUILD_CONTAINER_IMAGE):$(VERSION) --no-cache .
 	docker run --name $(BUILD_CONTAINER_NAME) \
 		-v /usr/bin/docker:/usr/bin/docker \
 		-v /var/run/docker.sock:/var/run/docker.sock \
@@ -269,7 +270,7 @@ hack-images:
 azure-vnet-plugin-image: azure-cnm-plugin
 	# Build the plugin image, keeping any old image during build for cache, but remove it afterwards.
 	docker images -q $(CNM_PLUGIN_ROOTFS):$(VERSION) > cid
-	docker build \
+	docker build --no-cache \
 		-f Dockerfile.cnm \
 		-t $(CNM_PLUGIN_ROOTFS):$(VERSION) \
 		--build-arg CNM_BUILD_DIR=$(CNM_BUILD_DIR) \
@@ -450,7 +451,15 @@ ifeq ($(GOOS),linux)
 	chown $(BUILD_USER):$(BUILD_USER) $(NPM_BUILD_DIR)/$(NPM_ARCHIVE_NAME)
 endif
 
+
+PRETTYGOTEST := $(shell command -v gotest 2> /dev/null)
+
 # run all tests
 .PHONY: test-all
 test-all:
-	go test -coverpkg=./... -v -race -covermode atomic -coverprofile=coverage.out ./...
+	go test -coverpkg=./... -v -race -covermode atomic -coverprofile=coverage.out ./...
+
+# run all tests
+.PHONY: test-integration
+test-integration:
+	go test -coverpkg=./... -v -race -covermode atomic -coverprofile=coverage.out -tags=integration ./test/integration...

Makefile

@@ -93,7 +93,7 @@ spec:
           effect: NoSchedule
       containers:
         - name: cns-container
-          image: mcr.microsoft.com/containernetworking/azure-cns:v1.2.0
+          image: mcr.microsoft.com/containernetworking/azure-cns:v1.2.0-alpha
           imagePullPolicy: IfNotPresent
           args: [ "-c", "tcp://$(CNSIpAddress):$(CNSPort)", "-t", "$(CNSLogTarget)"]
           volumeMounts:
@@ -158,4 +158,4 @@ data:
           "NodeSyncIntervalInSeconds": 30
       },
       "ChannelMode": "CRD"
-    }
+    }

cns/azure-cns.yaml

@@ -2,6 +2,7 @@ package cnsclient
 
 import (
 	"fmt"
+	"os"
 	"sort"
 	"strings"
 
@@ -20,7 +21,8 @@ const (
 	eth0InterfaceName   = "eth0"
 	azure0InterfaceName = "azure0"
 
-	defaultCNSURl = "http://localhost:10090"
+	envCNSIPAddress = "CNSIpAddress"
+	envCNSPort      = "CNSPort"
 )
 
 var (
@@ -36,7 +38,11 @@ var (
 )
 
 func HandleCNSClientCommands(cmd, arg string) error {
-	cnsClient, err := InitCnsClient(defaultCNSURl)
+
+	cnsIPAddress := os.Getenv(envCNSIPAddress)
+	cnsPort := os.Getenv(envCNSPort)
+
+	cnsClient, err := InitCnsClient("http://" + cnsIPAddress + ":" + cnsPort)
 	if err != nil {
 		return err
 	}