Skip to content

Commit a7d8c6c

Browse files
thatmattlongthatmattlong
authored
test: add azure cni overlay e2e (#2073)
1 parent a20fb84 commit a7d8c6c

File tree

8 files changed

+452
-54
lines changed

8 files changed

+452
-54
lines changed

‎.pipelines/pipeline.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -329,6 +329,14 @@ stages:
329329
testDropgz: ""
330330
clusterName: "overlaye2e"
331331

332+
- template: singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml
333+
parameters:
334+
name: "azure_cni_overlay_e2e"
335+
displayName: Azure CNI Overlay
336+
pipelineBuildImage: "$(BUILD_IMAGE)"
337+
testDropgz: ""
338+
clusterName: "azurecnioverlaye2e"
339+
332340
- template: singletenancy/aks-swift/e2e-job-template.yaml
333341
parameters:
334342
name: "aks_swift_e2e"

‎.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
parameters:
2+
name: ""
3+
displayName: ""
4+
pipelineBuildImage: "$(BUILD_IMAGE)"
5+
testDropgz: ""
6+
clusterName: ""
7+
8+
stages:
9+
- stage: ${{ parameters.name }}
10+
displayName: E2E - ${{ parameters.displayName }}
11+
dependsOn:
12+
- setup
13+
- publish
14+
jobs:
15+
- job: ${{ parameters.name }}
16+
displayName: Azure CNI Overlay Test Suite - (${{ parameters.name }})
17+
timeoutInMinutes: 120
18+
pool:
19+
name: $(BUILD_POOL_NAME_DEFAULT)
20+
demands:
21+
- agent.os -equals Linux
22+
- Role -equals $(CUSTOM_E2E_ROLE)
23+
variables:
24+
GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path
25+
GOBIN: "$(GOPATH)/bin" # Go binaries path
26+
modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking"
27+
steps:
28+
- template: azure-cni-overlay-e2e-step-template.yaml
29+
parameters:
30+
name: ${{ parameters.name }}
31+
testDropgz: ${{ parameters.testDropgz }}
32+
clusterName: ${{ parameters.clusterName }}

‎.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
parameters:
2+
name: ""
3+
testDropgz: ""
4+
clusterName: ""
5+
6+
steps:
7+
- bash: |
8+
echo $UID
9+
sudo rm -rf $(System.DefaultWorkingDirectory)/*
10+
displayName: "Set up OS environment"
11+
12+
- checkout: self
13+
14+
- bash: |
15+
go version
16+
go env
17+
mkdir -p '$(GOBIN)'
18+
mkdir -p '$(GOPATH)/pkg'
19+
mkdir -p '$(modulePath)'
20+
echo '##vso[task.prependpath]$(GOBIN)'
21+
echo '##vso[task.prependpath]$(GOROOT)/bin'
22+
name: "GoEnv"
23+
displayName: "Set up the Go environment"
24+
25+
- task: AzureCLI@1
26+
inputs:
27+
azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION)
28+
scriptLocation: "inlineScript"
29+
scriptType: "bash"
30+
addSpnToEnvironment: true
31+
inlineScript: |
32+
set -e
33+
mkdir -p ~/.kube/
34+
echo "Create AKS Overlay cluster"
35+
make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_OVERLAY_CLUSTER_TEST)
36+
make -C ./hack/aks overlay-byocni-up AZCLI=az REGION=$(REGION_OVERLAY_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) VM_SIZE=Standard_B2ms
37+
echo "Cluster successfully created"
38+
displayName: Create Overlay cluster
39+
condition: succeeded()
40+
41+
- script: |
42+
ls -lah
43+
pwd
44+
echo "installing kubectl"
45+
curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
46+
sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
47+
kubectl cluster-info
48+
kubectl get po -owide -A
49+
name: "installKubectl"
50+
displayName: "Install Kubectl"
51+
52+
- script: |
53+
echo "install kubetest2 and gsutils"
54+
go get github.com/onsi/ginkgo/ginkgo
55+
go get github.com/onsi/gomega/...
56+
go install github.com/onsi/ginkgo/ginkgo@latest
57+
go install sigs.k8s.io/kubetest2@latest
58+
go install sigs.k8s.io/kubetest2/kubetest2-noop@latest
59+
go install sigs.k8s.io/kubetest2/kubetest2-tester-ginkgo@latest
60+
wget https://storage.googleapis.com/pub/gsutil.tar.gz
61+
tar xfz gsutil.tar.gz
62+
sudo mv gsutil /usr/local/bin
63+
name: "installKubetest"
64+
displayName: "Set up Conformance Tests"
65+
66+
- script: |
67+
echo "Start Integration Tests on Overlay Cluster"
68+
echo "deploy ip-masq-agent for overlay"
69+
kubectl apply -f test/integration/manifests/ip-masq-agent/ip-masq-agent.yaml --validate=false
70+
cd test/integration/manifests/ip-masq-agent/
71+
kubectl apply -f config-custom.yaml
72+
kubectl apply -f config-reconcile.yaml
73+
cd ../../../..
74+
kubectl get po -owide -A
75+
CNS=$(make cns-version) DROPGZ=$(make cni-dropgz-version)
76+
sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=${CNS} CNI_DROPGZ_VERSION=${DROPGZ} INSTALL_CNS=true INSTALL_AZURE_CNI_OVERLAY=true TEST_DROPGZ=${{ parameters.testDropgz }}
77+
retryCountOnTaskFailure: 3
78+
name: "integrationTest"
79+
displayName: "Run CNS Integration Tests on AKS Overlay"
80+
81+
- script: |
82+
echo "Status of the nodes and pods after the test"
83+
kubectl get nodes -o wide
84+
kubectl get pods -A -o wide
85+
echo "Logs will be available as a build artifact"
86+
ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/
87+
echo $ARTIFACT_DIR
88+
sudo rm -rf $ARTIFACT_DIR
89+
sudo mkdir $ARTIFACT_DIR
90+
sudo cp test/integration/logs/* $ARTIFACT_DIR
91+
name: "GetLogs"
92+
displayName: "Get logs"
93+
condition: always()
94+
95+
- task: PublishBuildArtifacts@1
96+
inputs:
97+
artifactName: test-output
98+
pathtoPublish: "$(Build.ArtifactStagingDirectory)/test-output"
99+
condition: always()
100+
101+
- script: |
102+
echo "Run Service Conformance E2E"
103+
export PATH=${PATH}:/usr/local/bin/gsutil
104+
KUBECONFIG=~/.kube/config kubetest2 noop \
105+
--test ginkgo -- \
106+
--focus-regex "Services.*\[Conformance\].*"
107+
name: "servicesConformance"
108+
displayName: "Run Services Conformance Tests"
109+
110+
- script: |
111+
echo "Run wireserver and metadata connectivity Tests"
112+
bash test/network/wireserver_metadata_test.sh
113+
retryCountOnTaskFailure: 3
114+
name: "WireserverMetadataConnectivityTests"
115+
displayName: "Run Wireserver and Metadata Connectivity Tests"
116+
117+
- script: |
118+
ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/
119+
echo $ARTIFACT_DIR
120+
sudo rm -rf $ARTIFACT_DIR
121+
sudo rm -rf test/integration/logs
122+
name: "Cleanupartifactdir"
123+
displayName: "Cleanup artifact dir"
124+
condition: always()
125+
126+
- task: AzureCLI@2
127+
inputs:
128+
azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION)
129+
scriptLocation: "inlineScript"
130+
scriptType: "bash"
131+
addSpnToEnvironment: true
132+
inlineScript: |
133+
set -e
134+
echo "Deleting cluster"
135+
make -C ./hack/aks azcfg AZCLI=az
136+
make -C ./hack/aks down SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision)
137+
echo "Cluster and resources down"
138+
name: "Cleanupcluster"
139+
displayName: "Cleanup cluster"
140+
condition: always()

‎test/integration/manifests/cnsconfig/azurecnioverlayconfigmap.yaml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: cns-config
5+
namespace: kube-system
6+
data:
7+
cns_config.json: |
8+
{
9+
"TelemetrySettings": {
10+
"TelemetryBatchSizeBytes": 16384,
11+
"TelemetryBatchIntervalInSecs": 15,
12+
"RefreshIntervalInSecs": 15,
13+
"DisableAll": false,
14+
"HeartBeatIntervalInMins": 30,
15+
"DebugMode": false,
16+
"SnapshotIntervalInMins": 60
17+
},
18+
"ManagedSettings": {
19+
"PrivateEndpoint": "",
20+
"InfrastructureNetworkID": "",
21+
"NodeID": "",
22+
"NodeSyncIntervalInSeconds": 30
23+
},
24+
"ChannelMode": "CRD",
25+
"InitializeFromCNI": true,
26+
"ManageEndpointState": false,
27+
"ProgramSNATIPTables" : false,
28+
"EnableCNIConflistGeneration": true,
29+
"CNIConflistFilepath": "/etc/cni/net.d/15-azure-swift-overlay.conflist",
30+
"CNIConflistScenario": "v4overlay"
31+
}

‎test/integration/manifests/ip-masq-agent/config-custom.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
apiVersion: v1
22
kind: ConfigMap
33
metadata:
4-
name: ip-masq-agent-config
4+
name: azure-ip-masq-agent-config
55
namespace: kube-system
66
labels:
77
component: ip-masq-agent

‎test/integration/manifests/ip-masq-agent/config-reconcile.yaml‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,10 @@
11
apiVersion: v1
22
kind: ConfigMap
33
metadata:
4-
name: ip-masq-agent-config-reconciled
4+
name: azure-ip-masq-agent-config-user
55
namespace: kube-system
66
labels:
77
component: ip-masq-agent
8-
kubernetes.io/cluster-service: "true"
9-
addonmanager.kubernetes.io/mode: Reconcile
108
data:
119
ip-masq-agent-reconciled: |-
1210
nonMasqueradeCIDRs:

‎test/integration/manifests/ip-masq-agent/ip-masq-agent.yaml‎

Lines changed: 70 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,52 +1,91 @@
11
apiVersion: apps/v1
22
kind: DaemonSet
33
metadata:
4-
name: ip-masq-agent
4+
name: azure-ip-masq-agent-user
55
namespace: kube-system
6-
labels:
7-
component: ip-masq-agent
8-
kubernetes.io/cluster-service: "true"
9-
addonmanager.kubernetes.io/mode: Reconcile
106
spec:
117
selector:
128
matchLabels:
13-
k8s-app: ip-masq-agent
9+
k8s-app: azure-ip-masq-agent-user
1410
template:
1511
metadata:
1612
labels:
17-
k8s-app: ip-masq-agent
13+
k8s-app: azure-ip-masq-agent-user
1814
spec:
19-
hostNetwork: true
15+
affinity:
16+
nodeAffinity:
17+
requiredDuringSchedulingIgnoredDuringExecution:
18+
nodeSelectorTerms:
19+
- matchExpressions:
20+
- key: kubernetes.azure.com/cluster
21+
operator: Exists
22+
- key: type
23+
operator: NotIn
24+
values:
25+
- virtual-kubelet
26+
- key: kubernetes.io/os
27+
operator: In
28+
values:
29+
- linux
30+
automountServiceAccountToken: false
2031
containers:
21-
- name: ip-masq-agent
22-
image: mcr.microsoft.com/aks/ip-masq-agent-v2:v0.1.1
23-
imagePullPolicy: Always
32+
- args:
33+
- --v=2
34+
- --resync-interval=60
35+
image: mcr.microsoft.com/aks/ip-masq-agent-v2:v0.1.7
36+
imagePullPolicy: IfNotPresent
37+
name: azure-ip-masq-agent
38+
resources:
39+
limits:
40+
cpu: 500m
41+
memory: 250Mi
42+
requests:
43+
cpu: 100m
44+
memory: 50Mi
2445
securityContext:
25-
privileged: false
2646
capabilities:
27-
add: ["NET_ADMIN", "NET_RAW"]
28-
# Uses projected volumes to merge all data in /etc/config
47+
add:
48+
- NET_ADMIN
49+
- NET_RAW
50+
terminationMessagePath: /dev/termination-log
51+
terminationMessagePolicy: File
2952
volumeMounts:
30-
- name: ip-masq-agent-volume
31-
mountPath: /etc/config
32-
readOnly: true
53+
- mountPath: /etc/config
54+
name: azure-ip-masq-agent-config-volume
55+
- mountPath: /run/xtables.lock
56+
name: iptableslock
57+
dnsPolicy: ClusterFirst
58+
hostNetwork: true
59+
priorityClassName: system-node-critical
60+
restartPolicy: Always
61+
terminationGracePeriodSeconds: 30
62+
tolerations:
63+
- key: CriticalAddonsOnly
64+
operator: Exists
65+
- effect: NoExecute
66+
operator: Exists
67+
- effect: NoSchedule
68+
operator: Exists
3369
volumes:
34-
- name: ip-masq-agent-volume
70+
- name: azure-ip-masq-agent-config-volume
3571
projected:
72+
defaultMode: 420
3673
sources:
37-
# Note these ConfigMaps must be created in the same namespace as the daemonset
38-
- configMap:
39-
name: ip-masq-agent-config
40-
optional: true
74+
- configMap:
4175
items:
42-
- key: ip-masq-agent
43-
path: ip-masq-agent
44-
mode: 444
45-
- configMap:
46-
name: ip-masq-agent-config-reconciled
76+
- key: ip-masq-agent
77+
mode: 444
78+
path: ip-masq-agent
79+
name: azure-ip-masq-agent-config
4780
optional: true
81+
- configMap:
4882
items:
49-
# Avoiding duplicate paths
50-
- key: ip-masq-agent-reconciled
51-
path: ip-masq-agent-reconciled
52-
mode: 444
83+
- key: ip-masq-agent-reconciled
84+
mode: 444
85+
path: ip-masq-agent-reconciled
86+
name: azure-ip-masq-agent-config-user
87+
optional: true
88+
- hostPath:
89+
path: /run/xtables.lock
90+
type: FileOrCreate
91+
name: iptableslock

0 commit comments

Comments
 (0)