Skip to content

Commit 3a6959c

Browse files
behzad-mirbehzad-mir
authored
feat: adding stateless CNI pipeline test (#2914)
* feat: adding stateless CNI pipeline test * feat: making change for stateless CNI pipeline * feat: addressing the comments * fix: fixing stateles cni yaml * fix: stateless CNI delete fix * Addresing the comments * addresing john's comments
1 parent a9fccfa commit 3a6959c

File tree

8 files changed

+345
-23
lines changed

8 files changed

+345
-23
lines changed

‎.pipelines/pipeline.yaml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -330,6 +330,17 @@ stages:
330330
k8sVersion: ""
331331
dependsOn: "containerize"
332332

333+
# Azure Overlay E2E Stateless CNI tests
334+
- template: singletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e-job-template.yaml
335+
parameters:
336+
name: "azure_overlay_stateless_e2e"
337+
displayName: Azure Stateless CNI Overlay
338+
os: windows
339+
clusterType: overlay-byocni-up
340+
clusterName: "statelesswin"
341+
vmSize: Standard_B2ms
342+
dependsOn: "containerize"
343+
333344
# AKS Swift E2E tests
334345
- template: singletenancy/aks-swift/e2e-job-template.yaml
335346
parameters:
@@ -413,6 +424,7 @@ stages:
413424
dependsOn:
414425
- setup
415426
- azure_overlay_e2e
427+
- azure_overlay_stateless_e2e
416428
- aks_swift_e2e
417429
- cilium_e2e
418430
- cilium_overlay_e2e
@@ -448,6 +460,10 @@ stages:
448460
name: azure_overlay_e2e
449461
clusterName: "azovere2e"
450462
region: $(REGION_AKS_CLUSTER_TEST)
463+
azure_overlay_stateless_e2e:
464+
name: azure_overlay_stateless_e2e
465+
clusterName: "statelesswin"
466+
region: $(REGION_AKS_CLUSTER_TEST)
451467
aks_swift_e2e:
452468
name: aks_swift_e2e
453469
clusterName: "swifte2e"

‎.pipelines/singletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e-job-template.yaml‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
parameters:
2+
name: ""
3+
displayName: ""
4+
clusterType: ""
5+
clusterName: ""
6+
vmSize: ""
7+
k8sVersion: ""
8+
dependsOn: ""
9+
10+
stages:
11+
- stage: ${{ parameters.clusterName }}
12+
displayName: Create Cluster - ${{ parameters.displayName }}
13+
dependsOn:
14+
- ${{ parameters.dependsOn }}
15+
- setup
16+
pool:
17+
name: $(BUILD_POOL_NAME_DEFAULT)
18+
variables:
19+
commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ]
20+
jobs:
21+
- template: ../../templates/create-cluster.yaml
22+
parameters:
23+
name: ${{ parameters.name }}
24+
displayName: ${{ parameters.displayName }}
25+
clusterType: ${{ parameters.clusterType }}
26+
clusterName: ${{ parameters.clusterName }}-$(commitID)
27+
vmSize: ${{ parameters.vmSize }}
28+
k8sVersion: ${{ parameters.k8sVersion }}
29+
dependsOn: ${{ parameters.dependsOn }}
30+
region: $(REGION_AKS_CLUSTER_TEST)
31+
- job: windows_nodepool
32+
displayName: Add Windows Nodepool
33+
dependsOn: ${{ parameters.name }}
34+
pool:
35+
name: $(BUILD_POOL_NAME_DEFAULT)
36+
demands:
37+
- agent.os -equals Linux
38+
- Role -equals $(CUSTOM_E2E_ROLE)
39+
steps:
40+
- task: AzureCLI@2
41+
inputs:
42+
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
43+
scriptLocation: "inlineScript"
44+
scriptType: "bash"
45+
addSpnToEnvironment: true
46+
inlineScript: |
47+
set -e
48+
make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID)
49+
make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }}-$(commitID) VM_SIZE_WIN=${{ parameters.vmSize }}
50+
echo "Windows node are successfully added to v4 Overlay Cluster"
51+
kubectl cluster-info
52+
kubectl get node -owide
53+
kubectl get po -owide -A
54+
name: "Add_Windows_Node"
55+
displayName: "Add windows node on v4 overlay cluster"
56+
57+
- stage: ${{ parameters.name }}
58+
displayName: E2E - ${{ parameters.displayName }}
59+
dependsOn:
60+
- setup
61+
- publish
62+
- ${{ parameters.clusterName }}
63+
variables:
64+
commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ]
65+
GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path
66+
GOBIN: "$(GOPATH)/bin" # Go binaries path
67+
modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking"
68+
pool:
69+
name: $(BUILD_POOL_NAME_DEFAULT)
70+
jobs:
71+
- job: ${{ parameters.name }}_windows
72+
displayName: Azure Stateless CNI Overlay Test Suite | Windows - (${{ parameters.name }})
73+
pool:
74+
name: $(BUILD_POOL_NAME_DEFAULT)
75+
demands:
76+
- agent.os -equals Linux
77+
- Role -equals $(CUSTOM_E2E_ROLE)
78+
steps:
79+
- template: azure-cni-overlay-stateless-e2e-step-template.yaml
80+
parameters:
81+
name: ${{ parameters.name }}
82+
clusterName: ${{ parameters.clusterName }}-$(commitID)
83+
os: windows
84+
vmSizeWin: ${{ parameters.vmSize }}
85+
86+
- template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml
87+
parameters:
88+
sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
89+
clusterName: ${{ parameters.clusterName }}-$(commitID)
90+
os: windows
91+
dependsOn: ${{ parameters.name }}_windows
92+
datapath: true
93+
dns: true
94+
portforward: true
95+
hostport: true
96+
service: true
97+
hybridWin: true
98+
99+
- job: failedE2ELogs_windows
100+
displayName: "Windows Failure Logs"
101+
dependsOn:
102+
- ${{ parameters.name }}_windows
103+
- cni_windows
104+
condition: in(dependencies.${{ parameters.name }}_windows.result, 'Failed')
105+
steps:
106+
- template: ../../templates/log-template.yaml
107+
parameters:
108+
clusterName: ${{ parameters.clusterName }}-$(commitID)
109+
os: windows
110+
cni: cniv2
111+

‎.pipelines/singletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e-step-template.yaml‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
parameters:
2+
name: ""
3+
clusterName: ""
4+
os: ""
5+
6+
steps:
7+
- bash: |
8+
go version
9+
go env
10+
mkdir -p '$(GOBIN)'
11+
mkdir -p '$(GOPATH)/pkg'
12+
mkdir -p '$(modulePath)'
13+
echo '##vso[task.prependpath]$(GOBIN)'
14+
echo '##vso[task.prependpath]$(GOROOT)/bin'
15+
name: "GoEnv"
16+
displayName: "Set up the Go environment"
17+
18+
- task: KubectlInstaller@0
19+
inputs:
20+
kubectlVersion: latest
21+
22+
- task: AzureCLI@2
23+
inputs:
24+
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
25+
scriptLocation: "inlineScript"
26+
scriptType: "bash"
27+
addSpnToEnvironment: true
28+
inlineScript: |
29+
set -e
30+
make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}
31+
name: "kubeconfig"
32+
displayName: "Set Kubeconfig"
33+
34+
- script: |
35+
nodeList=`kubectl get node -owide | grep Windows | awk '{print $1}'`
36+
for node in $nodeList; do
37+
taint=`kubectl describe node $node | grep Taints | awk '{print $2}'`
38+
if [ $taint == "node.cloudprovider.kubernetes.io/uninitialized=true:NoSchedule" ]; then
39+
kubectl taint nodes $node node.cloudprovider.kubernetes.io/uninitialized=true:NoSchedule-
40+
fi
41+
done
42+
sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=windows CNI_TYPE=stateless VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_AZURE_VNET_STATELESS=true VALIDATE_V4OVERLAY=true CNS_VERSION=$(make cns-version) CNI_VERSION=$(make cni-version) CLEANUP=true
43+
name: "WindowsOverlayControlPlaneScaleTests"
44+
displayName: "Windows v4Overlay ControlPlane Scale Tests"
45+
retryCountOnTaskFailure: 2
46+
47+
- task: AzureCLI@2
48+
inputs:
49+
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
50+
scriptLocation: "inlineScript"
51+
scriptType: "bash"
52+
addSpnToEnvironment: true
53+
inlineScript: |
54+
set -e
55+
kubectl get po -owide -A
56+
clusterName=${{ parameters.clusterName }}
57+
echo "Restarting nodes"
58+
for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do
59+
make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val}
60+
done
61+
displayName: "Restart Nodes"
62+
63+
- task: AzureCLI@2
64+
inputs:
65+
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
66+
scriptLocation: "inlineScript"
67+
scriptType: "bash"
68+
addSpnToEnvironment: true
69+
inlineScript: |
70+
cd test/integration/load
71+
clusterName=${{ parameters.clusterName }}
72+
make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${clusterName}
73+
make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST)
74+
kubectl get pods -owide -A
75+
echo "Validating Node Restart"
76+
CNI_TYPE=stateless RESTART_CASE=true go test -timeout 30m -tags load -run ^TestValidateState$
77+
displayName: "Validate Node Restart"
78+
retryCountOnTaskFailure: 3
79+
80+
- script: |
81+
echo "Run wireserver and metadata connectivity Tests"
82+
bash test/network/wireserver_metadata_test.sh
83+
retryCountOnTaskFailure: 3
84+
name: "WireserverMetadataConnectivityTests"
85+
displayName: "Run Wireserver and Metadata Connectivity Tests"
86+
87+
- script: |
88+
echo "IPv4 Overlay DataPath Test"
89+
cd test/integration/datapath
90+
sudo -E env "PATH=$PATH" go test -count=1 datapath_windows_test.go -timeout 3m -tags connection -restartKubeproxy true -run ^TestDatapathWin$
91+
name: "WindowsV4OverlayDatapathTests"
92+
displayName: "Windows v4Overlay Datapath Tests"
93+
retryCountOnTaskFailure: 3

‎test/integration/manifests/cnsconfig/azurestatelesscnioverlaywindowsconfigmap.yaml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: cns-win-config
5+
namespace: kube-system
6+
data:
7+
cns_config.json: |
8+
{
9+
"TelemetrySettings": {
10+
"TelemetryBatchSizeBytes": 16384,
11+
"TelemetryBatchIntervalInSecs": 15,
12+
"RefreshIntervalInSecs": 15,
13+
"DisableAll": false,
14+
"HeartBeatIntervalInMins": 30,
15+
"DebugMode": false,
16+
"SnapshotIntervalInMins": 60
17+
},
18+
"ManagedSettings": {
19+
"PrivateEndpoint": "",
20+
"InfrastructureNetworkID": "",
21+
"NodeID": "",
22+
"NodeSyncIntervalInSeconds": 30
23+
},
24+
"EnableSubnetScarcity": false,
25+
"ChannelMode": "CRD",
26+
"InitializeFromCNI": false,
27+
"ManageEndpointState": true,
28+
"ProgramSNATIPTables" : false,
29+
"MetricsBindAddress": ":10092",
30+
"EnableCNIConflistGeneration": false,
31+
"CNIConflistFilepath": "C:\\k\\azurecni\\netconf\\10-azure.conflist",
32+
"CNIConflistScenario": "v4overlay",
33+
"EnableAsyncPodDelete": true,
34+
"AsyncPodDeletePath": "/var/run/azure-vnet/deleteIDs"
35+
}

‎test/internal/kubernetes/utils_create.go‎

Lines changed: 48 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,12 @@ import (
2424
type CNSScenario string
2525

2626
const (
27-
EnvInstallAzilium CNSScenario = "INSTALL_AZILIUM"
28-
EnvInstallAzureVnet CNSScenario = "INSTALL_AZURE_VNET"
29-
EnvInstallOverlay CNSScenario = "INSTALL_OVERLAY"
30-
EnvInstallAzureCNIOverlay CNSScenario = "INSTALL_AZURE_CNI_OVERLAY"
31-
EnvInstallDualStackOverlay CNSScenario = "INSTALL_DUALSTACK_OVERLAY"
27+
EnvInstallAzilium CNSScenario = "INSTALL_AZILIUM"
28+
EnvInstallAzureVnet CNSScenario = "INSTALL_AZURE_VNET"
29+
EnvInstallAzureVnetStateless CNSScenario = "INSTALL_AZURE_VNET_STATELESS"
30+
EnvInstallOverlay CNSScenario = "INSTALL_OVERLAY"
31+
EnvInstallAzureCNIOverlay CNSScenario = "INSTALL_AZURE_CNI_OVERLAY"
32+
EnvInstallDualStackOverlay CNSScenario = "INSTALL_DUALSTACK_OVERLAY"
3233
)
3334

3435
type cnsDetails struct {
@@ -333,6 +334,7 @@ func initCNSScenarioVars() (map[CNSScenario]map[corev1.OSName]cnsDetails, error)
333334
cnsOverlayConfigMapPath := cnsConfigFolder + "/overlayconfigmap.yaml"
334335
cnsAzureCNIOverlayLinuxConfigMapPath := cnsConfigFolder + "/azurecnioverlaylinuxconfigmap.yaml"
335336
cnsAzureCNIOverlayWindowsConfigMapPath := cnsConfigFolder + "/azurecnioverlaywindowsconfigmap.yaml"
337+
cnsAzureStatelessCNIOverlayWindowsConfigMapPath := cnsConfigFolder + "/azurestatelesscnioverlaywindowsconfigmap.yaml"
336338
cnsAzureCNIDualStackLinuxConfigMapPath := cnsConfigFolder + "/azurecnidualstackoverlaylinuxconfigmap.yaml"
337339
cnsAzureCNIDualStackWindowsConfigMapPath := cnsConfigFolder + "/azurecnidualstackoverlaywindowsconfigmap.yaml"
338340
cnsRolePath := cnsManifestFolder + "/role.yaml"
@@ -393,6 +395,47 @@ func initCNSScenarioVars() (map[CNSScenario]map[corev1.OSName]cnsDetails, error)
393395
installIPMasqAgent: false,
394396
},
395397
},
398+
EnvInstallAzureVnetStateless: {
399+
corev1.Linux: {
400+
daemonsetPath: cnsLinuxDaemonSetPath,
401+
labelSelector: cnsLinuxLabelSelector,
402+
rolePath: cnsRolePath,
403+
roleBindingPath: cnsRoleBindingPath,
404+
clusterRolePath: cnsClusterRolePath,
405+
clusterRoleBindingPath: cnsClusterRoleBindingPath,
406+
serviceAccountPath: cnsServiceAccountPath,
407+
initContainerArgs: []string{
408+
"deploy",
409+
"azure-vnet", "-o", "/opt/cni/bin/azure-vnet",
410+
"azure-vnet-telemetry", "-o", "/opt/cni/bin/azure-vnet-telemetry",
411+
},
412+
initContainerName: initContainerNameCNI,
413+
volumes: volumesForAzureCNIOverlayLinux(),
414+
initContainerVolumeMounts: dropgzVolumeMountsForAzureCNIOverlayLinux(),
415+
containerVolumeMounts: cnsVolumeMountsForAzureCNIOverlayLinux(),
416+
configMapPath: cnsAzureCNIOverlayLinuxConfigMapPath,
417+
installIPMasqAgent: true,
418+
},
419+
corev1.Windows: {
420+
daemonsetPath: cnsWindowsDaemonSetPath,
421+
labelSelector: cnsWindowsLabelSelector,
422+
rolePath: cnsRolePath,
423+
roleBindingPath: cnsRoleBindingPath,
424+
clusterRolePath: cnsClusterRolePath,
425+
clusterRoleBindingPath: cnsClusterRoleBindingPath,
426+
serviceAccountPath: cnsServiceAccountPath,
427+
initContainerArgs: []string{
428+
"deploy",
429+
"azure-vnet-stateless", "-o", "/k/azurecni/bin/azure-vnet.exe",
430+
},
431+
initContainerName: initContainerNameCNI,
432+
volumes: volumesForAzureCNIOverlayWindows(),
433+
initContainerVolumeMounts: dropgzVolumeMountsForAzureCNIOverlayWindows(),
434+
containerVolumeMounts: cnsVolumeMountsForAzureCNIOverlayWindows(),
435+
configMapPath: cnsAzureStatelessCNIOverlayWindowsConfigMapPath,
436+
installIPMasqAgent: true,
437+
},
438+
},
396439
EnvInstallAzilium: {
397440
corev1.Linux: {
398441
daemonsetPath: cnsLinuxDaemonSetPath,

‎test/validate/linux_validate.go‎

Lines changed: 0 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -206,24 +206,6 @@ type AzureVnetEndpointInfo struct {
206206
PodName string
207207
}
208208

209-
func cnsManagedStateFileIps(result []byte) (map[string]string, error) {
210-
var cnsResult CnsManagedState
211-
err := json.Unmarshal(result, &cnsResult)
212-
if err != nil {
213-
return nil, errors.Wrapf(err, "failed to unmarshal cns endpoint list")
214-
}
215-
216-
cnsPodIps := make(map[string]string)
217-
for _, v := range cnsResult.Endpoints {
218-
for ifName, ip := range v.IfnameToIPMap {
219-
if ifName == "eth0" {
220-
cnsPodIps[ip.IPv4[0].IP.String()] = v.PodName
221-
}
222-
}
223-
}
224-
return cnsPodIps, nil
225-
}
226-
227209
func cnsManagedStateFileDualStackIps(result []byte) (map[string]string, error) {
228210
var cnsResult CnsManagedState
229211
err := json.Unmarshal(result, &cnsResult)

‎test/validate/validate.go‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -261,3 +261,21 @@ func cnsCacheStateFileIps(result []byte) (map[string]string, error) {
261261
}
262262
return cnsPodIps, nil
263263
}
264+
265+
func cnsManagedStateFileIps(result []byte) (map[string]string, error) {
266+
var cnsResult CnsManagedState
267+
err := json.Unmarshal(result, &cnsResult)
268+
if err != nil {
269+
return nil, errors.Wrapf(err, "failed to unmarshal cns endpoint list")
270+
}
271+
272+
cnsPodIps := make(map[string]string)
273+
for _, v := range cnsResult.Endpoints {
274+
for ifName, ip := range v.IfnameToIPMap {
275+
if ifName == "eth0" {
276+
cnsPodIps[ip.IPv4[0].IP.String()] = v.PodName
277+
}
278+
}
279+
}
280+
return cnsPodIps, nil
281+
}

0 commit comments

Comments
 (0)