Skip to content

Commit f955ae0

Browse files
camrynlpaulyufan2
camrynl
and
paulyufan2
authored
ci: cilium load test uses cns write conflist (#1928)
* change cilium install method * update deploy cil stage * cns writes cilium conflist * adding cns ds * delete deployment container, too many pods * reuse cns ds from pr pipeline * update cns + dropgz versions in ds for load test pipeline * revert back to separate cns ds * revert change for cns/daemonset.yaml --------- Co-authored-by: Paul Yu <[email protected]>
1 parent 4f76b19 commit f955ae0

File tree

6 files changed

+272
-5
lines changed

6 files changed

+272
-5
lines changed

‎.pipelines/cni/cilium/cilium-cni-load-test.yaml‎

Lines changed: 40 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,48 @@ stages:
1616
inlineScript: |
1717
set -ex
1818
make -C ./hack/swift azcfg AZCLI=az REGION=$(LOCATION)
19-
make -C ./hack/swift overlay-cilium-up AZCLI=az REGION=$(LOCATION) SUB=$(SUBSCRIPTION_ID) CLUSTER=${RESOURCE_GROUP} NODE_COUNT=10 VM_SIZE=Standard_DS4_v2
19+
make -C ./hack/swift overlay-byocni-up AZCLI=az REGION=$(LOCATION) SUB=$(SUBSCRIPTION_ID) CLUSTER=${RESOURCE_GROUP} NODE_COUNT=10 VM_SIZE=Standard_DS4_v2
2020
name: "CreateAksCluster"
2121
displayName: "Create AKS Cluster"
22-
- stage: pod_deployment
22+
- stage: install_cilium
2323
dependsOn: creating_aks_cluster
24+
displayName: "Install Cilium on AKS Overlay"
25+
jobs:
26+
- job: deploy_cilium_components
27+
steps:
28+
- task: AzureCLI@1
29+
displayName: "Install Cilium, CNS, and ip-masq-agent"
30+
inputs:
31+
azureSubscription: $(TEST_SUB_SERVICE_CONNECTION)
32+
scriptLocation: "inlineScript"
33+
scriptType: "bash"
34+
addSpnToEnvironment: true
35+
inlineScript: |
36+
set -ex
37+
az extension add --name aks-preview
38+
make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}
39+
ls -lah
40+
pwd
41+
kubectl cluster-info
42+
kubectl get po -owide -A
43+
echo "Deploy Azure-CNS"
44+
kubectl apply -f test/integration/manifests/cilium/cns-write-ovly.yaml
45+
echo "deploy Cilium ConfigMap"
46+
kubectl apply -f cilium/configmap.yaml
47+
kubectl apply -f test/integration/manifests/cilium/cilium-config.yaml
48+
echo "install Cilium onto Overlay Cluster"
49+
kubectl apply -f test/integration/manifests/cilium/cilium-agent
50+
kubectl apply -f test/integration/manifests/cilium/cilium-operator
51+
kubectl get po -owide -A
52+
echo "deploy ip-masq-agent for overlay"
53+
kubectl create -f test/integration/manifests/ip-masq-agent/ip-masq-agent.yaml --validate=false
54+
cd test/integration/manifests/ip-masq-agent/
55+
kubectl create configmap config-custom.yaml
56+
kubectl create configmap config-reconcile.yaml
57+
cd ../../../..
58+
kubectl get po -owide -A
59+
- stage: pod_deployment
60+
dependsOn: install_cilium
2461
displayName: "Pod Deployment"
2562
jobs:
2663
- job: deploy_pods
@@ -115,6 +152,7 @@ stages:
115152
name: "GetCluster"
116153
displayName: "Get AKS Cluster"
117154
- script: |
155+
k delete deployment container -n default
118156
cilium connectivity test
119157
retryCountOnTaskFailure: 6
120158
name: "CiliumConnectivityTests"

‎test/integration/manifests/cilium/cns-write-ovly.yaml‎

Lines changed: 228 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,228 @@
1+
apiVersion: v1
2+
kind: ServiceAccount
3+
metadata:
4+
name: azure-cns
5+
namespace: kube-system
6+
---
7+
apiVersion: rbac.authorization.k8s.io/v1
8+
kind: Role
9+
metadata:
10+
namespace: kube-system
11+
name: nodeNetConfigEditor
12+
rules:
13+
- apiGroups: ["acn.azure.com"]
14+
resources: ["nodenetworkconfigs"]
15+
verbs: ["get", "list", "watch", "patch", "update"]
16+
---
17+
apiVersion: rbac.authorization.k8s.io/v1
18+
kind: ClusterRole
19+
metadata:
20+
name: pod-reader-all-namespaces
21+
rules:
22+
- apiGroups: [""]
23+
resources: ["pods"]
24+
verbs: ["get", "watch", "list"]
25+
- apiGroups: [""]
26+
resources: ["nodes"]
27+
verbs: ["get"]
28+
---
29+
apiVersion: rbac.authorization.k8s.io/v1
30+
kind: RoleBinding
31+
metadata:
32+
name: nodeNetConfigEditorRoleBinding
33+
namespace: kube-system
34+
subjects:
35+
- kind: ServiceAccount
36+
name: azure-cns
37+
namespace: kube-system
38+
roleRef:
39+
kind: Role
40+
name: nodeNetConfigEditor
41+
apiGroup: rbac.authorization.k8s.io
42+
---
43+
apiVersion: rbac.authorization.k8s.io/v1
44+
kind: ClusterRoleBinding
45+
metadata:
46+
name: pod-reader-all-namespaces-binding
47+
subjects:
48+
- kind: ServiceAccount
49+
name: azure-cns
50+
namespace: kube-system
51+
roleRef:
52+
kind: ClusterRole
53+
name: pod-reader-all-namespaces
54+
apiGroup: rbac.authorization.k8s.io
55+
---
56+
apiVersion: apps/v1
57+
kind: DaemonSet
58+
metadata:
59+
name: azure-cns
60+
namespace: kube-system
61+
labels:
62+
app: azure-cns
63+
spec:
64+
selector:
65+
matchLabels:
66+
k8s-app: azure-cns
67+
template:
68+
metadata:
69+
labels:
70+
k8s-app: azure-cns
71+
annotations:
72+
cluster-autoscaler.kubernetes.io/daemonset-pod: "true"
73+
spec:
74+
affinity:
75+
nodeAffinity:
76+
requiredDuringSchedulingIgnoredDuringExecution:
77+
nodeSelectorTerms:
78+
- matchExpressions:
79+
- key: kubernetes.azure.com/cluster
80+
operator: Exists
81+
- key: type
82+
operator: NotIn
83+
values:
84+
- virtual-kubelet
85+
- key: beta.kubernetes.io/os
86+
operator: In
87+
values:
88+
- linux
89+
priorityClassName: system-node-critical
90+
tolerations:
91+
- key: CriticalAddonsOnly
92+
operator: Exists
93+
- operator: "Exists"
94+
effect: NoExecute
95+
- operator: "Exists"
96+
effect: NoSchedule
97+
containers:
98+
- name: cns-container
99+
image: acnpublic.azurecr.io/azure-cns:write-cilium-conf2
100+
imagePullPolicy: IfNotPresent
101+
args: [ "-c", "tcp://$(CNSIpAddress):$(CNSPort)", "-t", "$(CNSLogTarget)"]
102+
securityContext:
103+
capabilities:
104+
add:
105+
- NET_ADMIN
106+
volumeMounts:
107+
- name: log
108+
mountPath: /var/log
109+
- name: cns-state
110+
mountPath: /var/lib/azure-network
111+
- name: cns-config
112+
mountPath: /etc/azure-cns
113+
- name: cni-bin
114+
mountPath: /opt/cni/bin
115+
- name: azure-vnet
116+
mountPath: /var/run/azure-vnet
117+
- name: legacy-cni-state
118+
mountPath: /var/run/azure-vnet.json
119+
- name: xtables-lock
120+
mountPath: /run/xtables.lock
121+
- name: cni-ipam-state
122+
mountPath: /var/run/azure-cns
123+
- name: cni-conflist
124+
mountPath: /etc/cni/net.d
125+
ports:
126+
- containerPort: 10090
127+
env:
128+
- name: CNSIpAddress
129+
value: "127.0.0.1"
130+
- name: CNSPort
131+
value: "10090"
132+
- name: CNSLogTarget
133+
value: "stdoutfile"
134+
- name: CNS_CONFIGURATION_PATH
135+
value: /etc/azure-cns/cns_config.json
136+
- name: NODENAME
137+
valueFrom:
138+
fieldRef:
139+
apiVersion: v1
140+
fieldPath: spec.nodeName
141+
initContainers:
142+
- name: init-cni-dropgz
143+
image: acnpublic.azurecr.io/cni-dropgz:v0.0.4
144+
imagePullPolicy: Always
145+
command: ["/dropgz"]
146+
args:
147+
- deploy
148+
- azure-ipam
149+
- -o
150+
- /opt/cni/bin/azure-ipam
151+
# - azilium.conflist
152+
# - -o
153+
# - /etc/cni/net.d/05-cilium.conflist
154+
volumeMounts:
155+
- name: cni-bin
156+
mountPath: /opt/cni/bin
157+
# - name: cni-conflist
158+
# mountPath: /etc/cni/net.d
159+
hostNetwork: true
160+
volumes:
161+
- name: cni-conflist
162+
hostPath:
163+
path: /etc/cni/net.d
164+
type: Directory
165+
- name: log
166+
hostPath:
167+
path: /var/log
168+
type: Directory
169+
- name: cns-state
170+
hostPath:
171+
path: /var/lib/azure-network
172+
type: DirectoryOrCreate
173+
- name: cni-bin
174+
hostPath:
175+
path: /opt/cni/bin
176+
type: Directory
177+
- name: azure-vnet
178+
hostPath:
179+
path: /var/run/azure-vnet
180+
type: DirectoryOrCreate
181+
- name: legacy-cni-state
182+
hostPath:
183+
path: /var/run/azure-vnet.json
184+
type: FileOrCreate
185+
- name: cns-config
186+
configMap:
187+
name: cns-config
188+
- name: xtables-lock
189+
hostPath:
190+
path: /run/xtables.lock
191+
type: File
192+
- name: cni-ipam-state
193+
hostPath:
194+
path: /var/run/azure-cns
195+
type: DirectoryOrCreate
196+
serviceAccountName: azure-cns
197+
---
198+
apiVersion: v1
199+
kind: ConfigMap
200+
metadata:
201+
name: cns-config
202+
namespace: kube-system
203+
data:
204+
cns_config.json: |
205+
{
206+
"TelemetrySettings": {
207+
"TelemetryBatchSizeBytes": 16384,
208+
"TelemetryBatchIntervalInSecs": 15,
209+
"RefreshIntervalInSecs": 15,
210+
"DisableAll": false,
211+
"HeartBeatIntervalInMins": 30,
212+
"DebugMode": false,
213+
"SnapshotIntervalInMins": 60
214+
},
215+
"ManagedSettings": {
216+
"PrivateEndpoint": "",
217+
"InfrastructureNetworkID": "",
218+
"NodeID": "",
219+
"NodeSyncIntervalInSeconds": 30
220+
},
221+
"ChannelMode": "CRD",
222+
"InitializeFromCNI": false,
223+
"ManageEndpointState": true,
224+
"ProgramSNATIPTables": false,
225+
"EnableCNIConflistGeneration": true,
226+
"CNIConflistFilepath": "/etc/cni/net.d/05-cilium.conflist",
227+
"CNIConflistScenario": "cilium"
228+
}

‎test/integration/manifests/cns/ciliumconfigmap.yaml‎ renamed to ‎test/integration/manifests/cnsconfig/ciliumconfigmap.yaml‎

File renamed without changes.

‎test/integration/manifests/cns/overlayconfigmap.yaml‎ renamed to ‎test/integration/manifests/cnsconfig/overlayconfigmap.yaml‎

File renamed without changes.

‎test/integration/manifests/cns/swiftconfigmap.yaml‎ renamed to ‎test/integration/manifests/cnsconfig/swiftconfigmap.yaml‎

File renamed without changes.

‎test/integration/setup_test.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,13 @@ const (
2626

2727
// relative cns manifest paths
2828
cnsManifestFolder = "manifests/cns"
29+
cnsConfigFolder = "manifests/cnsconfig"
2930
cnsDaemonSetPath = cnsManifestFolder + "/daemonset.yaml"
3031
cnsClusterRolePath = cnsManifestFolder + "/clusterrole.yaml"
3132
cnsClusterRoleBindingPath = cnsManifestFolder + "/clusterrolebinding.yaml"
32-
cnsSwiftConfigMapPath = cnsManifestFolder + "/swiftconfigmap.yaml"
33-
cnsCiliumConfigMapPath = cnsManifestFolder + "/ciliumconfigmap.yaml"
34-
cnsOverlayConfigMapPath = cnsManifestFolder + "/overlayconfigmap.yaml"
33+
cnsSwiftConfigMapPath = cnsConfigFolder + "/swiftconfigmap.yaml"
34+
cnsCiliumConfigMapPath = cnsConfigFolder + "/ciliumconfigmap.yaml"
35+
cnsOverlayConfigMapPath = cnsConfigFolder + "/overlayconfigmap.yaml"
3536
cnsRolePath = cnsManifestFolder + "/role.yaml"
3637
cnsRoleBindingPath = cnsManifestFolder + "/rolebinding.yaml"
3738
cnsServiceAccountPath = cnsManifestFolder + "/serviceaccount.yaml"

0 commit comments

Comments
 (0)