Skip to content

Commit 0c5c12a

Browse files
camrynlcamrynl
authored
Add Cilium on AKS Overlay e2e scenario (#1609)
* add hack overlay make target and e2e templates * update pipeline.yaml with new overlay stage * use dropgz/overlay-byocni in acn pr and submodule pipeline * name overlay clusters for submodule pipeline * modify dropgz setup * test ipam + overlay * use azilium conflist * add overlay configmap * add deployment of ip-masq-agent to cluster * set flag in ip-masq-agent deployment * update deployment * configmap creation
1 parent 39b6a33 commit 0c5c12a

File tree

11 files changed

+341
-0
lines changed

11 files changed

+341
-0
lines changed

‎.pipelines/pipeline.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -282,6 +282,14 @@ stages:
282282
testDropgz: ""
283283
clusterName: "ciliume2e"
284284

285+
- template: singletenancy/overlay/overlay-e2e-job-template.yaml
286+
parameters:
287+
name: "overlay_e2e"
288+
displayName: Cilium on AKS Overlay
289+
pipelineBuildImage: "$(BUILD_IMAGE)"
290+
testDropgz: ""
291+
clusterName: "overlaye2e"
292+
285293
- template: singletenancy/aks-swift/e2e-job-template.yaml
286294
parameters:
287295
name: "aks_swift_e2e"

‎.pipelines/singletenancy/overlay/overlay-e2e-job-template.yaml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
parameters:
2+
name: ""
3+
displayName: ""
4+
pipelineBuildImage: "$(BUILD_IMAGE)"
5+
testDropgz: ""
6+
clusterName: ""
7+
8+
stages:
9+
- stage: ${{ parameters.name }}
10+
displayName: E2E - ${{ parameters.displayName }}
11+
dependsOn:
12+
- setup
13+
- publish
14+
jobs:
15+
- job: ${{ parameters.name }}
16+
displayName: Overlay Test Suite - (${{ parameters.name }})
17+
timeoutInMinutes: 120
18+
pool:
19+
name: $(BUILD_POOL_NAME_DEFAULT)
20+
demands:
21+
- agent.os -equals Linux
22+
- Role -equals $(CUSTOM_E2E_ROLE)
23+
variables:
24+
GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path
25+
GOBIN: "$(GOPATH)/bin" # Go binaries path
26+
modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking"
27+
steps:
28+
- template: overlay-e2e-step-template.yaml
29+
parameters:
30+
name: ${{ parameters.name }}
31+
testDropgz: ${{ parameters.testDropgz }}
32+
clusterName: ${{ parameters.clusterName }}

‎.pipelines/singletenancy/overlay/overlay-e2e-step-template.yaml‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
parameters:
2+
name: ""
3+
testDropgz: ""
4+
clusterName: ""
5+
6+
steps:
7+
- bash: |
8+
echo $UID
9+
sudo rm -rf $(System.DefaultWorkingDirectory)/*
10+
displayName: "Set up OS environment"
11+
12+
- checkout: self
13+
14+
- bash: |
15+
go version
16+
go env
17+
mkdir -p '$(GOBIN)'
18+
mkdir -p '$(GOPATH)/pkg'
19+
mkdir -p '$(modulePath)'
20+
echo '##vso[task.prependpath]$(GOBIN)'
21+
echo '##vso[task.prependpath]$(GOROOT)/bin'
22+
name: "GoEnv"
23+
displayName: "Set up the Go environment"
24+
25+
- task: AzureCLI@1
26+
inputs:
27+
azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION)
28+
scriptLocation: "inlineScript"
29+
scriptType: "bash"
30+
addSpnToEnvironment: true
31+
inlineScript: |
32+
mkdir -p ~/.kube/
33+
echo "Create AKS Overlay cluster"
34+
make -C ./hack/swift azcfg AZCLI=az REGION=$(REGION_OVERLAY_CLUSTER_TEST)
35+
make -C ./hack/swift overlay-byocni-up AZCLI=az REGION=$(REGION_OVERLAY_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision)
36+
echo "Cluster successfully created"
37+
displayName: Create Overlay cluster
38+
condition: succeeded()
39+
40+
- script: |
41+
ls -lah
42+
pwd
43+
echo "installing kubectl"
44+
curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
45+
sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
46+
kubectl cluster-info
47+
kubectl get po -owide -A
48+
curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
49+
chmod 700 get_helm.sh
50+
./get_helm.sh
51+
echo "deploy Cilium ConfigMap"
52+
kubectl apply -f cilium/configmap.yaml
53+
echo "install Cilium onto Overlay Cluster"
54+
helm repo add cilium https://helm.cilium.io/
55+
helm install cilium cilium/cilium --version 1.12.1.1 --namespace kube-system -f cilium/cilium_helm_values.yaml
56+
name: "installCilium"
57+
displayName: "Install Cilium on AKS Overlay"
58+
59+
- script: |
60+
echo "install cilium CLI"
61+
CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/master/stable.txt)
62+
CLI_ARCH=amd64
63+
if [ "$(uname -m)" = "aarch64" ]; then CLI_ARCH=arm64; fi
64+
curl -L --fail --remote-name-all https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum}
65+
sha256sum --check cilium-linux-${CLI_ARCH}.tar.gz.sha256sum
66+
sudo tar xzvfC cilium-linux-${CLI_ARCH}.tar.gz /usr/local/bin
67+
rm cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum}
68+
cilium status
69+
name: "installCiliumCLI"
70+
displayName: "Install Cilium CLI"
71+
72+
- script: |
73+
echo "install kubetest2 and gsutils"
74+
go get github.com/onsi/ginkgo/ginkgo
75+
go get github.com/onsi/gomega/...
76+
go install github.com/onsi/ginkgo/ginkgo@latest
77+
go install sigs.k8s.io/kubetest2@latest
78+
go install sigs.k8s.io/kubetest2/kubetest2-noop@latest
79+
go install sigs.k8s.io/kubetest2/kubetest2-tester-ginkgo@latest
80+
wget https://storage.googleapis.com/pub/gsutil.tar.gz
81+
tar xfz gsutil.tar.gz
82+
sudo mv gsutil /usr/local/bin
83+
name: "installKubetest"
84+
displayName: "Set up Conformance Tests"
85+
86+
- script: |
87+
echo "Start Azilium E2E Tests on Overlay Cluster"
88+
echo "deploy ip-masq-agent for overlay"
89+
kubectl create -f test/integration/manifests/ip-masq-agent/ip-masq-agent.yaml --validate=false
90+
cd test/integration/manifests/ip-masq-agent/
91+
kubectl create configmap config-custom.yaml
92+
kubectl create configmap config-reconcile.yaml
93+
cd ../../../..
94+
kubectl get po -owide -A
95+
sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=$(make cns-version) CNI_DROPGZ_VERSION=$(make cni-dropgz-version) INSTALL_CNS=true INSTALL_OVERLAY=true TEST_DROPGZ=${{ parameters.testDropgz }}
96+
retryCountOnTaskFailure: 3
97+
name: "aziliumTest"
98+
displayName: "Run Azilium E2E on AKS Overlay"
99+
100+
- script: |
101+
echo "Logs will be available as a build artifact"
102+
ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/
103+
echo $ARTIFACT_DIR
104+
sudo rm -rf $ARTIFACT_DIR
105+
sudo mkdir $ARTIFACT_DIR
106+
sudo cp test/integration/logs/* $ARTIFACT_DIR
107+
name: "GetLogs"
108+
displayName: "Get logs"
109+
condition: always()
110+
111+
- task: PublishBuildArtifacts@1
112+
inputs:
113+
artifactName: test-output
114+
pathtoPublish: "$(Build.ArtifactStagingDirectory)/test-output"
115+
condition: always()
116+
117+
- script: |
118+
echo "Run Service Conformance E2E"
119+
export PATH=${PATH}:/usr/local/bin/gsutil
120+
KUBECONFIG=~/.kube/config kubetest2 noop \
121+
--test ginkgo -- \
122+
--focus-regex "Services.*\[Conformance\].*"
123+
name: "servicesConformance"
124+
displayName: "Run Services Conformance Tests"
125+
126+
- script: |
127+
echo "Run Cilium Connectivity Tests"
128+
cilium status
129+
cilium connectivity test
130+
name: "ciliumConnectivityTests"
131+
displayName: "Run Cilium Connectivity Tests"
132+
133+
- script: |
134+
ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/
135+
echo $ARTIFACT_DIR
136+
sudo rm -rf $ARTIFACT_DIR
137+
sudo rm -rf test/integration/logs
138+
name: "Cleanupartifactdir"
139+
displayName: "Cleanup artifact dir"
140+
condition: always()
141+
142+
- task: AzureCLI@2
143+
inputs:
144+
azureSubscription: "Azure Container Networking - Test"
145+
scriptLocation: "inlineScript"
146+
scriptType: "bash"
147+
addSpnToEnvironment: true
148+
inlineScript: |
149+
echo "Deleting cluster"
150+
make -C ./hack/swift azcfg AZCLI=az
151+
make -C ./hack/swift down SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision)
152+
echo "Cluster and resources down"
153+
name: "Cleanupcluster"
154+
displayName: "Cleanup cluster"
155+
condition: always()

‎.pipelines/submodules-pipeline.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -245,6 +245,14 @@ stages:
245245
pipelineBuildImage: "$(BUILD_IMAGE)"
246246
testDropgz: true
247247
clusterName: "submodules-ciliume2e"
248+
249+
- template: singletenancy/overlay/overlay-e2e-job-template.yaml
250+
parameters:
251+
name: "overlay_e2e"
252+
displayName: Cilium on AKS Overlay
253+
pipelineBuildImage: "$(BUILD_IMAGE)"
254+
testDropgz: true
255+
clusterName: "submodules-overlaye2e"
248256

249257
- template: singletenancy/aks-swift/e2e-job-template.yaml
250258
parameters:

‎dropgz/build/cniTest.Dockerfile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ COPY dropgz .
1717
COPY --from=azure-ipam /azure-ipam/*.conflist pkg/embed/fs
1818
COPY --from=azure-ipam /azure-ipam/bin/* pkg/embed/fs
1919
COPY --from=azure-vnet /azure-container-networking/cni/azure-$OS-swift.conflist pkg/embed/fs/azure-swift.conflist
20+
COPY --from=azure-vnet /azure-container-networking/cni/azure-$OS-swift-overlay.conflist pkg/embed/fs/azure-swift-overlay.conflist
2021
COPY --from=azure-vnet /azure-container-networking/bin/* pkg/embed/fs
2122
RUN cd pkg/embed/fs/ && sha256sum * > sum.txt
2223
RUN gzip --verbose --best --recursive pkg/embed/fs && for f in pkg/embed/fs/*.gz; do mv -- "$f" "${f%%.gz}"; done

‎hack/swift/Makefile‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,19 @@ byocni-up: swift-byocni-up ## Alias to swift-byocni-up
7979
cilium-up: swift-cilium-up ## Alias to swift-cilium-up
8080
up: swift-up ## Alias to swift-up
8181

82+
overlay-byocni-up: rg-up overlay-net-up ## Brings up an Overlay BYO CNI cluster
83+
$(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \
84+
--node-count 2 \
85+
--node-vm-size Standard_B2s \
86+
--load-balancer-sku basic \
87+
--network-plugin none \
88+
--network-plugin-mode overlay \
89+
--pod-cidr 192.168.0.0/16 \
90+
--vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \
91+
--no-ssh-key \
92+
--yes
93+
@$(MAKE) set-kubeconf
94+
8295
overlay-up: rg-up overlay-net-up ## Brings up an Overlay AzCNI cluster
8396
$(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \
8497
--node-count 2 \

‎test/integration/manifests/cns/overlayconfigmap.yaml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: cns-config
5+
namespace: kube-system
6+
data:
7+
cns_config.json: |
8+
{
9+
"TelemetrySettings": {
10+
"TelemetryBatchSizeBytes": 16384,
11+
"TelemetryBatchIntervalInSecs": 15,
12+
"RefreshIntervalInSecs": 15,
13+
"DisableAll": false,
14+
"HeartBeatIntervalInMins": 30,
15+
"DebugMode": false,
16+
"SnapshotIntervalInMins": 60
17+
},
18+
"ManagedSettings": {
19+
"PrivateEndpoint": "",
20+
"InfrastructureNetworkID": "",
21+
"NodeID": "",
22+
"NodeSyncIntervalInSeconds": 30
23+
},
24+
"ChannelMode": "CRD",
25+
"InitializeFromCNI": false,
26+
"ManageEndpointState": true,
27+
"ProgramSNATIPTables" : false
28+
}

‎test/integration/manifests/ip-masq-agent/config-custom.yaml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: ip-masq-agent-config
5+
namespace: kube-system
6+
labels:
7+
component: ip-masq-agent
8+
kubernetes.io/cluster-service: "true"
9+
addonmanager.kubernetes.io/mode: EnsureExists
10+
data:
11+
ip-masq-agent: |-
12+
nonMasqueradeCIDRs:
13+
- 192.168.0.0/16
14+
masqLinkLocal: false
15+
masqLinkLocalIPv6: true

‎test/integration/manifests/ip-masq-agent/config-reconcile.yaml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: ip-masq-agent-config-reconciled
5+
namespace: kube-system
6+
labels:
7+
component: ip-masq-agent
8+
kubernetes.io/cluster-service: "true"
9+
addonmanager.kubernetes.io/mode: Reconcile
10+
data:
11+
ip-masq-agent-reconciled: |-
12+
nonMasqueradeCIDRs:
13+
- 192.168.0.0/16
14+
masqLinkLocal: true

‎test/integration/manifests/ip-masq-agent/ip-masq-agent.yaml‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
apiVersion: apps/v1
2+
kind: DaemonSet
3+
metadata:
4+
name: ip-masq-agent
5+
namespace: kube-system
6+
labels:
7+
component: ip-masq-agent
8+
kubernetes.io/cluster-service: "true"
9+
addonmanager.kubernetes.io/mode: Reconcile
10+
spec:
11+
selector:
12+
matchLabels:
13+
k8s-app: ip-masq-agent
14+
template:
15+
metadata:
16+
labels:
17+
k8s-app: ip-masq-agent
18+
spec:
19+
hostNetwork: true
20+
containers:
21+
- name: ip-masq-agent
22+
image: mcr.microsoft.com/aks/ip-masq-agent-v2:v0.1.1
23+
imagePullPolicy: Always
24+
securityContext:
25+
privileged: false
26+
capabilities:
27+
add: ["NET_ADMIN", "NET_RAW"]
28+
# Uses projected volumes to merge all data in /etc/config
29+
volumeMounts:
30+
- name: ip-masq-agent-volume
31+
mountPath: /etc/config
32+
readOnly: true
33+
volumes:
34+
- name: ip-masq-agent-volume
35+
projected:
36+
sources:
37+
# Note these ConfigMaps must be created in the same namespace as the daemonset
38+
- configMap:
39+
name: ip-masq-agent-config
40+
optional: true
41+
items:
42+
- key: ip-masq-agent
43+
path: ip-masq-agent
44+
mode: 444
45+
- configMap:
46+
name: ip-masq-agent-config-reconciled
47+
optional: true
48+
items:
49+
# Avoiding duplicate paths
50+
- key: ip-masq-agent-reconciled
51+
path: ip-masq-agent-reconciled
52+
mode: 444

0 commit comments

Comments
 (0)