Skip to content

Commit 846771a

Browse files
MikeZappa87MikeZappa87
authored
CI: Setup Cilium with Hubble Enabled nightly run (#2514)
ci: provide hubble+cilium nightly runs
1 parent 5c1fe96 commit 846771a

File tree

14 files changed

+1461
-42
lines changed

14 files changed

+1461
-42
lines changed

‎.pipelines/networkobservability/pipeline.yaml‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
pr: none
2+
trigger: none
3+
4+
schedules:
5+
- cron: '0 0 * * *'
6+
displayName: Daily midnight Cilium with Hubble
7+
branches:
8+
include:
9+
- master
10+
11+
variables:
12+
clustername: ciliumhubble-$(Build.SourceBranchName)-$(Build.BuildId)
13+
cilium_version: v1.14.4
14+
15+
stages:
16+
- stage: createCluster
17+
pool:
18+
name: $(BUILD_POOL_NAME_DEFAULT)
19+
displayName: "create cluster"
20+
jobs:
21+
- template: ../templates/create-cluster.yaml
22+
parameters:
23+
name: cilium_overlay_hubble
24+
displayName: Cilium on AKS Overlay with Hubble Enabled
25+
clusterType: overlay-byocni-nokubeproxy-up
26+
clusterName: $(clustername)
27+
vmSize: Standard_B2ms
28+
k8sVersion: ""
29+
region: $(REGION_AKS_CLUSTER_TEST)
30+
- stage: setupCluster
31+
pool:
32+
name: $(BUILD_POOL_NAME_DEFAULT)
33+
jobs:
34+
- job: "ciliuminstall"
35+
steps:
36+
- bash: |
37+
go version
38+
go env
39+
mkdir -p '$(GOBIN)'
40+
mkdir -p '$(GOPATH)/pkg'
41+
mkdir -p '$(modulePath)'
42+
echo '##vso[task.prependpath]$(GOBIN)'
43+
echo '##vso[task.prependpath]$(GOROOT)/bin'
44+
name: "GoEnv"
45+
displayName: "Set up the Go environment"
46+
47+
- task: KubectlInstaller@0
48+
inputs:
49+
kubectlVersion: latest
50+
51+
- task: AzureCLI@1
52+
inputs:
53+
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
54+
scriptLocation: "inlineScript"
55+
scriptType: "bash"
56+
addSpnToEnvironment: true
57+
inlineScript: |
58+
set -e
59+
make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=$(clustername)
60+
name: "setupkubeconf"
61+
displayName: "Set up kubeconfig"
62+
63+
- script: |
64+
CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/main/stable.txt)
65+
CLI_ARCH=amd64
66+
if [ "$(uname -m)" = "aarch64" ]; then CLI_ARCH=arm64; fi
67+
curl -L --fail --remote-name-all https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum}
68+
sha256sum --check cilium-linux-${CLI_ARCH}.tar.gz.sha256sum
69+
sudo tar xzvfC cilium-linux-${CLI_ARCH}.tar.gz /usr/local/bin
70+
rm cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum}
71+
name: "installCiliumCLI"
72+
displayName: "Install Cilium CLI"
73+
74+
- task: AzureCLI@1
75+
inputs:
76+
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
77+
scriptLocation: "inlineScript"
78+
scriptType: "bash"
79+
addSpnToEnvironment: true
80+
inlineScript: |
81+
kubectl apply -f test/integration/manifests/cilium/$(cilium_version)/cilium-config/cilium-config-hubble.yaml
82+
kubectl apply -f test/integration/manifests/cilium/$(cilium_version)/cilium-agent/files
83+
kubectl apply -f test/integration/manifests/cilium/$(cilium_version)/cilium-operator/files
84+
envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/$(cilium_version)/cilium-agent/templates/daemonset.tpl | kubectl apply -f -
85+
envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/$(cilium_version)/cilium-operator/templates/deployment.tpl | kubectl apply -f -
86+
# Use different file directories for nightly and current cilium version
87+
name: "installCilium"
88+
displayName: "Install Cilium on AKS Overlay"
89+
90+
- script: |
91+
echo "Start Azilium E2E Tests on Overlay Cluster"
92+
if [ "$CILIUM_VERSION_TAG" = "cilium-nightly-pipeline" ]
93+
then
94+
CNS=$(CNS_VERSION) IPAM=$(AZURE_IPAM_VERSION) && echo "Running nightly"
95+
else
96+
CNS=$(make cns-version) IPAM=$(make azure-ipam-version)
97+
fi
98+
sudo -E env "PATH=$PATH" make test-integration AZURE_IPAM_VERSION=${IPAM} CNS_VERSION=${CNS} INSTALL_CNS=true INSTALL_OVERLAY=true
99+
retryCountOnTaskFailure: 3
100+
name: "aziliumTest"
101+
displayName: "Install Azure-CNS and Run Azilium E2E on AKS Overlay"
102+
enabled: true
103+
104+
- script: |
105+
cilium status --wait --wait-duration 5m
106+
name: waitforhealthy
107+
displayName: "Wait for healthy cilium pods"
108+
109+
- script: |
110+
kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=5m
111+
name: waitforallpodsrunning
112+
displayName: "Wait for all pods to be running"
113+
retryCountOnTaskFailure: 3
114+
115+
- script: |
116+
echo "Run Cilium Connectivity Tests"
117+
cilium status
118+
cilium connectivity test --connect-timeout 4s --request-timeout 30s --test '!pod-to-pod-encryption,!node-to-node-encryption'
119+
retryCountOnTaskFailure: 3
120+
name: "ciliumConnectivityTests"
121+
displayName: "Run Cilium Connectivity Tests"
122+
enabled: true
123+
124+
- script: |
125+
kubectl apply -f test/integration/manifests/cilium/$(cilium_version)/hubble/hubble-peer-svc.yaml
126+
kubectl get pods -Aowide
127+
echo "verify Hubble metrics endpoint is usable"
128+
go test ./test/integration/networkobservability -tags=networkobservability
129+
retryCountOnTaskFailure: 3
130+
name: "HubbleConnectivityTests"
131+
displayName: "Run Hubble Connectivity Tests"
132+
133+
- stage: deleteCluster
134+
condition: always()
135+
dependsOn:
136+
- createCluster
137+
- setupCluster
138+
jobs:
139+
- job: delete
140+
steps:
141+
- template: ../templates/delete-cluster.yaml
142+
parameters:
143+
name: cilium_overlay_e2e
144+
clusterName: $(clustername)
145+
region: $(REGION_AKS_CLUSTER_TEST)

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ require (
9696
github.com/pelletier/go-toml/v2 v2.1.0 // indirect
9797
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
9898
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
99-
github.com/prometheus/common v0.45.0 // indirect
99+
github.com/prometheus/common v0.45.0
100100
github.com/prometheus/procfs v0.12.0 // indirect
101101
github.com/sirupsen/logrus v1.9.3
102102
github.com/spf13/afero v1.11.0 // indirect

‎test/integration/manifests/cilium/v1.14.4/cilium-agent/files/clusterrole.yaml‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
apiVersion: rbac.authorization.k8s.io/v1
2+
kind: ClusterRole
3+
metadata:
4+
name: cilium
5+
rules:
6+
- apiGroups:
7+
- networking.k8s.io
8+
resources:
9+
- networkpolicies
10+
verbs:
11+
- get
12+
- list
13+
- watch
14+
- apiGroups:
15+
- discovery.k8s.io
16+
resources:
17+
- endpointslices
18+
verbs:
19+
- get
20+
- list
21+
- watch
22+
- apiGroups:
23+
- ""
24+
resources:
25+
- namespaces
26+
- services
27+
- pods
28+
- endpoints
29+
- nodes
30+
verbs:
31+
- get
32+
- list
33+
- watch
34+
- apiGroups:
35+
- apiextensions.k8s.io
36+
resources:
37+
- customresourcedefinitions
38+
verbs:
39+
- list
40+
- watch
41+
# This is used when validating policies in preflight. This will need to stay
42+
# until we figure out how to avoid "get" inside the preflight, and then
43+
# should be removed ideally.
44+
- get
45+
- apiGroups:
46+
- cilium.io
47+
resources:
48+
#Naming changed from ciliumbgploadbalancerippools
49+
- ciliumloadbalancerippools
50+
- ciliumbgppeeringpolicies
51+
- ciliumclusterwideenvoyconfigs
52+
- ciliumclusterwidenetworkpolicies
53+
- ciliumegressgatewaypolicies
54+
- ciliumendpoints
55+
- ciliumendpointslices
56+
- ciliumenvoyconfigs
57+
- ciliumidentities
58+
- ciliumlocalredirectpolicies
59+
- ciliumnetworkpolicies
60+
- ciliumnodes
61+
- ciliumnodeconfigs
62+
#Added in 1.14.0 snapshot 2
63+
- ciliumcidrgroups
64+
verbs:
65+
- list
66+
- watch
67+
- apiGroups:
68+
- cilium.io
69+
resources:
70+
- ciliumidentities
71+
- ciliumendpoints
72+
- ciliumnodes
73+
verbs:
74+
- create
75+
- apiGroups:
76+
- cilium.io
77+
resources:
78+
- ciliumidentities
79+
verbs:
80+
- update
81+
- apiGroups:
82+
- cilium.io
83+
resources:
84+
- ciliumendpoints
85+
verbs:
86+
- delete
87+
- get
88+
- apiGroups:
89+
- cilium.io
90+
resources:
91+
- ciliumnodes
92+
- ciliumnodes/status
93+
verbs:
94+
- get
95+
- update
96+
- apiGroups:
97+
- cilium.io
98+
resources:
99+
- ciliumnetworkpolicies/status
100+
- ciliumclusterwidenetworkpolicies/status
101+
- ciliumendpoints/status
102+
- ciliumendpoints
103+
verbs:
104+
- patch

‎test/integration/manifests/cilium/v1.14.4/cilium-agent/files/clusterrolebinding.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
apiVersion: rbac.authorization.k8s.io/v1
2+
kind: ClusterRoleBinding
3+
metadata:
4+
name: cilium
5+
roleRef:
6+
apiGroup: rbac.authorization.k8s.io
7+
kind: ClusterRole
8+
name: cilium
9+
subjects:
10+
- kind: ServiceAccount
11+
name: "cilium"
12+
namespace: kube-system

‎test/integration/manifests/cilium/v1.14.4/cilium-agent/files/serviceaccount.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
apiVersion: v1
2+
kind: ServiceAccount
3+
metadata:
4+
name: "cilium"
5+
namespace: kube-system
6+

0 commit comments

Comments
 (0)