Skip to content

Commit 9c44f1a

Browse files
barbacbdbarbacbd
committed
Adding e2e tests for shared vpc installs
1 parent cd4236f commit 9c44f1a

File tree

4 files changed

+214
-35
lines changed

4 files changed

+214
-35
lines changed

‎scripts/ci-e2e.sh‎

Lines changed: 61 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -102,27 +102,38 @@ EOF
102102

103103
# initialize a router and cloud NAT
104104
init_networks() {
105-
if [[ ${GCP_NETWORK_NAME} != "default" ]]; then
106-
gcloud compute networks create --project "$GCP_PROJECT" "${GCP_NETWORK_NAME}" --subnet-mode auto --quiet
107-
gcloud compute firewall-rules create "${GCP_NETWORK_NAME}"-allow-http --project "$GCP_PROJECT" \
108-
--allow tcp:80 --network "${GCP_NETWORK_NAME}" --quiet
109-
gcloud compute firewall-rules create "${GCP_NETWORK_NAME}"-allow-https --project "$GCP_PROJECT" \
110-
--allow tcp:443 --network "${GCP_NETWORK_NAME}" --quiet
111-
gcloud compute firewall-rules create "${GCP_NETWORK_NAME}"-allow-icmp --project "$GCP_PROJECT" \
112-
--allow icmp --network "${GCP_NETWORK_NAME}" --priority 65534 --quiet
113-
gcloud compute firewall-rules create "${GCP_NETWORK_NAME}"-allow-internal --project "$GCP_PROJECT" \
114-
--allow "tcp:0-65535,udp:0-65535,icmp" --network "${GCP_NETWORK_NAME}" --priority 65534 --quiet
115-
fi
116105

117-
gcloud compute firewall-rules list --project "$GCP_PROJECT"
118-
gcloud compute networks list --project="${GCP_PROJECT}"
119-
gcloud compute networks describe "${GCP_NETWORK_NAME}" --project="${GCP_PROJECT}"
106+
ARRAY=( "$GCP_PROJECT:$GCP_NETWORK_NAME"
107+
"$GCP_HOST_PROJECT:${GCP_NETWORK_NAME}-shared-vpc" )
108+
109+
for data in "${ARRAY[@]}"; do
110+
PROJECT=${data%%:*}
111+
NETWORK=${data#*:}
112+
113+
echo "Creating network resources in project ${PROJECT}"
114+
115+
if [[ ${NETWORK} != "default" ]]; then
116+
gcloud compute networks create --project "$PROJECT" "${NETWORK}" --subnet-mode auto --quiet
117+
gcloud compute firewall-rules create "${NETWORK}"-allow-http --project "$PROJECT" \
118+
--allow tcp:80 --network "${NETWORK}" --quiet
119+
gcloud compute firewall-rules create "${NETWORK}"-allow-https --project "$PROJECT" \
120+
--allow tcp:443 --network "${NETWORK}" --quiet
121+
gcloud compute firewall-rules create "${NETWORK}"-allow-icmp --project "$PROJECT" \
122+
--allow icmp --network "${NETWORK}" --priority 65534 --quiet
123+
gcloud compute firewall-rules create "${NETWORK}"-allow-internal --project "$PROJECT" \
124+
--allow "tcp:0-65535,udp:0-65535,icmp" --network "${NETWORK}" --priority 65534 --quiet
125+
fi
126+
127+
gcloud compute firewall-rules list --project "$PROJECT"
128+
gcloud compute networks list --project="${$PROJECT}"
129+
gcloud compute networks describe "${NETWORK}" --project="${$PROJECT}"
120130

121-
gcloud compute routers create "${TEST_NAME}-myrouter" --project="${GCP_PROJECT}" \
122-
--region="${GCP_REGION}" --network="${GCP_NETWORK_NAME}"
123-
gcloud compute routers nats create "${TEST_NAME}-mynat" --project="${GCP_PROJECT}" \
124-
--router-region="${GCP_REGION}" --router="${TEST_NAME}-myrouter" \
125-
--nat-all-subnet-ip-ranges --auto-allocate-nat-external-ips
131+
gcloud compute routers create "${TEST_NAME}-myrouter" --project="${$PROJECT}" \
132+
--region="${GCP_REGION}" --network="${NETWORK}"
133+
gcloud compute routers nats create "${TEST_NAME}-mynat" --project="${$PROJECT}" \
134+
--router-region="${GCP_REGION}" --router="${TEST_NAME}-myrouter" \
135+
--nat-all-subnet-ip-ranges --auto-allocate-nat-external-ips
136+
done
126137
}
127138

128139

@@ -146,23 +157,38 @@ cleanup() {
146157
(gcloud compute instance-groups list --project "$GCP_PROJECT" | grep capg-e2e \
147158
| awk '{print "gcloud compute instance-groups unmanaged delete --project '"$GCP_PROJECT"' --quiet " $1 " --zone " $2 "\n"}' \
148159
| bash) || true
149-
(gcloud compute firewall-rules list --project "$GCP_PROJECT" | grep capg-e2e \
150-
| awk '{print "gcloud compute firewall-rules delete --project '"$GCP_PROJECT"' --quiet " $1 "\n"}' \
151-
| bash) || true
152160

153-
# cleanup the networks
154-
gcloud compute routers nats delete "${TEST_NAME}-mynat" --project="${GCP_PROJECT}" \
155-
--router-region="${GCP_REGION}" --router="${TEST_NAME}-myrouter" --quiet || true
156-
gcloud compute routers delete "${TEST_NAME}-myrouter" --project="${GCP_PROJECT}" \
157-
--region="${GCP_REGION}" --quiet || true
158-
159-
if [[ ${GCP_NETWORK_NAME} != "default" ]]; then
160-
(gcloud compute firewall-rules list --project "$GCP_PROJECT" | grep "$GCP_NETWORK_NAME" \
161-
| awk '{print "gcloud compute firewall-rules delete --project '"$GCP_PROJECT"' --quiet " $1 "\n"}' \
162-
| bash) || true
163-
gcloud compute networks delete --project="${GCP_PROJECT}" \
164-
--quiet "${GCP_NETWORK_NAME}" || true
165-
fi
161+
162+
ARRAY=( "$GCP_PROJECT:$GCP_NETWORK_NAME"
163+
"$GCP_HOST_PROJECT:${GCP_NETWORK_NAME}-shared-vpc" )
164+
165+
for data in "${ARRAY[@]}"; do
166+
PROJECT=${data%%:*}
167+
NETWORK=${data#*:}
168+
169+
echo "Cleaning up network resources from project $PROJECT"
170+
171+
(gcloud compute firewall-rules list --project "$PROJECT" | grep capg-e2e \
172+
| awk '{print "gcloud compute firewall-rules delete --project '"$PROJECT"' --quiet " $1 "\n"}' \
173+
| bash) || true
174+
175+
# cleanup the networks
176+
gcloud compute routers nats delete "${TEST_NAME}-mynat" --project="${PROJECT}" \
177+
--router-region="${GCP_REGION}" --router="${TEST_NAME}-myrouter" --quiet || true
178+
gcloud compute routers delete "${TEST_NAME}-myrouter" --project="${PROJECT}" \
179+
--region="${GCP_REGION}" --quiet || true
180+
181+
if [[ ${NETWORK} != "default" ]]; then
182+
(gcloud compute firewall-rules list --project "$PROJECT" | grep "$NETWORK" \
183+
| awk '{print "gcloud compute firewall-rules delete --project '"$PROJECT"' --quiet " $1 "\n"}' \
184+
| bash) || true
185+
gcloud compute networks delete --project="${PROJECT}" \
186+
--quiet "${NETWORK}" || true
187+
fi
188+
189+
done
190+
191+
166192

167193
if [[ -n "${SKIP_INIT_IMAGE:-}" ]]; then
168194
echo "Skipping GCP image deletion..."

‎test/e2e/config/gcp-ci.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ providers:
7272
- sourcePath: "${PWD}/test/e2e/data/infrastructure-gcp/cluster-template-ci-gke.yaml"
7373
- sourcePath: "${PWD}/test/e2e/data/infrastructure-gcp/cluster-template-ci-gke-autopilot.yaml"
7474
- sourcePath: "${PWD}/test/e2e/data/infrastructure-gcp/cluster-template-ci-gke-custom-subnet.yaml"
75+
- sourcePath: "${PWD}/test/e2e/data/infrastructure-gcp/cluster-template-ci-with-shared-vpc.yaml"
7576

7677
variables:
7778
KUBERNETES_VERSION: "${KUBERNETES_VERSION:-v1.29.0}"

‎test/e2e/data/infrastructure-gcp/cluster-template-ci-with-shared-vpc.yaml‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
---
2+
apiVersion: cluster.x-k8s.io/v1beta1
3+
kind: Cluster
4+
metadata:
5+
name: "${CLUSTER_NAME}"
6+
labels:
7+
cni: "${CLUSTER_NAME}-shared-vpc"
8+
spec:
9+
clusterNetwork:
10+
pods:
11+
cidrBlocks: ["192.168.0.0/16"]
12+
infrastructureRef:
13+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
14+
kind: GCPCluster
15+
name: "${CLUSTER_NAME}"
16+
controlPlaneRef:
17+
kind: KubeadmControlPlane
18+
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
19+
name: "${CLUSTER_NAME}-control-plane"
20+
---
21+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
22+
kind: GCPCluster
23+
metadata:
24+
name: "${CLUSTER_NAME}"
25+
spec:
26+
project: "${GCP_PROJECT}"
27+
region: "${GCP_REGION}"
28+
network:
29+
name: "${GCP_NETWORK_NAME}"
30+
hostProject: "${GCP_HOST_PROJECT}"
31+
---
32+
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
33+
kind: KubeadmControlPlane
34+
metadata:
35+
name: "${CLUSTER_NAME}-control-plane"
36+
spec:
37+
replicas: ${CONTROL_PLANE_MACHINE_COUNT}
38+
machineTemplate:
39+
infrastructureRef:
40+
kind: GCPMachineTemplate
41+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
42+
name: "${CLUSTER_NAME}-control-plane"
43+
kubeadmConfigSpec:
44+
useExperimentalRetryJoin: true
45+
initConfiguration:
46+
nodeRegistration:
47+
name: '{{ ds.meta_data.local_hostname.split(".")[0] }}'
48+
kubeletExtraArgs:
49+
cloud-provider: gce
50+
clusterConfiguration:
51+
apiServer:
52+
timeoutForControlPlane: 20m
53+
extraArgs:
54+
cloud-provider: gce
55+
controllerManager:
56+
extraArgs:
57+
cloud-provider: gce
58+
allocate-node-cidrs: "false"
59+
kubernetesVersion: "${KUBERNETES_VERSION}"
60+
joinConfiguration:
61+
nodeRegistration:
62+
name: '{{ ds.meta_data.local_hostname.split(".")[0] }}'
63+
kubeletExtraArgs:
64+
cloud-provider: gce
65+
version: "${KUBERNETES_VERSION}"
66+
---
67+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
68+
kind: GCPMachineTemplate
69+
metadata:
70+
name: "${CLUSTER_NAME}-control-plane"
71+
spec:
72+
template:
73+
spec:
74+
instanceType: "${GCP_CONTROL_PLANE_MACHINE_TYPE}"
75+
image: "${IMAGE_ID}"
76+
---
77+
apiVersion: cluster.x-k8s.io/v1beta1
78+
kind: MachineDeployment
79+
metadata:
80+
name: "${CLUSTER_NAME}-md-0"
81+
spec:
82+
clusterName: "${CLUSTER_NAME}"
83+
replicas: ${WORKER_MACHINE_COUNT}
84+
selector:
85+
matchLabels:
86+
template:
87+
spec:
88+
clusterName: "${CLUSTER_NAME}"
89+
version: "${KUBERNETES_VERSION}"
90+
bootstrap:
91+
configRef:
92+
name: "${CLUSTER_NAME}-md-0"
93+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
94+
kind: KubeadmConfigTemplate
95+
infrastructureRef:
96+
name: "${CLUSTER_NAME}-md-0"
97+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
98+
kind: GCPMachineTemplate
99+
---
100+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
101+
kind: GCPMachineTemplate
102+
metadata:
103+
name: "${CLUSTER_NAME}-md-0"
104+
spec:
105+
template:
106+
spec:
107+
instanceType: "${GCP_NODE_MACHINE_TYPE}"
108+
image: "${IMAGE_ID}"
109+
---
110+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
111+
kind: KubeadmConfigTemplate
112+
metadata:
113+
name: "${CLUSTER_NAME}-md-0"
114+
spec:
115+
template:
116+
spec:
117+
joinConfiguration:
118+
nodeRegistration:
119+
name: '{{ ds.meta_data.local_hostname.split(".")[0] }}'
120+
kubeletExtraArgs:
121+
cloud-provider: gce
122+
---
123+
apiVersion: v1
124+
kind: ConfigMap
125+
metadata:
126+
name: "${CLUSTER_NAME}-shared-vpc"
127+
data: ${CNI_RESOURCES}

‎test/e2e/e2e_test.go‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,4 +182,29 @@ var _ = Describe("Workload cluster creation", func() {
182182
}, result)
183183
})
184184
})
185+
186+
Context("Creating a control-plane cluster with a shared vpc", func() {
187+
It("Should create a cluster with 1 control-plane and 1 worker node where the network exists in a host project", func() {
188+
By("Creating a cluster where the host project shares network resources with the service project")
189+
clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
190+
ClusterProxy: bootstrapClusterProxy,
191+
ConfigCluster: clusterctl.ConfigClusterInput{
192+
LogFolder: clusterctlLogFolder,
193+
ClusterctlConfigPath: clusterctlConfigPath,
194+
KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(),
195+
InfrastructureProvider: clusterctl.DefaultInfrastructureProvider,
196+
Flavor: "ci-with-shared-vpc",
197+
Namespace: namespace.Name,
198+
ClusterName: clusterName,
199+
KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion),
200+
ControlPlaneMachineCount: ptr.To[int64](1),
201+
WorkerMachineCount: ptr.To[int64](1),
202+
},
203+
WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"),
204+
WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"),
205+
WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"),
206+
}, result)
207+
})
208+
})
209+
185210
})

0 commit comments

Comments
 (0)