Skip to content

Commit 62c1e10

Browse files
nawazkhnawazkh
committed
wip of running e2e tests locally
1 parent c10ed44 commit 62c1e10

File tree

8 files changed

+130
-9
lines changed

8 files changed

+130
-9
lines changed

‎Makefile‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,7 @@ E2E_CONF_FILE_ENVSUBST := $(ROOT_DIR)/test/e2e/config/azure-dev-envsubst.yaml
186186
SKIP_CLEANUP ?= false
187187
AZWI_SKIP_CLEANUP ?= false
188188
SKIP_LOG_COLLECTION ?= false
189+
MGMT_CLUSTER_TYPE ?= kind
189190
# @sonasingh46: Skip creating mgmt cluster for ci as workload identity needs kind cluster
190191
# to be created with extra mounts for key pairs which is not yet supported
191192
# by existing e2e framework. A mgmt cluster(kind) is created as part of e2e suite
@@ -724,7 +725,13 @@ test-cover: test ## Run tests with code coverage and generate reports.
724725

725726
.PHONY: kind-create-bootstrap
726727
kind-create-bootstrap: $(KUBECTL) ## Create capz kind bootstrap cluster.
727-
KIND_CLUSTER_NAME=capz-e2e ./scripts/kind-with-registry.sh
728+
if [ "$(MGMT_CLUSTER_TYPE)" == "aks" ]; then \
729+
MGMT_CLUSTER_NAME=capz-e2e \
730+
AKS_RESOURCE_GROUP=capz-e2e \
731+
./scripts/aks-as-mgmt.sh; \
732+
else \
733+
KIND_CLUSTER_NAME=capz-e2e ./scripts/kind-with-registry.sh; \
734+
fi
728735

729736
.PHONY: cleanup-workload-identity
730737
cleanup-workload-identity: ## Cleanup CI workload-identity infra

‎scripts/aks-as-mgmt.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,7 @@ kustomize_substitutions:
211211
ASO_CREDENTIAL_SECRET_MODE: "${ASO_CREDENTIAL_SECRET_MODE}"
212212
REGISTRY: "${REGISTRY}"
213213
APISERVER_LB_DNS_SUFFIX: "${APISERVER_LB_DNS_SUFFIX}"
214+
AZURE_LOCATION: "${AZURE_LOCATION}"
214215
allowed_contexts:
215216
- "$MGMT_CLUSTER_NAME"
216217
- "kind-capz"

‎scripts/ci-e2e.sh‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,6 @@ set -o pipefail
2525

2626
# Install kubectl
2727
REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
28-
KUBECTL="${REPO_ROOT}/hack/tools/bin/kubectl"
29-
KIND="${REPO_ROOT}/hack/tools/bin/kind"
30-
make --directory="${REPO_ROOT}" "${KUBECTL##*/}" "${KIND##*/}"
3128

3229
# shellcheck source=hack/ensure-go.sh
3330
source "${REPO_ROOT}/hack/ensure-go.sh"

‎scripts/peer-vnets.sh‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
#!/usr/bin/env bash
2+
# Copyright 2024 The Kubernetes Authors.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
set -o errexit # exit immediately if a command exits with a non-zero status.
17+
set -o nounset # exit when script tries to use undeclared variables.
18+
set -o pipefail # make the pipeline fail if any command in it fails.
19+
20+
REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
21+
22+
source_tilt_settings() {
23+
if [ "$#" -ne 1 ]; then
24+
echo "Usage: $0 <tilt-settings.yaml>"
25+
exit 1
26+
fi
27+
28+
TILT_SETTINGS_FILE="$1"
29+
30+
# Check that the file exists
31+
if [ ! -f "$TILT_SETTINGS_FILE" ]; then
32+
echo "File not found: $TILT_SETTINGS_FILE"
33+
exit 1
34+
fi
35+
36+
echo "Reading variables from $TILT_SETTINGS_FILE under 'kustomize_substition'..."
37+
38+
# Get the list of keys under kustomize_substition
39+
VAR_KEYS=$(yq e '.kustomize_substition | keys | .[]' "$TILT_SETTINGS_FILE" 2>/dev/null || true)
40+
41+
# If there's no such key or it's empty, VAR_KEYS will be empty
42+
if [ -z "$VAR_KEYS" ]; then
43+
echo "No variables found under 'kustomize_substition'."
44+
else
45+
for key in $VAR_KEYS; do
46+
# Read the value of each key
47+
value=$(yq e ".kustomize_substition[\"$key\"]" "$TILT_SETTINGS_FILE")
48+
# Export the key/value pair
49+
export "$key=$value"
50+
echo "Exported $key=$value"
51+
done
52+
fi
53+
54+
echo "All variables exported"
55+
}
56+
57+
58+
peer_vnets() {
59+
# ------------------------------------------------------------------------------
60+
# Peer Vnets
61+
# ------------------------------------------------------------------------------
62+
63+
echo "--------Peering VNETs--------"
64+
az network vnet wait --resource-group ${AKS_RESOURCE_GROUP} --name ${AKS_MGMT_VNET_NAME} --created --timeout 180
65+
export MGMT_VNET_ID=$(az network vnet show --resource-group ${AKS_RESOURCE_GROUP} --name ${AKS_MGMT_VNET_NAME} --query id --output tsv)
66+
echo " 1/8 ${AKS_MGMT_VNET_NAME} found "
67+
68+
# wait for workload VNet to be created
69+
az network vnet wait --resource-group ${CLUSTER_NAME} --name ${CLUSTER_NAME}-vnet --created --timeout 180
70+
export WORKLOAD_VNET_ID=$(az network vnet show --resource-group ${CLUSTER_NAME} --name ${CLUSTER_NAME}-vnet --query id --output tsv)
71+
echo " 2/8 ${CLUSTER_NAME}-vnet found"
72+
73+
# peer mgmt vnet
74+
az network vnet peering create --name mgmt-to-${CLUSTER_NAME} --resource-group ${AKS_RESOURCE_GROUP} --vnet-name ${AKS_MGMT_VNET_NAME} --remote-vnet \"${WORKLOAD_VNET_ID}\" --allow-vnet-access true --allow-forwarded-traffic true --only-show-errors --output none
75+
az network vnet peering wait --name mgmt-to-${CLUSTER_NAME} --resource-group ${AKS_RESOURCE_GROUP} --vnet-name ${AKS_MGMT_VNET_NAME} --created --timeout 300 --only-show-errors --output none
76+
echo " 3/8 mgmt-to-${CLUSTER_NAME} peering created in ${AKS_MGMT_VNET_NAME}"
77+
78+
# peer workload vnet
79+
az network vnet peering create --name ${CLUSTER_NAME}-to-mgmt --resource-group ${CLUSTER_NAME} --vnet-name ${CLUSTER_NAME}-vnet --remote-vnet \"${MGMT_VNET_ID}\" --allow-vnet-access true --allow-forwarded-traffic true --only-show-errors --output none
80+
az network vnet peering wait --name ${CLUSTER_NAME}-to-mgmt --resource-group ${CLUSTER_NAME} --vnet-name ${CLUSTER_NAME}-vnet --created --timeout 300 --only-show-errors --output none
81+
echo " 4/8 ${CLUSTER_NAME}-to-mgmt peering created in ${CLUSTER_NAME}-vnet"
82+
83+
# create private DNS zone
84+
az network private-dns zone create --resource-group ${CLUSTER_NAME} --name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com --only-show-errors --output none
85+
az network private-dns zone wait --resource-group ${CLUSTER_NAME} --name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com --created --timeout 300 --only-show-errors --output none
86+
echo " 5/8 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com private DNS zone created in ${CLUSTER_NAME}"
87+
88+
# link private DNS Zone to workload vnet
89+
az network private-dns link vnet create --resource-group ${CLUSTER_NAME} --zone-name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com --name ${CLUSTER_NAME}-to-mgmt --virtual-network \"${WORKLOAD_VNET_ID}\" --registration-enabled false --only-show-errors --output none
90+
az network private-dns link vnet wait --resource-group ${CLUSTER_NAME} --zone-name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com --name ${CLUSTER_NAME}-to-mgmt --created --timeout 300 --only-show-errors --output none
91+
echo " 6/8 workload cluster vnet ${CLUSTER_NAME}-vnet linked with private DNS zone"
92+
93+
# link private DNS Zone to mgmt vnet
94+
az network private-dns link vnet create --resource-group ${CLUSTER_NAME} --zone-name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com --name mgmt-to-${CLUSTER_NAME} --virtual-network \"${MGMT_VNET_ID}\" --registration-enabled false --only-show-errors --output none
95+
az network private-dns link vnet wait --resource-group ${CLUSTER_NAME} --zone-name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com --name mgmt-to-${CLUSTER_NAME} --created --timeout 300 --only-show-errors --output none
96+
echo " 7/8 management cluster vnet ${AKS_MGMT_VNET_NAME} linked with private DNS zone"
97+
98+
# create private DNS zone record
99+
az network private-dns record-set a add-record --resource-group ${CLUSTER_NAME} --zone-name ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com --record-set-name \"@\" --ipv4-address ${AZURE_INTERNAL_LB_PRIVATE_IP} --only-show-errors --output none
100+
echo " 8/8 \"@\" private DNS zone record created to point ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com to ${AZURE_INTERNAL_LB_PRIVATE_IP}"
101+
}

‎test/e2e/azure_apiserver_ilb.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -462,3 +462,7 @@ func AzureAPIServerILBSpec(ctx context.Context, inputGetter func() AzureAPIServe
462462
err = wait.ExponentialBackoffWithContext(ctx, backoff, retryDSFn)
463463
Expect(err).NotTo(HaveOccurred())
464464
}
465+
466+
func PeerVnets(ctx context.Context, inputGetter func() AzureAPIServerILBSpecInput) {
467+
468+
}

‎test/e2e/azure_test.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1180,6 +1180,11 @@ var _ = Describe("Workload cluster creation", func() {
11801180
}
11811181
})
11821182
}),
1183+
withPreWaitForCluster(func() {
1184+
// TODO: only invoke this in local runs
1185+
// Peer VNets of the mgmt cluster and workload cluster
1186+
1187+
}),
11831188
), result)
11841189

11851190
By("Probing workload cluster with APIServerILB feature gate", func() {

‎test/e2e/common.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -443,3 +443,9 @@ func withAzureCNIv1Manifest(manifestPath string) func(*clusterctl.ApplyClusterTe
443443
input.CNIManifestPath = manifestPath
444444
}
445445
}
446+
447+
func withPreWaitForCluster(preWaitForCluster func()) func(*clusterctl.ApplyClusterTemplateAndWaitInput) {
448+
return func(input *clusterctl.ApplyClusterTemplateAndWaitInput) {
449+
input.PreWaitForCluster = preWaitForCluster
450+
}
451+
}

‎test/e2e/config/azure-dev.yaml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -226,9 +226,9 @@ variables:
226226
CONFORMANCE_IMAGE: "${CONFORMANCE_IMAGE:-}"
227227
CONFORMANCE_NODES: "${CONFORMANCE_NODES:-1}"
228228
IP_FAMILY: "IPv4"
229-
CLUSTER_IDENTITY_NAME: "cluster-identity-ci"
230-
ASO_CREDENTIAL_SECRET_NAME: "aso-credentials"
231-
ASO_CREDENTIAL_SECRET_MODE: workloadidentity
229+
CLUSTER_IDENTITY_NAME: "${CLUSTER_IDENTITY_NAME:-cluster-identity-ci}"
230+
ASO_CREDENTIAL_SECRET_NAME: "${ASO_CREDENTIAL_SECRET_NAME:-aso-credentials}"
231+
ASO_CREDENTIAL_SECRET_MODE: "${ASO_CREDENTIAL_SECRET_MODE:-workloadidentity}
232232
NODE_DRAIN_TIMEOUT: "60s"
233233
CI_VERSION: ""
234234
KUBETEST_CONFIGURATION: "./data/kubetest/conformance.yaml"
@@ -240,8 +240,8 @@ variables:
240240
LATEST_PROVIDER_UPGRADE_VERSION: "v1.18.0"
241241
OLD_CAAPH_UPGRADE_VERSION: "v0.1.0-alpha.10"
242242
LATEST_CAAPH_UPGRADE_VERSION: "v0.2.5"
243-
CI_RG: capz-ci
244-
USER_IDENTITY: cloud-provider-user-identity
243+
CI_RG: "${CI_RG:-capz-ci}"
244+
USER_IDENTITY: "${USER_IDENTITY:-cloud-provider-user-identity}"
245245
EXP_APISERVER_ILB: "true"
246246

247247
intervals:

0 commit comments

Comments
 (0)