Skip to content

Commit 309ade3

Browse files
willie-yaowillie-yao
committed
Add templates to test Azure Linux 3
1 parent 4ad8f96 commit 309ade3

14 files changed

+2368
-3
lines changed

‎templates/test/ci/cluster-template-prow-azl3.yaml‎

Lines changed: 449 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎templates/test/ci/cluster-template-prow-ci-version-azl3.yaml‎

Lines changed: 1510 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎templates/test/ci/prow-azl3/kustomization.yaml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
apiVersion: kustomize.config.k8s.io/v1beta1
2+
kind: Kustomization
3+
namespace: default
4+
resources:
5+
- ../../../flavors/default
6+
- ../../../addons/cluster-api-helm/calico.yaml
7+
- ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml
8+
- ../../../addons/cluster-api-helm/cloud-provider-azure.yaml
9+
- ../../../addons/cluster-api-helm/cloud-provider-azure-ci.yaml
10+
patches:
11+
- path: ../patches/tags.yaml
12+
- path: ../patches/mhc.yaml
13+
- path: ../patches/controller-manager.yaml
14+
- path: ../patches/uami-md-0.yaml
15+
- path: ../patches/uami-control-plane.yaml
16+
- path: ../patches/cluster-label-calico.yaml
17+
- path: ../patches/cluster-label-cloud-provider-azure.yaml
18+
- path: patches/controller-manager.yaml
19+
target:
20+
group: controlplane.cluster.x-k8s.io
21+
kind: KubeadmControlPlane
22+
name: .*-control-plane
23+
version: v1beta1
24+
- path: patches/kubeadm-config-template-azl3.yaml
25+
target:
26+
group: bootstrap.cluster.x-k8s.io
27+
kind: KubeadmConfigTemplate
28+
name: .*-md-0
29+
namespace: default
30+
version: v1beta1
31+
- path: patches/azuremachinetemplate-azl3-image.yaml
32+
- path: patches/cloud-provider-azure-cacertdir.yaml
33+
- path: patches/cloud-provider-azure-ci-cacertdir.yaml
34+
35+
sortOptions:
36+
order: fifo

‎templates/test/ci/prow-azl3/patches/azuremachinetemplate-azl3-image.yaml‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
2+
kind: AzureMachineTemplate
3+
metadata:
4+
name: ${CLUSTER_NAME}-control-plane
5+
spec:
6+
template:
7+
spec:
8+
image:
9+
computeGallery:
10+
gallery: ClusterAPI-f72ceb4f-5159-4c26-a0fe-2ea738f0d019
11+
name: capi-azurelinux-3
12+
version: ${AZL3_VERSION}
13+
---
14+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
15+
kind: AzureMachineTemplate
16+
metadata:
17+
name: ${CLUSTER_NAME}-md-0
18+
spec:
19+
template:
20+
spec:
21+
image:
22+
computeGallery:
23+
gallery: ClusterAPI-f72ceb4f-5159-4c26-a0fe-2ea738f0d019
24+
name: capi-azurelinux-3
25+
version: ${AZL3_VERSION}

‎templates/test/ci/prow-azl3/patches/cloud-provider-azure-cacertdir.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
apiVersion: addons.cluster.x-k8s.io/v1alpha1
2+
kind: HelmChartProxy
3+
metadata:
4+
name: cloud-provider-azure-chart
5+
spec:
6+
valuesTemplate: |
7+
infra:
8+
clusterName: {{ .Cluster.metadata.name }}
9+
cloudControllerManager:
10+
caCertDir: "/etc/pki/tls/certs"
11+
clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }}
12+
logVerbosity: 4

‎templates/test/ci/prow-azl3/patches/cloud-provider-azure-ci-cacertdir.yaml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
apiVersion: addons.cluster.x-k8s.io/v1alpha1
2+
kind: HelmChartProxy
3+
metadata:
4+
name: cloud-provider-azure-chart-ci
5+
spec:
6+
valuesTemplate: |
7+
infra:
8+
clusterName: {{ .Cluster.metadata.name }}
9+
cloudControllerManager:
10+
caCertDir: "/etc/pki/tls/certs"
11+
cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"}
12+
cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""}
13+
clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }}
14+
imageName: "${CCM_IMAGE_NAME:-""}"
15+
imageRepository: "${IMAGE_REGISTRY:-""}"
16+
imageTag: "${IMAGE_TAG_CCM:-""}"
17+
logVerbosity: ${CCM_LOG_VERBOSITY:-4}
18+
replicas: ${CCM_COUNT:-1}
19+
enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false}
20+
cloudNodeManager:
21+
imageName: "${CNM_IMAGE_NAME:-""}"
22+
imageRepository: "${IMAGE_REGISTRY:-""}"
23+
imageTag: "${IMAGE_TAG_CNM:-""}"

‎templates/test/ci/prow-azl3/patches/controller-manager.yaml‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
- op: add
2+
path: /spec/kubeadmConfigSpec/files/0
3+
value:
4+
content: |
5+
#!/bin/bash
6+
7+
set -o nounset
8+
set -o pipefail
9+
set -o errexit
10+
11+
# Install ca-certificates packages for Azure Linux
12+
tdnf install -y ca-certificates ca-certificates-legacy
13+
update-ca-trust
14+
15+
# Allow Azure service IP addresses (required for Azure resources)
16+
iptables -A INPUT -s 168.63.129.16 -j ACCEPT
17+
iptables -A OUTPUT -d 168.63.129.16 -j ACCEPT
18+
19+
# Kubernetes API Server (port 6443) - bound to all IPv6 interfaces, needs external access
20+
iptables -A INPUT -p tcp --dport 6443 -j ACCEPT
21+
22+
# etcd server communication - external access needed for cluster communication
23+
# Port 2379 is bound to node IP (10.0.0.5), needs cluster access
24+
iptables -A INPUT -p tcp --dport 2379 -j ACCEPT
25+
# Port 2380 is bound to node IP (10.0.0.5), needs cluster access
26+
iptables -A INPUT -p tcp --dport 2380 -j ACCEPT
27+
# Port 2381 is localhost only, no external rule needed
28+
29+
# Allow traffic to Kubernetes service network (10.96.0.0/12) - CRITICAL: required for pod-to-service communication
30+
iptables -A OUTPUT -d 10.96.0.0/12 -j ACCEPT
31+
iptables -A INPUT -s 10.96.0.0/12 -j ACCEPT
32+
33+
# Allow traffic to/from node network (10.1.0.0/24) - required for node-to-node communication
34+
iptables -A OUTPUT -d 10.1.0.0/24 -j ACCEPT
35+
iptables -A INPUT -s 10.1.0.0/24 -j ACCEPT
36+
37+
# Allow traffic to/from Calico pod network - more restrictive than full 192.168.0.0/16
38+
# Only allow the specific pod CIDR ranges that Calico actually uses
39+
iptables -A OUTPUT -d 192.168.0.0/24 -j ACCEPT
40+
iptables -A INPUT -s 192.168.0.0/24 -j ACCEPT
41+
42+
# Save the rules following Azure Linux 3 approach
43+
iptables-save > /etc/systemd/scripts/ip4save
44+
path: /tmp/azl3-setup.sh
45+
owner: "root:root"
46+
permissions: "0744"
47+
- op: add
48+
path: /spec/kubeadmConfigSpec/preKubeadmCommands/0
49+
value:
50+
bash -c /tmp/azl3-setup.sh

‎templates/test/ci/prow-azl3/patches/disable-vm-bootstrap-extension.yaml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
2+
kind: AzureMachineTemplate
3+
metadata:
4+
name: ${CLUSTER_NAME}-control-plane
5+
spec:
6+
template:
7+
spec:
8+
disableVMBootstrapExtension: true
9+
---
10+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
11+
kind: AzureMachineTemplate
12+
metadata:
13+
name: ${CLUSTER_NAME}-md-0
14+
spec:
15+
template:
16+
spec:
17+
disableVMBootstrapExtension: true

‎templates/test/ci/prow-azl3/patches/kubeadm-config-template-azl3.yaml‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
- op: add
2+
path: /spec/template/spec/files/0
3+
value:
4+
content: |
5+
#!/bin/bash
6+
7+
set -o nounset
8+
set -o pipefail
9+
set -o errexit
10+
11+
# Install ca-certificates packages for Azure Linux
12+
tdnf install -y ca-certificates ca-certificates-legacy
13+
update-ca-trust
14+
15+
# Azure Linux 3 firewall configuration for worker nodes
16+
# Keep the default DROP policy for security, only add specific ACCEPT rules
17+
18+
# Allow Azure service IP addresses (required for Azure resources)
19+
iptables -A INPUT -s 168.63.129.16 -j ACCEPT
20+
iptables -A OUTPUT -d 168.63.129.16 -j ACCEPT
21+
ip6tables -A INPUT -s fe80::1234:5678:9abc -j ACCEPT
22+
ip6tables -A OUTPUT -d fe80::1234:5678:9abc -j ACCEPT
23+
24+
# Allow localhost traffic (required for many localhost-bound services)
25+
iptables -A INPUT -i lo -j ACCEPT
26+
iptables -A OUTPUT -o lo -j ACCEPT
27+
ip6tables -A INPUT -i lo -j ACCEPT
28+
ip6tables -A OUTPUT -o lo -j ACCEPT
29+
30+
# Allow established and related connections
31+
iptables -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
32+
iptables -A OUTPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
33+
ip6tables -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
34+
ip6tables -A OUTPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
35+
36+
# SSH (port 22) - bound to all interfaces, needs external access
37+
iptables -A INPUT -p tcp --dport 22 -j ACCEPT
38+
ip6tables -A INPUT -p tcp --dport 22 -j ACCEPT
39+
40+
# Kubelet API (port 10250) - bound to all IPv6 interfaces, needs cluster access
41+
iptables -A INPUT -p tcp --dport 10250 -j ACCEPT
42+
ip6tables -A INPUT -p tcp --dport 10250 -j ACCEPT
43+
44+
# kube-proxy (port 10256) - bound to all IPv6 interfaces, needs cluster access
45+
iptables -A INPUT -p tcp --dport 10256 -j ACCEPT
46+
ip6tables -A INPUT -p tcp --dport 10256 -j ACCEPT
47+
48+
# Calico networking requirements
49+
# Calico Typha (port 5473) - bound to all IPv6 interfaces, needs cluster access
50+
iptables -A INPUT -p tcp --dport 5473 -j ACCEPT
51+
ip6tables -A INPUT -p tcp --dport 5473 -j ACCEPT
52+
53+
# VXLAN for overlay networking (port 4789 UDP) - bound to all interfaces
54+
iptables -A INPUT -p udp --dport 4789 -j ACCEPT
55+
ip6tables -A INPUT -p udp --dport 4789 -j ACCEPT
56+
57+
# Calico metrics ports (29603, 29605) - bound to all IPv6 interfaces
58+
iptables -A INPUT -p tcp --dport 29603 -j ACCEPT
59+
iptables -A INPUT -p tcp --dport 29605 -j ACCEPT
60+
ip6tables -A INPUT -p tcp --dport 29603 -j ACCEPT
61+
ip6tables -A INPUT -p tcp --dport 29605 -j ACCEPT
62+
63+
# BGP for node-to-node communication (port 179) - not in netstat but needed for Calico
64+
iptables -A INPUT -p tcp --dport 179 -j ACCEPT
65+
ip6tables -A INPUT -p tcp --dport 179 -j ACCEPT
66+
67+
# IP-in-IP protocol for Calico
68+
iptables -A INPUT -p 4 -j ACCEPT
69+
ip6tables -A INPUT -p 41 -j ACCEPT
70+
71+
# DHCP client (port 68 UDP) - for IP assignment
72+
iptables -A INPUT -p udp --dport 68 -j ACCEPT
73+
ip6tables -A INPUT -p udp --dport 68 -j ACCEPT
74+
75+
# NTP (port 323 UDP) - for time synchronization
76+
iptables -A INPUT -p udp --dport 323 -j ACCEPT
77+
ip6tables -A INPUT -p udp --dport 323 -j ACCEPT
78+
79+
# Allow ICMP for connectivity checks
80+
iptables -A INPUT -p icmp -j ACCEPT
81+
ip6tables -A INPUT -p ipv6-icmp -j ACCEPT
82+
83+
# Allow traffic to Kubernetes service network (10.96.0.0/12) - required for pod-to-service communication
84+
iptables -A OUTPUT -d 10.96.0.0/12 -j ACCEPT
85+
iptables -A INPUT -s 10.96.0.0/12 -j ACCEPT
86+
87+
# Allow traffic to/from Calico pod network (192.168.0.0/16) - required for pod-to-pod communication
88+
iptables -A OUTPUT -d 192.168.0.0/16 -j ACCEPT
89+
iptables -A INPUT -s 192.168.0.0/16 -j ACCEPT
90+
91+
# Allow traffic to/from node network (10.1.0.0/24) - required for node-to-node communication
92+
iptables -A OUTPUT -d 10.1.0.0/24 -j ACCEPT
93+
iptables -A INPUT -s 10.1.0.0/24 -j ACCEPT
94+
95+
# Save the rules following Azure Linux 3 approach
96+
iptables-save > /etc/systemd/scripts/ip4save
97+
ip6tables-save > /etc/systemd/scripts/ip6save
98+
path: /tmp/azl3-setup.sh
99+
owner: "root:root"
100+
permissions: "0744"
101+
- op: add
102+
path: /spec/template/spec/preKubeadmCommands/0
103+
value:
104+
bash -c /tmp/azl3-setup.sh

‎templates/test/ci/prow-azl3/patches/remove-marketplace-image.yaml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
2+
kind: AzureMachineTemplate
3+
metadata:
4+
name: ${CLUSTER_NAME}-control-plane
5+
spec:
6+
template:
7+
spec:
8+
image:
9+
marketplace: null
10+
---
11+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
12+
kind: AzureMachineTemplate
13+
metadata:
14+
name: ${CLUSTER_NAME}-md-0
15+
spec:
16+
template:
17+
spec:
18+
image:
19+
marketplace: null

0 commit comments

Comments
 (0)