Skip to content

Commit ccb2cca

Browse files
author
Winnie Kwon
committed
Add multi-tenancy test based on ClusterClass
Also, renamed unmanaged_CAPI_quick_test_cluster_class.go to unmanaged_CAPI_quick_clusterclass_test.go as there are linter rules that applies to test files and the assumption is that test files end with `_test.go`.
1 parent b1b95ee commit ccb2cca

File tree

9 files changed

+433
-27
lines changed

9 files changed

+433
-27
lines changed

‎test/e2e/data/e2e_conf.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -229,6 +229,8 @@ providers:
229229
- sourcePath: "./infrastructure-aws/generated/cluster-template-peered-remote.yaml"
230230
- sourcePath: "./infrastructure-aws/generated/cluster-template-internal-elb.yaml"
231231
- sourcePath: "./infrastructure-aws/kustomize_sources/topology/clusterclass-quick-start.yaml"
232+
- sourcePath: "./infrastructure-aws/generated/cluster-template-nested-multitenancy-clusterclass.yaml"
233+
- sourcePath: "./infrastructure-aws/kustomize_sources/nested-multitenancy-clusterclass/clusterclass-multi-tenancy.yaml"
232234
- sourcePath: "./shared/v1beta1_provider/metadata.yaml"
233235
- sourcePath: "./infrastructure-aws/generated/cluster-template-ignition.yaml"
234236
replacements:

‎test/e2e/data/infrastructure-aws/kustomize_sources/nested-multitenancy-clusterclass/cluster-template.yaml

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
---
2+
apiVersion: cluster.x-k8s.io/v1beta1
3+
kind: Cluster
4+
metadata:
5+
labels:
6+
cni: ${CLUSTER_NAME}-crs-0
7+
name: "${CLUSTER_NAME}"
8+
spec:
9+
clusterNetwork:
10+
pods:
11+
cidrBlocks: ["192.168.0.0/16"]
12+
topology:
13+
class: "multi-tenancy"
14+
version: "${KUBERNETES_VERSION}"
15+
controlPlane:
16+
replicas: ${CONTROL_PLANE_MACHINE_COUNT}
17+
workers:
18+
machineDeployments:
19+
- class: "default-worker"
20+
name: "md-0"
21+
replicas: ${WORKER_MACHINE_COUNT}
22+
variables:
23+
- name: region
24+
value: ${AWS_REGION}
25+
- name: sshKeyName
26+
value: ${AWS_SSH_KEY_NAME}
27+
- name: controlPlaneMachineType
28+
value: ${AWS_CONTROL_PLANE_MACHINE_TYPE}
29+
- name: workerMachineType
30+
value: ${AWS_NODE_MACHINE_TYPE}
31+
- name: bastionEnabled
32+
value: true
33+
- name: vpcAZUsageLimit
34+
value: 1
35+
- name: identityRef
36+
value:
37+
kind: AWSClusterRoleIdentity
38+
name: "${MULTI_TENANCY_NESTED_IDENTITY_NAME}"
39+
---
40+
apiVersion: v1
41+
data: ${CNI_RESOURCES}
42+
kind: ConfigMap
43+
metadata:
44+
name: cni-${CLUSTER_NAME}-crs-0
45+
---
46+
apiVersion: addons.cluster.x-k8s.io/v1beta1
47+
kind: ClusterResourceSet
48+
metadata:
49+
name: ${CLUSTER_NAME}-crs-0
50+
spec:
51+
clusterSelector:
52+
matchLabels:
53+
cni: ${CLUSTER_NAME}-crs-0
54+
resources:
55+
- kind: ConfigMap
56+
name: cni-${CLUSTER_NAME}-crs-0
57+
strategy: ApplyOnce

‎test/e2e/data/infrastructure-aws/kustomize_sources/nested-multitenancy-clusterclass/clusterclass-multi-tenancy.yaml

Lines changed: 214 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,214 @@
1+
apiVersion: cluster.x-k8s.io/v1beta1
2+
kind: ClusterClass
3+
metadata:
4+
name: multi-tenancy
5+
spec:
6+
controlPlane:
7+
ref:
8+
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
9+
kind: KubeadmControlPlaneTemplate
10+
name: multi-tenancy-control-plane
11+
machineInfrastructure:
12+
ref:
13+
kind: AWSMachineTemplate
14+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
15+
name: multi-tenancy-control-plane
16+
infrastructure:
17+
ref:
18+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
19+
kind: AWSClusterTemplate
20+
name: multi-tenancy
21+
workers:
22+
machineDeployments:
23+
- class: default-worker
24+
template:
25+
bootstrap:
26+
ref:
27+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
28+
kind: KubeadmConfigTemplate
29+
name: multi-tenancy-worker-bootstraptemplate
30+
infrastructure:
31+
ref:
32+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
33+
kind: AWSMachineTemplate
34+
name: multi-tenancy-worker-machinetemplate
35+
variables:
36+
- name: region
37+
required: true
38+
schema:
39+
openAPIV3Schema:
40+
type: string
41+
default: us-east-1
42+
- name: sshKeyName
43+
required: true
44+
schema:
45+
openAPIV3Schema:
46+
type: string
47+
default: default
48+
- name: controlPlaneMachineType
49+
required: true
50+
schema:
51+
openAPIV3Schema:
52+
type: string
53+
default: t3.large
54+
- name: workerMachineType
55+
required: true
56+
schema:
57+
openAPIV3Schema:
58+
type: string
59+
default: t3.large
60+
- name: bastionEnabled
61+
required: false
62+
schema:
63+
openAPIV3Schema:
64+
type: boolean
65+
- name: vpcAZUsageLimit
66+
required: false
67+
schema:
68+
openAPIV3Schema:
69+
type: integer
70+
- name: identityRef
71+
required: false
72+
schema:
73+
openAPIV3Schema:
74+
type: object
75+
properties:
76+
kind:
77+
type: string
78+
name:
79+
type: string
80+
required:
81+
- kind
82+
- name
83+
patches:
84+
- name: awsClusterTemplateGeneral
85+
definitions:
86+
- selector:
87+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
88+
kind: AWSClusterTemplate
89+
matchResources:
90+
infrastructureCluster: true
91+
jsonPatches:
92+
- op: add
93+
path: "/spec/template/spec/region"
94+
valueFrom:
95+
variable: region
96+
- op: add
97+
path: "/spec/template/spec/sshKeyName"
98+
valueFrom:
99+
variable: sshKeyName
100+
- op: replace
101+
path: "/spec/template/spec/bastion/enabled"
102+
valueFrom:
103+
variable: bastionEnabled
104+
- op: replace
105+
path: "/spec/template/spec/network/vpc/availabilityZoneUsageLimit"
106+
valueFrom:
107+
variable: vpcAZUsageLimit
108+
- op: replace
109+
path: "/spec/template/spec/identityRef"
110+
valueFrom:
111+
variable: identityRef
112+
- name: awsMachineTemplateControlPlane
113+
definitions:
114+
- selector:
115+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
116+
kind: AWSMachineTemplate
117+
matchResources:
118+
controlPlane: true
119+
jsonPatches:
120+
- op: replace
121+
path: "/spec/template/spec/instanceType"
122+
valueFrom:
123+
variable: controlPlaneMachineType
124+
- op: add
125+
path: "/spec/template/spec/sshKeyName"
126+
valueFrom:
127+
variable: sshKeyName
128+
- name: awsMachineTemplateWorker
129+
definitions:
130+
- selector:
131+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
132+
kind: AWSMachineTemplate
133+
matchResources:
134+
machineDeploymentClass:
135+
names:
136+
- default-worker
137+
jsonPatches:
138+
- op: replace
139+
path: "/spec/template/spec/instanceType"
140+
valueFrom:
141+
variable: workerMachineType
142+
- op: add
143+
path: "/spec/template/spec/sshKeyName"
144+
valueFrom:
145+
variable: sshKeyName
146+
---
147+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
148+
kind: AWSClusterTemplate
149+
metadata:
150+
name: multi-tenancy
151+
spec:
152+
template:
153+
spec: {}
154+
---
155+
kind: KubeadmControlPlaneTemplate
156+
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
157+
metadata:
158+
name: multi-tenancy-control-plane
159+
spec:
160+
template:
161+
spec:
162+
kubeadmConfigSpec:
163+
clusterConfiguration:
164+
apiServer:
165+
extraArgs:
166+
cloud-provider: aws
167+
controllerManager:
168+
extraArgs:
169+
cloud-provider: aws
170+
initConfiguration:
171+
nodeRegistration:
172+
name: '{{ ds.meta_data.local_hostname }}'
173+
kubeletExtraArgs:
174+
cloud-provider: aws
175+
joinConfiguration:
176+
nodeRegistration:
177+
name: '{{ ds.meta_data.local_hostname }}'
178+
kubeletExtraArgs:
179+
cloud-provider: aws
180+
---
181+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
182+
kind: AWSMachineTemplate
183+
metadata:
184+
name: multi-tenancy-control-plane
185+
spec:
186+
template:
187+
spec:
188+
# instanceType is a required field (OpenAPI schema).
189+
instanceType: REPLACEME
190+
iamInstanceProfile: "control-plane.cluster-api-provider-aws.sigs.k8s.io"
191+
---
192+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
193+
kind: AWSMachineTemplate
194+
metadata:
195+
name: multi-tenancy-worker-machinetemplate
196+
spec:
197+
template:
198+
spec:
199+
# instanceType is a required field (OpenAPI schema).
200+
instanceType: REPLACEME
201+
iamInstanceProfile: "nodes.cluster-api-provider-aws.sigs.k8s.io"
202+
---
203+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
204+
kind: KubeadmConfigTemplate
205+
metadata:
206+
name: "multi-tenancy-worker-bootstraptemplate"
207+
spec:
208+
template:
209+
spec:
210+
joinConfiguration:
211+
nodeRegistration:
212+
name: '{{ ds.meta_data.local_hostname }}'
213+
kubeletExtraArgs:
214+
cloud-provider: aws

‎test/e2e/data/infrastructure-aws/kustomize_sources/nested-multitenancy-clusterclass/kustomization.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
apiVersion: kustomize.config.k8s.io/v1beta1
2+
kind: Kustomization
3+
resources:
4+
- cluster-template.yaml
5+
- role.yaml

‎test/e2e/data/infrastructure-aws/kustomize_sources/nested-multitenancy-clusterclass/role.yaml

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
---
2+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
3+
kind: AWSClusterRoleIdentity
4+
metadata:
5+
name: "${MULTI_TENANCY_JUMP_IDENTITY_NAME}"
6+
spec:
7+
roleARN: "${MULTI_TENANCY_JUMP_ROLE_ARN}"
8+
durationSeconds: 900
9+
sessionName: "${MULTI_TENANCY_JUMP_IDENTITY_NAME}-session"
10+
sourceIdentityRef:
11+
kind: AWSClusterControllerIdentity
12+
name: "default"
13+
allowedNamespaces: {}
14+
---
15+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
16+
kind: AWSClusterRoleIdentity
17+
metadata:
18+
name: "${MULTI_TENANCY_NESTED_IDENTITY_NAME}"
19+
spec:
20+
roleARN: "${MULTI_TENANCY_NESTED_ROLE_ARN}"
21+
sessionName: "${MULTI_TENANCY_NESTED_IDENTITY_NAME}-session"
22+
sourceIdentityRef:
23+
kind: AWSClusterRoleIdentity
24+
name: "${MULTI_TENANCY_JUMP_IDENTITY_NAME}"
25+
allowedNamespaces: {}

‎test/e2e/shared/defaults.go

Lines changed: 26 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -35,31 +35,32 @@ import (
3535

3636
// Constants.
3737
const (
38-
DefaultSSHKeyPairName = "cluster-api-provider-aws-sigs-k8s-io"
39-
AMIPrefix = "capa-ami-ubuntu-18.04-"
40-
DefaultImageLookupOrg = "258751437250"
41-
KubernetesVersion = "KUBERNETES_VERSION"
42-
KubernetesVersionManagement = "KUBERNETES_VERSION_MANAGEMENT"
43-
CNIPath = "CNI"
44-
CNIResources = "CNI_RESOURCES"
45-
AwsNodeMachineType = "AWS_NODE_MACHINE_TYPE"
46-
AwsAvailabilityZone1 = "AWS_AVAILABILITY_ZONE_1"
47-
AwsAvailabilityZone2 = "AWS_AVAILABILITY_ZONE_2"
48-
MultiAzFlavor = "multi-az"
49-
LimitAzFlavor = "limit-az"
50-
SpotInstancesFlavor = "spot-instances"
51-
SSMFlavor = "ssm"
52-
UpgradeToMain = "upgrade-to-main"
53-
ExternalCloudProvider = "external-cloud-provider"
54-
SimpleMultitenancyFlavor = "simple-multitenancy"
55-
NestedMultitenancyFlavor = "nested-multitenancy"
56-
KCPScaleInFlavor = "kcp-scale-in"
57-
IgnitionFlavor = "ignition"
58-
StorageClassOutTreeZoneLabel = "topology.ebs.csi.aws.com/zone"
59-
GPUFlavor = "gpu"
60-
InstanceVcpu = "AWS_MACHINE_TYPE_VCPU_USAGE"
61-
PreCSIKubernetesVer = "PRE_1_23_KUBERNETES_VERSION"
62-
PostCSIKubernetesVer = "POST_1_23_KUBERNETES_VERSION"
38+
DefaultSSHKeyPairName = "cluster-api-provider-aws-sigs-k8s-io"
39+
AMIPrefix = "capa-ami-ubuntu-18.04-"
40+
DefaultImageLookupOrg = "258751437250"
41+
KubernetesVersion = "KUBERNETES_VERSION"
42+
KubernetesVersionManagement = "KUBERNETES_VERSION_MANAGEMENT"
43+
CNIPath = "CNI"
44+
CNIResources = "CNI_RESOURCES"
45+
AwsNodeMachineType = "AWS_NODE_MACHINE_TYPE"
46+
AwsAvailabilityZone1 = "AWS_AVAILABILITY_ZONE_1"
47+
AwsAvailabilityZone2 = "AWS_AVAILABILITY_ZONE_2"
48+
MultiAzFlavor = "multi-az"
49+
LimitAzFlavor = "limit-az"
50+
SpotInstancesFlavor = "spot-instances"
51+
SSMFlavor = "ssm"
52+
UpgradeToMain = "upgrade-to-main"
53+
ExternalCloudProvider = "external-cloud-provider"
54+
SimpleMultitenancyFlavor = "simple-multitenancy"
55+
NestedMultitenancyFlavor = "nested-multitenancy"
56+
NestedMultitenancyClusterClassFlavor = "nested-multitenancy-clusterclass"
57+
KCPScaleInFlavor = "kcp-scale-in"
58+
IgnitionFlavor = "ignition"
59+
StorageClassOutTreeZoneLabel = "topology.ebs.csi.aws.com/zone"
60+
GPUFlavor = "gpu"
61+
InstanceVcpu = "AWS_MACHINE_TYPE_VCPU_USAGE"
62+
PreCSIKubernetesVer = "PRE_1_23_KUBERNETES_VERSION"
63+
PostCSIKubernetesVer = "POST_1_23_KUBERNETES_VERSION"
6364
)
6465

6566
var ResourceQuotaFilePath = "/tmp/capa-e2e-resource-usage.lock"

‎test/e2e/suites/unmanaged/helpers_test.go

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,12 +76,21 @@ type statefulSetInfo struct {
7676

7777
// GetClusterByName returns a Cluster object given his name.
7878
func GetAWSClusterByName(ctx context.Context, namespace, name string) (*infrav1.AWSCluster, error) {
79-
awsCluster := &infrav1.AWSCluster{}
79+
cluster := &clusterv1.Cluster{}
8080
key := crclient.ObjectKey{
8181
Namespace: namespace,
8282
Name: name,
8383
}
84-
err := e2eCtx.Environment.BootstrapClusterProxy.GetClient().Get(ctx, key, awsCluster)
84+
if err := e2eCtx.Environment.BootstrapClusterProxy.GetClient().Get(ctx, key, cluster); err != nil {
85+
return nil, err
86+
}
87+
88+
awsCluster := &infrav1.AWSCluster{}
89+
awsClusterKey := crclient.ObjectKey{
90+
Namespace: namespace,
91+
Name: cluster.Spec.InfrastructureRef.Name,
92+
}
93+
err := e2eCtx.Environment.BootstrapClusterProxy.GetClient().Get(ctx, awsClusterKey, awsCluster)
8594
return awsCluster, err
8695
}
8796

‎test/e2e/suites/unmanaged/unmanaged_CAPI_quick_test_cluster_class.go renamed to ‎test/e2e/suites/unmanaged/unmanaged_CAPI_quick_clusterclass_test.go

File renamed without changes.

0 commit comments

Comments
 (0)