diff --git a/CHANGELOG.md b/CHANGELOG.md index 8aad1d181..e17eabe07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- *This change will roll the nodes* Add Crossplane IAM Roles, policies and instance profiles for worker and control plane nodes. Instead of having an IAM Role per node pool, now we'll use the same for all node pools. + +### Removed + +- Removed `reducedInstanceProfileIamPermissionsForWorkers` value, as that's the default behavior now. + ## [6.4.0] - 2025-10-28 ### Added diff --git a/helm/cluster-aws/README.md b/helm/cluster-aws/README.md index dec66b20f..97a7bb23e 100644 --- a/helm/cluster-aws/README.md +++ b/helm/cluster-aws/README.md @@ -32,7 +32,6 @@ Properties within the `.global.providerSpecific` object | `global.providerSpecific.irsaCrossplane` | **Use Crossplane to provision IRSA infrastructure** - Defaults to true. Crossplane will adopt all the resources created by IRSA Operator. If set to false, the IRSA Operator will take over the infrastructure again.|**Type:** `[boolean]`
**Default:** `true`| | `global.providerSpecific.nodePoolAmi` | **Amazon machine image (AMI) for node pools** - If specified, this image will be used to provision EC2 instances for node pools.|**Type:** `[string]`
| | `global.providerSpecific.nodeTerminationHandlerEnabled` | **Use the AWS Node Termination Handler app** - Defaults to true. Whether or not to enable the Auto Scaling Groups lifecycle hooks and use the node-termination-handler app (NTH) to manage the termination of EC2 instances.|**Type:** `[boolean]`
**Default:** `true`| -| `global.providerSpecific.reducedInstanceProfileIamPermissionsForWorkers` | **Use reduced IAM permissions on worker nodes instance profile** - Defaults to true. If something breaks, this can temporarily be disabled in order to bring certain IAM permissions (e.g. EC2) back for the worker nodes' IAM instance profile. Applications must use [IRSA](https://docs.giantswarm.io/tutorials/access-management/iam-roles-for-service-accounts/) to authenticate with the AWS API instead of falling back to the instance profile.|**Type:** `[boolean]`
**Default:** `true`| | `global.providerSpecific.region` | **Region**|**Type:** `[string]`
| ### Apps diff --git a/helm/cluster-aws/templates/_aws_cluster.tpl b/helm/cluster-aws/templates/_aws_cluster.tpl index ae16f7f48..a7c54ee61 100644 --- a/helm/cluster-aws/templates/_aws_cluster.tpl +++ b/helm/cluster-aws/templates/_aws_cluster.tpl @@ -255,11 +255,9 @@ spec: {{- end }} sshKeyName: ssh-key s3Bucket: - controlPlaneIAMInstanceProfile: control-plane-{{ include "resource.default.name" $ }} + controlPlaneIAMInstanceProfile: {{ include "resource.default.name" $ }}-control-plane name: {{ include "aws-region" . }}-capa-{{ include "resource.default.name" $ }} nodesIAMInstanceProfiles: - {{- range $name, $value := .Values.global.nodePools | default .Values.cluster.providerIntegration.workers.defaultNodePools }} - - nodes-{{ $name }}-{{ include "resource.default.name" $ }} - {{- end }} + - {{ include "resource.default.name" $ }}-worker region: {{ include "aws-region" . }} {{ end }} diff --git a/helm/cluster-aws/templates/_control_plane.tpl b/helm/cluster-aws/templates/_control_plane.tpl index 5084a5a40..c91f4b9e7 100644 --- a/helm/cluster-aws/templates/_control_plane.tpl +++ b/helm/cluster-aws/templates/_control_plane.tpl @@ -31,7 +31,7 @@ nonRootVolumes: rootVolume: size: {{ .Values.global.controlPlane.rootVolumeSizeGB }} type: gp3 -iamInstanceProfile: control-plane-{{ include "resource.default.name" $ }} +iamInstanceProfile: {{ include "resource.default.name" $ }}-control-plane {{- if .Values.global.controlPlane.additionalSecurityGroups }} additionalSecurityGroups: {{- toYaml .Values.global.controlPlane.additionalSecurityGroups | nindent 2 }} diff --git a/helm/cluster-aws/templates/_karpenter_machine_pools.tpl b/helm/cluster-aws/templates/_karpenter_machine_pools.tpl index 02a3d7206..954e14ca1 100644 --- a/helm/cluster-aws/templates/_karpenter_machine_pools.tpl +++ b/helm/cluster-aws/templates/_karpenter_machine_pools.tpl @@ -7,9 +7,6 @@ metadata: labels: giantswarm.io/machine-pool: {{ include "resource.default.name" $ }}-{{ $name }} {{- include "labels.common" $ | nindent 4 }} - {{- if $.Values.global.providerSpecific.reducedInstanceProfileIamPermissionsForWorkers }} - alpha.aws.giantswarm.io/reduced-instance-permissions-workers: "true" - {{- end }} app.kubernetes.io/version: {{ $.Chart.Version | quote }} name: {{ include "resource.default.name" $ }}-{{ $name }} namespace: {{ $.Release.Namespace }} @@ -37,7 +34,7 @@ spec: volumeSize: {{ $value.logVolumeSizeGB | default 30}}Gi volumeType: gp3 deleteOnTermination: true - instanceProfile: nodes-{{ $name }}-{{ include "resource.default.name" $ }} + instanceProfile: {{ include "resource.default.name" $ }}-worker metadataOptions: {{- if eq $.Values.global.connectivity.cilium.ipamMode "eni" }} httpPutResponseHopLimit: 2 diff --git a/helm/cluster-aws/templates/_machine_pools.tpl b/helm/cluster-aws/templates/_machine_pools.tpl index e71b5ef1a..96a21447a 100644 --- a/helm/cluster-aws/templates/_machine_pools.tpl +++ b/helm/cluster-aws/templates/_machine_pools.tpl @@ -7,9 +7,6 @@ metadata: labels: giantswarm.io/machine-pool: {{ include "resource.default.name" $ }}-{{ $name }} {{- include "labels.common" $ | nindent 4 }} - {{- if $.Values.global.providerSpecific.reducedInstanceProfileIamPermissionsForWorkers }} - alpha.aws.giantswarm.io/reduced-instance-permissions-workers: "true" - {{- end }} {{- if eq $.Values.global.connectivity.cilium.ipamMode "eni" }} alpha.aws.giantswarm.io/ipam-mode: "eni" {{- end }} @@ -50,7 +47,7 @@ spec: {{- else }} {{- include "imageLookupParameters" $ | nindent 4 }} {{- end }} - iamInstanceProfile: nodes-{{ $name }}-{{ include "resource.default.name" $ }} + iamInstanceProfile: {{ include "resource.default.name" $ }}-worker instanceType: {{ $value.instanceType | default "r6i.xlarge" }} rootVolume: size: {{ $value.rootVolumeSizeGB | default 8 }} diff --git a/helm/cluster-aws/templates/crossplane-iam-role-control-plane.yaml b/helm/cluster-aws/templates/crossplane-iam-role-control-plane.yaml new file mode 100644 index 000000000..bdec694f2 --- /dev/null +++ b/helm/cluster-aws/templates/crossplane-iam-role-control-plane.yaml @@ -0,0 +1,203 @@ +--- +apiVersion: iam.aws.upbound.io/v1beta1 +kind: Role +metadata: + name: {{ include "resource.default.name" $ }}-control-plane + labels: + {{- include "labels.common" $ | nindent 4 }} + app.kubernetes.io/version: {{ .Chart.Version | quote }} +spec: + forProvider: + assumeRolePolicy: | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com{{- if hasPrefix "cn-" (include "aws-region" .) }}.cn{{- end }}" + }, + "Action": "sts:AssumeRole" + } + ] + } + tags: + managed-by: "cluster-aws" + giantswarm.io/cluster: {{ include "resource.default.name" $ }} + giantswarm.io/installation: {{ .Values.global.managementCluster }} + {{- if .Values.global.providerSpecific.additionalResourceTags -}}{{- toYaml .Values.global.providerSpecific.additionalResourceTags | nindent 4 }}{{- end}} + providerConfigRef: + name: {{ include "resource.default.name" $ }} +--- +apiVersion: iam.aws.upbound.io/v1beta1 +kind: RolePolicy +metadata: + name: {{ include "resource.default.name" $ }}-control-plane + labels: + cluster.x-k8s.io/cluster-name: {{ include "resource.default.name" $ }} +spec: + forProvider: + roleRef: + name: {{ include "resource.default.name" $ }}-control-plane + policy: | + { + "Version": "2012-10-17", + "Statement": [ + { + "Action": "elasticloadbalancing:*", + "Resource": "*", + "Effect": "Allow" + }, + { + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeAutoScalingInstances", + "autoscaling:DescribeTags", + "autoscaling:DescribeLaunchConfigurations", + "ec2:DescribeLaunchTemplateVersions" + ], + "Resource": "*", + "Effect": "Allow" + }, + { + "Action": [ + "ecr:GetAuthorizationToken", + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:GetRepositoryPolicy", + "ecr:DescribeRepositories", + "ecr:ListImages", + "ecr:BatchGetImage" + ], + "Resource": "*", + "Effect": "Allow" + }, + { + "Action": [ + "ec2:AssignPrivateIpAddresses", + "ec2:AttachNetworkInterface", + "ec2:CreateNetworkInterface", + "ec2:DeleteNetworkInterface", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypes", + "ec2:DescribeTags", + "ec2:DescribeNetworkInterfaces", + "ec2:DetachNetworkInterface", + "ec2:ModifyNetworkInterfaceAttribute", + "ec2:UnassignPrivateIpAddresses" + ], + "Resource": "*", + "Effect": "Allow" + }, + { + "Action": [ + "autoscaling:DescribeAutoScalingGroups", + "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeTags", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeInstances", + "ec2:DescribeImages", + "ec2:DescribeRegions", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeVolumes", + "ec2:CreateSecurityGroup", + "ec2:CreateTags", + "ec2:CreateVolume", + "ec2:ModifyInstanceAttribute", + "ec2:ModifyVolume", + "ec2:AttachVolume", + "ec2:DescribeVolumesModifications", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:CreateRoute", + "ec2:DeleteRoute", + "ec2:DeleteSecurityGroup", + "ec2:DeleteVolume", + "ec2:DetachVolume", + "ec2:RevokeSecurityGroupIngress", + "ec2:DescribeVpcs", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:AttachLoadBalancerToSubnets", + "elasticloadbalancing:ApplySecurityGroupsToLoadBalancer", + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateLoadBalancerPolicy", + "elasticloadbalancing:CreateLoadBalancerListeners", + "elasticloadbalancing:ConfigureHealthCheck", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:DeleteLoadBalancerListeners", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DetachLoadBalancerFromSubnets", + "elasticloadbalancing:DeregisterInstancesFromLoadBalancer", + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:RegisterInstancesWithLoadBalancer", + "elasticloadbalancing:SetLoadBalancerPoliciesForBackendServer", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:CreateTargetGroup", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeLoadBalancerPolicies", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:ModifyListener", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:SetLoadBalancerPoliciesOfListener", + "iam:CreateServiceLinkedRole", + "kms:DescribeKey" + ], + "Resource": [ + "*" + ], + "Effect": "Allow" + }, + { + "Action": [ + "secretsmanager:GetSecretValue", + "secretsmanager:DeleteSecret" + ], + "Resource": "arn:*:secretsmanager:*:*:secret:aws.cluster.x-k8s.io/*", + "Effect": "Allow" + } + ] + } + providerConfigRef: + name: {{ include "resource.default.name" $ }} +--- +apiVersion: iam.aws.upbound.io/v1beta1 +kind: RolePolicyAttachment +metadata: + name: {{ include "resource.default.name" $ }}-control-plane + labels: + {{- include "labels.common" $ | nindent 4 }} + app.kubernetes.io/version: {{ .Chart.Version | quote }} +spec: + forProvider: + roleRef: + name: {{ include "resource.default.name" $ }}-control-plane + policyArnRef: + name: {{ include "resource.default.name" $ }}-control-plane + providerConfigRef: + name: {{ include "resource.default.name" $ }} + +--- +apiVersion: iam.aws.upbound.io/v1beta1 +kind: InstanceProfile +metadata: + name: {{ include "resource.default.name" $ }}-control-plane + labels: + {{- include "labels.common" $ | nindent 4 }} + app.kubernetes.io/version: {{ .Chart.Version | quote }} +spec: + forProvider: + role: {{ include "resource.default.name" $ }}-control-plane + tags: + managed-by: "cluster-aws" + giantswarm.io/cluster: {{ include "resource.default.name" $ }} + giantswarm.io/installation: {{ .Values.global.managementCluster }} + {{- if .Values.global.providerSpecific.additionalResourceTags -}}{{- toYaml .Values.global.providerSpecific.additionalResourceTags | nindent 4 }}{{- end}} + providerConfigRef: + name: {{ include "resource.default.name" $ }} diff --git a/helm/cluster-aws/templates/crossplane-iam-role-worker.yaml b/helm/cluster-aws/templates/crossplane-iam-role-worker.yaml new file mode 100644 index 000000000..95475aec4 --- /dev/null +++ b/helm/cluster-aws/templates/crossplane-iam-role-worker.yaml @@ -0,0 +1,120 @@ +--- +apiVersion: iam.aws.upbound.io/v1beta1 +kind: Role +metadata: + name: {{ include "resource.default.name" $ }}-worker + labels: + {{- include "labels.common" $ | nindent 4 }} + app.kubernetes.io/version: {{ .Chart.Version | quote }} +spec: + forProvider: + assumeRolePolicy: | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com{{- if hasPrefix "cn-" (include "aws-region" .) }}.cn{{- end }}" + }, + "Action": "sts:AssumeRole" + } + ] + } + tags: + managed-by: "cluster-aws" + giantswarm.io/cluster: {{ include "resource.default.name" $ }} + giantswarm.io/installation: {{ .Values.global.managementCluster }} + {{- if .Values.global.providerSpecific.additionalResourceTags -}}{{- toYaml .Values.global.providerSpecific.additionalResourceTags | nindent 4 }}{{- end}} + providerConfigRef: + name: {{ include "resource.default.name" $ }} +--- +apiVersion: iam.aws.upbound.io/v1beta1 +kind: RolePolicy +metadata: + name: {{ include "resource.default.name" $ }}-worker + labels: + {{- include "labels.common" $ | nindent 4 }} + app.kubernetes.io/version: {{ .Chart.Version | quote }} +spec: + forProvider: + roleRef: + name: {{ include "resource.default.name" $ }}-worker + policy: | + { + "Version": "2012-10-17", + "Statement": [ + {{- if eq .Values.global.connectivity.cilium.ipamMode "eni" }} + { + "Action": [ + "ec2:AssignPrivateIpAddresses", + "ec2:AttachNetworkInterface", + "ec2:CreateNetworkInterface", + "ec2:CreateTags", + "ec2:DeleteNetworkInterface", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypes", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeTags", + "ec2:DescribeVpcs", + "ec2:ModifyNetworkInterfaceAttribute", + "ec2:UnassignPrivateIpAddresses" + ], + "Resource": "*", + "Effect": "Allow" + }, + {{- end }} + { + "Action": [ + "ecr:BatchCheckLayerAvailability", + "ecr:BatchGetImage", + "ecr:DescribeRepositories", + "ecr:GetAuthorizationToken", + "ecr:GetDownloadUrlForLayer", + "ecr:GetRepositoryPolicy", + "ecr:ListImages" + ], + "Resource": "*", + "Effect": "Allow" + } + ] + } + providerConfigRef: + name: {{ include "resource.default.name" $ }} +--- +apiVersion: iam.aws.upbound.io/v1beta1 +kind: RolePolicyAttachment +metadata: + name: {{ include "resource.default.name" $ }}-worker + labels: + {{- include "labels.common" $ | nindent 4 }} + app.kubernetes.io/version: {{ .Chart.Version | quote }} +spec: + forProvider: + roleRef: + name: {{ include "resource.default.name" $ }}-worker + policyArnRef: + name: {{ include "resource.default.name" $ }}-worker + providerConfigRef: + name: {{ include "resource.default.name" $ }} +--- +apiVersion: iam.aws.upbound.io/v1beta1 +kind: InstanceProfile +metadata: + name: {{ include "resource.default.name" $ }}-worker + labels: + {{- include "labels.common" $ | nindent 4 }} + app.kubernetes.io/version: {{ .Chart.Version | quote }} +spec: + forProvider: + role: {{ include "resource.default.name" $ }}-worker + tags: + managed-by: "cluster-aws" + giantswarm.io/cluster: {{ include "resource.default.name" $ }} + giantswarm.io/installation: {{ .Values.global.managementCluster }} + {{- if .Values.global.providerSpecific.additionalResourceTags -}}{{- toYaml .Values.global.providerSpecific.additionalResourceTags | nindent 4 }}{{- end}} + providerConfigRef: + name: {{ include "resource.default.name" $ }} diff --git a/helm/cluster-aws/templates/karpenter-crossplane-resources-helmrelease.yaml b/helm/cluster-aws/templates/karpenter-crossplane-resources-helmrelease.yaml index ca41363a3..7d9eb889e 100644 --- a/helm/cluster-aws/templates/karpenter-crossplane-resources-helmrelease.yaml +++ b/helm/cluster-aws/templates/karpenter-crossplane-resources-helmrelease.yaml @@ -30,6 +30,8 @@ spec: upgrade: remediation: retries: -1 + values: + workersIamRole: arn:{{ include "aws-partition" $ }}:iam::{{ include "aws-account-id" $ }}:role/{{ include "resource.default.name" $ }}-worker valuesFrom: - kind: ConfigMap name: {{ include "resource.default.name" $ }}-crossplane-config diff --git a/helm/cluster-aws/templates/required.yaml b/helm/cluster-aws/templates/required.yaml index 53e451c4a..571c640cb 100644 --- a/helm/cluster-aws/templates/required.yaml +++ b/helm/cluster-aws/templates/required.yaml @@ -3,4 +3,3 @@ {{- $_ := required "global.connectivity.cilium.ipamMode is required" .Values.global.connectivity.cilium.ipamMode }} {{- $_ := required "global.connectivity.network.pods.cidrBlocks is required" .Values.global.connectivity.network.pods.cidrBlocks }} {{- $_ := required "You must provide an existing organization name in .global.metadata.organization" .Values.global.metadata.organization }} -{{- $_ := required "global.providerSpecific.reducedInstanceProfileIamPermissionsForWorkers is required" $.Values.global.providerSpecific.reducedInstanceProfileIamPermissionsForWorkers }} diff --git a/helm/cluster-aws/values.schema.json b/helm/cluster-aws/values.schema.json index 2ad1b58bb..dd870f021 100644 --- a/helm/cluster-aws/values.schema.json +++ b/helm/cluster-aws/values.schema.json @@ -2482,12 +2482,6 @@ "description": "Defaults to true. Whether or not to enable the Auto Scaling Groups lifecycle hooks and use the node-termination-handler app (NTH) to manage the termination of EC2 instances.", "default": true }, - "reducedInstanceProfileIamPermissionsForWorkers": { - "type": "boolean", - "title": "Use reduced IAM permissions on worker nodes instance profile", - "description": "Defaults to true. If something breaks, this can temporarily be disabled in order to bring certain IAM permissions (e.g. EC2) back for the worker nodes' IAM instance profile. Applications must use [IRSA](https://docs.giantswarm.io/tutorials/access-management/iam-roles-for-service-accounts/) to authenticate with the AWS API instead of falling back to the instance profile.", - "default": true - }, "region": { "type": "string", "title": "Region" diff --git a/helm/cluster-aws/values.yaml b/helm/cluster-aws/values.yaml index ecb93c1db..fbf1f4645 100644 --- a/helm/cluster-aws/values.yaml +++ b/helm/cluster-aws/values.yaml @@ -406,6 +406,5 @@ global: httpTokens: required irsaCrossplane: true nodeTerminationHandlerEnabled: true - reducedInstanceProfileIamPermissionsForWorkers: true release: {} internal: {}